diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml index bb5bdba37ae..0febb6f8948 100644 --- a/.github/workflows/adhoc-mac-build.yml +++ b/.github/workflows/adhoc-mac-build.yml @@ -6,7 +6,9 @@ name: Adhoc macOS + Windows Dev Build # # Deliberately narrow scope: # - macOS and Windows desktop installers. Linux keeps using RC/stable. -# - No tests, no lint, no e2e. PR CI and release-cut remain the gates. +# - No tests, no lint, no e2e. PR CI and release-cut remain the gates. The one +# exception is the orcad template: like release-cut, it is merged from the +# node-server lanes that build and qualify each SSH target's slot. # - macOS is signed and notarized so TCC grants survive updates. # - Windows is unsigned; the published release notes explain the one-time # SmartScreen/manual-install requirement. @@ -81,21 +83,116 @@ env: ADHOC_RETAIN_DAYS: 30 jobs: + # Why: every job below checks out its own copy, and a branch name read per job builds whatever + # the branch points at when that job starts. A push mid-run then mixes commits, e.g. slot lanes + # from one commit merged by a template step from the next. Resolving once pins the whole run. + # The mac job still vets this commit's reachability before anything is signed. + resolve-ref: + if: github.repository == 'stablyai/orca' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + sha: ${{ steps.resolve.outputs.sha }} + steps: + - name: Resolve the requested ref to one commit + id: resolve + shell: bash + env: + REQUESTED_REF: ${{ inputs.ref || github.ref_name }} + REPO_URL: https://github.com/${{ github.repository }} + run: | + set -euo pipefail + case "$REQUESTED_REF" in + refs/pull/*|pull/*) + echo "::error::Refusing to build PR ref '$REQUESTED_REF'; push the code to a branch of stablyai/orca instead." + exit 1 + ;; + esac + if [[ "$REQUESTED_REF" =~ ^[0-9a-f]{40}$ ]]; then + sha="$REQUESTED_REF" + else + # Branch first, as the mac job's vet does; a peeled annotated tag names its commit. + refs="$(git ls-remote "$REPO_URL" "refs/heads/$REQUESTED_REF" "refs/tags/$REQUESTED_REF" "refs/tags/$REQUESTED_REF^{}")" + sha="" + for name in "refs/heads/$REQUESTED_REF" "refs/tags/$REQUESTED_REF^{}" "refs/tags/$REQUESTED_REF"; do + sha="$(awk -v ref="$name" '$2 == ref { print $1; exit }' <<<"$refs")" + [[ -n "$sha" ]] && break + done + fi + if [[ -z "$sha" ]]; then + echo "::error::'$REQUESTED_REF' is not a branch, tag, or full commit SHA of stablyai/orca." + exit 1 + fi + echo "Resolved $REQUESTED_REF -> $sha" + echo "sha=$sha" >>"$GITHUB_OUTPUT" + # Why its own job: this package ships relays for Windows SSH hosts, and only a # Windows runner compiles the addon that launches one outside sshd's job. # Why the unvetted ref is safe here: this job holds no secrets, and the mac job # vets the same ref before it downloads anything, so fork code never gets signed. relay-windows-process-tree: - if: github.repository == 'stablyai/orca' + needs: resolve-ref permissions: contents: read uses: ./.github/workflows/relay-windows-process-tree.yml with: - ref: ${{ inputs.ref || github.ref_name }} + ref: ${{ needs.resolve-ref.outputs.sha }} + + # Why: a branch cut before the orcad template landed has none to build or ship. + orcad-template-support: + needs: resolve-ref + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + ships: ${{ steps.detect.outputs.ships }} + steps: + - name: Checkout the template packager only + uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve-ref.outputs.sha }} + sparse-checkout: | + /config/scripts/packaged-orcad-template.cjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Detect whether the ref ships the orcad template + id: detect + shell: bash + run: | + if [[ -f config/scripts/packaged-orcad-template.cjs ]]; then + echo "ships=true" >>"$GITHUB_OUTPUT" + else + echo "ships=false" >>"$GITHUB_OUTPUT" + echo "::notice::This ref predates the orcad template; the build ships without it." + fi + + # Design D2, as release-cut does: every desktop build ships the orcad template (server JS plus + # every target's addons), merged from the node-server lanes that qualify each slot at this ref. + # Without it an adhoc build cannot deploy managed orcad to an SSH host. No secrets, like the + # relay job, and the mac job vets the ref before anything is signed. + orcad-template: + needs: [resolve-ref, orcad-template-support] + if: needs.orcad-template-support.outputs.ships == 'true' + permissions: + contents: read + uses: ./.github/workflows/node-server-tests.yml + with: + ref: ${{ needs.resolve-ref.outputs.sha }} + build_template: true build-adhoc-mac: - needs: relay-windows-process-tree - if: github.repository == 'stablyai/orca' + needs: [resolve-ref, relay-windows-process-tree, orcad-template-support, orcad-template] + # Why not the implicit success(): a ref without the orcad template skips that job on purpose. + if: >- + !cancelled() && github.repository == 'stablyai/orca' && + needs.relay-windows-process-tree.result == 'success' && + needs.orcad-template-support.result == 'success' && + (needs.orcad-template.result == 'success' || + (needs.orcad-template.result == 'skipped' && + needs.orcad-template-support.outputs.ships == 'false')) # Why an environment: it gives the signing/notary/App secrets somewhere to # live that a stale copy of this workflow on an old branch cannot reach. # Referencing it is a no-op until repo settings give it teeth; the intended @@ -109,6 +206,7 @@ jobs: version: ${{ steps.adhoc.outputs.version }} head_sha: ${{ steps.adhoc.outputs.head_sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} + ships_orcad_template: ${{ needs.orcad-template-support.outputs.ships }} runs-on: blacksmith-6vcpu-macos-15 # Why 150: it must exceed the worst case the retry budgets below can produce # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or @@ -128,7 +226,8 @@ jobs: id: vetted shell: bash env: - REQUESTED_REF: ${{ inputs.ref || github.ref_name }} + # The commit resolve-ref pinned for every job; vetting it keeps the reachability test. + REQUESTED_REF: ${{ needs.resolve-ref.outputs.sha }} REPO_URL: https://github.com/${{ github.repository }} run: | set -euo pipefail @@ -292,6 +391,14 @@ jobs: # sshd's job for a standard user on a Windows SSH host. ORCA_REQUIRE_RELAY_NATIVE_ADDONS: x64,arm64 + # After the app build so nothing that cleans out/ can drop it; electron-builder ships it. + - name: Download the orcad deployment template + if: needs.orcad-template-support.outputs.ships == 'true' + uses: actions/download-artifact@v8 + with: + name: orcad-template + path: out/orcad-template + # Why the token is minted here and not at the top: installation tokens live # one hour, everything before this point writes nothing, and the notary round # trip inside the publish step can be tens of minutes. Minting after the build @@ -366,6 +473,8 @@ jobs: GH_TOKEN: ${{ steps.app_token.outputs.token }} ORCA_ADHOC_BUILD_VERSION: ${{ steps.adhoc.outputs.version }} ORCA_BUILD_COMMIT: ${{ steps.adhoc.outputs.commit }} + # beforePack and afterPack fail the package when the template is absent. + ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ needs.orcad-template-support.outputs.ships == 'true' && '1' || '' }} CSC_LINK: ${{ secrets.MAC_CERTS }} CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }} # Why all three: electron-builder's notarize step authenticates to the @@ -514,3 +623,4 @@ jobs: tag: ${{ needs.build-adhoc-mac.outputs.tag }} ref: ${{ needs.build-adhoc-mac.outputs.head_sha }} version: ${{ needs.build-adhoc-mac.outputs.version }} + orcad_template: ${{ needs.build-adhoc-mac.outputs.ships_orcad_template == 'true' }} diff --git a/.github/workflows/dev-channel-win-build.yml b/.github/workflows/dev-channel-win-build.yml index 3f8e162e073..04e3b84ff7f 100644 --- a/.github/workflows/dev-channel-win-build.yml +++ b/.github/workflows/dev-channel-win-build.yml @@ -58,6 +58,11 @@ on: description: Version to package, without the leading v required: true type: string + orcad_template: + description: Ship the orcad-template artifact the calling run built (adhoc does; hourly and daily do not yet) + required: false + type: boolean + default: false workflow_dispatch: inputs: channel: @@ -264,6 +269,14 @@ jobs: # is correct for unvetted artifacts. Same as the mac dev channels. ORCA_DIAGNOSTICS_TOKEN_URL: https://www.onorca.dev/diagnostics/token + # After the app build so nothing that cleans out/ can drop it; electron-builder ships it. + - name: Download the orcad deployment template + if: inputs.orcad_template + uses: actions/download-artifact@v8 + with: + name: orcad-template + path: out/orcad-template + # Why the token is minted here and not at the top: installation tokens live # one hour and nothing before this point writes anything. - name: Mint dev channel repo token @@ -301,6 +314,8 @@ jobs: env: GH_TOKEN: ${{ steps.app_token.outputs.token }} ORCA_BUILD_COMMIT: ${{ inputs.ref }} + # beforePack and afterPack fail the package when the template is absent. + ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ inputs.orcad_template && '1' || '' }} # Why: electron-publish refuses to upload into a release published more # than two hours ago (gitHubPublisher.getOrCreateRelease). The mac leg # publishes the draft live as soon as *it* finishes, so a slow notary diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a862d0d9395..be2386e2447 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -341,7 +341,11 @@ jobs: . != "tests/e2e/paired-startup-exec-readiness.spec.ts" and . != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and . != "tests/e2e/ssh-localhost.spec.ts" and - . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" + . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" and + . != "tests/e2e/orcad-serve-mode-switch.spec.ts" and + . != "tests/e2e/ssh-orcad-auto-convert.spec.ts" and + . != "tests/e2e/windows-missing-appdata-startup.spec.ts" and + . != "tests/e2e/ssh-orcad-idle-exit.spec.ts" )' <<<"$TEST_FILES_JSON" > "$RUNNER_TEMP/general-e2e-specs" fi mapfile -t TEST_FILES < "$RUNNER_TEMP/general-e2e-specs" @@ -510,6 +514,157 @@ jobs: ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1' run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts + orcad-serve-mode-switch: + name: orca serve Electron/orcad mode switch (D7) + needs: [build, prepare-native-cache] + if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - name: Install headless tools + run: sudo apt-get update && sudo apt-get install -y build-essential ripgrep xvfb openbox x11-utils + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + - uses: actions/download-artifact@v8 + with: + name: e2e-build-out + path: out/ + # The packaged orcad slot the T6-11 launcher runs: its own node-pty under the pinned Node. + # The template is what `orca serve`'s default selection materializes that slot from. + - name: Build this runner's orcad slot and template + run: | + slot="$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)" + pnpm build:orcad-prebuilds + pnpm build:orcad-prebuilds --require-slots "$slot" + pnpm build:orcad + node config/scripts/build-orcad-template.mjs --targets "$slot" + - name: Switch serve hosts on one profile + env: + SKIP_BUILD: '1' + ORCA_E2E_ORCAD_SERVE: '1' + ORCA_E2E_FORWARD_APP_LOGS: '1' + run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/orcad-serve-mode-switch.spec.ts --project=electron-headless --workers=1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: orcad-serve-mode-switch-traces + path: test-results/ + retention-days: 7 + if-no-files-found: ignore + + # #24979 on a real host: a relay-era Docker host converts to managed orcad on connect, and a managed + # orcad idles out and restarts on the next connect. Needs the + # orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds. + orcad-auto-convert-docker: + name: ssh host auto-converts to managed orcad (Docker) + needs: [build, prepare-native-cache] + if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-auto-convert.spec.ts') || contains(inputs.test_files, 'tests/e2e/ssh-orcad-idle-exit.spec.ts') + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - name: Install headless tools + run: sudo apt-get update && sudo apt-get install -y build-essential ripgrep xvfb openbox x11-utils + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + - uses: actions/download-artifact@v8 + with: + name: e2e-build-out + path: out/ + # Built to a path the app does not look at by default, so the relay phase has no template. + - name: Build the linux-x64-glibc orcad template + run: | + pnpm build:orcad-prebuilds + pnpm build:orcad-prebuilds --require-slots linux-x64-glibc + pnpm build:orcad + node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc + mv out/orcad-template "$RUNNER_TEMP/orcad-convert-template" + - name: Convert a relay-era Docker host + env: + SKIP_BUILD: '1' + ORCA_E2E_SSH_DOCKER: '1' + ORCA_E2E_ORCAD_CONVERT_HOST: docker + ORCA_E2E_FORWARD_APP_LOGS: '1' + ORCA_RELAY_PATH: ${{ github.workspace }}/out/relay + run: | + export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template" + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad-auto-convert.spec.ts tests/e2e/ssh-orcad-idle-exit.spec.ts --project=electron-headless --workers=1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: orcad-auto-convert-docker-traces + path: test-results/ + retention-days: 7 + if-no-files-found: ignore + + # D7 on Windows: both hosts share \daemon and so one daemon pipe. Built here rather than + # from the Linux e2e build because the slot, template and addons are Windows-native. Also runs the + # missing-AppData startup check, which needs the same Windows e2e build. + orcad-serve-mode-switch-windows: + name: orca serve on Windows (D7 mode switch, missing AppData) + if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') || contains(inputs.test_files, 'tests/e2e/windows-missing-appdata-startup.spec.ts') + runs-on: windows-2022 + timeout-minutes: 45 + env: + NODE_OPTIONS: --max-old-space-size=4096 + ORCA_BACKGROUND_LAUNCH: '1' + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + - uses: ./.github/actions/prepare-orcad-prebuilds + with: + resolve-windows-cache: 'true' + restore-windows-cache: 'true' + - name: Build the e2e app, CLI, orcad slot and template + shell: bash + run: | + pnpm run build:relay + pnpm exec electron-vite build --mode e2e + pnpm run build:cli + pnpm build:orcad + node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=x64 + node config/scripts/build-orcad-template.mjs --targets win32-x64 + # A profile-less Windows session has no roaming AppData; Electron 43 used to crash natively there. + - name: Start serve with no AppData folder + if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/windows-missing-appdata-startup.spec.ts') + env: + SKIP_BUILD: '1' + ORCA_E2E_FORWARD_APP_LOGS: '1' + run: pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/windows-missing-appdata-startup.spec.ts --project=electron-headless --workers=1 + - name: Switch serve hosts on one profile + if: ${{ !cancelled() && (inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts')) }} + env: + SKIP_BUILD: '1' + ORCA_E2E_ORCAD_SERVE: '1' + ORCA_E2E_FORWARD_APP_LOGS: '1' + ORCA_E2E_PRESERVE_PROFILE_LOGS_DIR: ${{ github.workspace }}\test-results\profile-logs + run: pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/orcad-serve-mode-switch.spec.ts --project=electron-headless --workers=1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: orcad-serve-mode-switch-windows-traces + path: test-results/ + retention-days: 7 + if-no-files-found: ignore + ssh-localhost: name: localhost SSH terminal and hooks needs: [build, prepare-native-cache] diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cd6d63011f3..de3c64d0c79 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1269,6 +1269,7 @@ jobs: src/main/cursor/hook-service.test.ts src/main/orca-profiles/profile-index-store.test.ts src/main/startup/windows-install-dir-acl-repair.win32.test.ts + src/main/startup/windows-app-data-path.test.ts src/main/runtime/repo-worktree-admin-fingerprint.test.ts src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts src/shared/secure-file-fsync-flags.test.ts diff --git a/.github/workflows/relay-windows-process-tree.yml b/.github/workflows/relay-windows-process-tree.yml index 4aadd22a692..a61f2fae8e9 100644 --- a/.github/workflows/relay-windows-process-tree.yml +++ b/.github/workflows/relay-windows-process-tree.yml @@ -60,6 +60,23 @@ jobs: node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=x64 node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=arm64 + # orcad's instance lock and stop proof read one PID's creation time through this addon on + # Windows SSH hosts; prove the x64 binary this runner can load answers it, and answers + # nothing for a PID that does not exist rather than a stale or zero time. + - name: Assert the addon reads process creation time + shell: bash + run: | + node -e " + const { assertWindowsProcessTreeCreationTime } = require('./config/scripts/windows-process-tree-creation-time.cjs') + const addon = require('./.build/windows-process-tree/x64/windows-process-tree.node') + assertWindowsProcessTreeCreationTime({ module: addon }) + const own = addon.getProcessCreationTime(process.pid) + const started = Date.now() - process.uptime() * 1000 + if (typeof own !== 'number' || Math.abs(own - started) > 5000) throw new Error('creation time ' + own + ' vs ' + started) + if (addon.getProcessCreationTime(0x7ffffff0) !== undefined) throw new Error('a missing PID must read undefined') + console.log('creation time ok:', own) + " + - name: Upload relay addons uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/ssh-windows-hosts.yml b/.github/workflows/ssh-windows-hosts.yml index f822dd9a53d..2fc86af5115 100644 --- a/.github/workflows/ssh-windows-hosts.yml +++ b/.github/workflows/ssh-windows-hosts.yml @@ -31,8 +31,18 @@ on: - 'config/scripts/relay-windows-process-tree-prepared-addon*.mjs' - 'config/scripts/windows-process-tree-gyp-rebuild.mjs' - 'src/shared/relay-windows-breakaway-launch.ts' + - 'src/shared/windows-breakaway-launch*.ts' + - 'src/main/ipc/ssh-host-server-*.ts' - '!src/**/*.test.ts' - 'src/main/ssh/ssh-relay-windows-host-lane.test.ts' + - 'src/main/ssh/orcad-windows-host-lane.test.ts' + - 'tests/e2e/ssh-orcad-auto-convert.spec.ts' + - 'tests/e2e/helpers/orcad-convert-host.ts' + - 'tests/e2e/helpers/orcad-convert-flow.ts' + - 'tests/e2e/helpers/orcad-upgrade-profile.ts' + - 'tests/e2e/ssh-orcad-windows-cli-matrix.spec.ts' + - 'tests/e2e/helpers/compiled-orca-cli.ts' + - 'tests/e2e/helpers/windows-host-orcad-processes.ts' - 'config/ci/windows-ssh-provider/**' - '.github/workflows/ssh-windows-hosts.yml' - '.github/actions/prepare-orcad-prebuilds/**' @@ -41,7 +51,7 @@ on: workflow_dispatch: inputs: cells: - description: Comma-separated cell ids from src/main/ssh/ssh-windows-host-cells.ts; empty runs all. + description: Comma-separated cell ids from src/main/ssh/ssh-windows-host-cells.ts; empty runs the default set (orcad-cli-* cells run only when named). required: false default: '' @@ -75,8 +85,9 @@ jobs: server: preview archive: OpenSSH-ARM64.zip runs-on: ${{ matrix.runner }} - # Installing the inbox capability alone can take several minutes on a fresh image. - timeout-minutes: 75 + # Installing the inbox capability alone can take several minutes on a fresh image; the CLI matrix + # cells launch the e2e app several times each. + timeout-minutes: ${{ contains(github.event.inputs.cells || '', 'orcad-cli') && 160 || 75 }} env: ORCA_BACKGROUND_LAUNCH: '1' ORCA_ISOLATED_SSH_CI: '1' @@ -85,16 +96,14 @@ jobs: with: persist-credentials: false # Keep complete server/test trees; missing future imports fail the unchanged builds. + # All of src: the orcad-convert cell builds the full e2e app and the bundled CLI. sparse-checkout: | .github config native resources tests - src/main - src/shared - src/relay - src/types + src - name: Self-test the provisioning scripts before touching the machine shell: pwsh run: | @@ -149,7 +158,7 @@ jobs: - wait: inbox-capability - name: Run the Windows host cells against a private ${{ matrix.server }} sshd shell: pwsh - timeout-minutes: 50 + timeout-minutes: ${{ contains(github.event.inputs.cells || '', 'orcad-cli') && 130 || 50 }} env: CELLS: ${{ github.event.inputs.cells || '' }} run: | @@ -158,7 +167,13 @@ jobs: $receipts=Join-Path $pwd '.build/ssh-windows-host-receipts' New-Item -ItemType Directory -Force -Path $receipts | Out-Null $cells=@($env:CELLS -split ',' | ForEach-Object {$_.Trim()} | Where-Object {$_}) - if(-not $cells.Count){$cells=@('pinned-cmd','pinned-powershell','legacy-opt-out')} + if(-not $cells.Count){$cells=@('pinned-cmd','pinned-powershell','legacy-opt-out','orcad-cmd','orcad-powershell')} + # One Windows host runs the app-level conversion; it is last because it switches native modules. + if(-not $env:CELLS -and '${{ matrix.arch }}' -eq 'x64' -and '${{ matrix.server }}' -eq 'inbox'){$cells+='orcad-convert'} + # App cells reach the managed server through an SSH local forward, which provisioning grants to the last accounts only. + $appCellIds=@('orcad-convert','orcad-cli-managed','orcad-cli-convert','orcad-cli-relay-kept') + $cells=@($cells | Where-Object {$appCellIds -notcontains $_})+@($cells | Where-Object {$appCellIds -contains $_}) + $forwarding=@($cells | Where-Object {$appCellIds -contains $_}).Count $archive='' $preparation='' if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation=Join-Path $receipts 'inbox-capability-preparation.json'} @@ -171,7 +186,7 @@ jobs: $callback={param($context) & (Join-Path $tools 'invoke-pinned-relay-cells.ps1') -SourceRoot $sourceRoot -Context $context -Target 'win32-${{ matrix.arch }}' -ReceiptRoot $receipts -Cells $cells }.GetNewClosure() - & (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -InboxPreparationReceipt $preparation -Accounts $cells.Count -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log') + & (Join-Path $tools 'preview-ssh/prove-preview-openssh.ps1') -Archive $archive -Arch '${{ matrix.arch }}' -Server '${{ matrix.server }}' -Receipt (Join-Path $receipts 'provider-server.json') -InboxPreparationReceipt $preparation -Accounts $cells.Count -ForwardingAccounts $forwarding -HiddenTools @('npm','npx','node-gyp','gcc','g++','cc','c++','make','cl','clang','clang++','msbuild','cmake') -HostCellProbe $callback 2>&1 | Tee-Object (Join-Path $receipts 'provision.log') - uses: actions/upload-artifact@v7 if: always() with: diff --git a/.github/workflows/win-orcad-serve-switch-e2e.yml b/.github/workflows/win-orcad-serve-switch-e2e.yml new file mode 100644 index 00000000000..19ed618dfb5 --- /dev/null +++ b/.github/workflows/win-orcad-serve-switch-e2e.yml @@ -0,0 +1,104 @@ +name: Windows packaged orcad serve switch E2E + +# Why: proves D7 on a real Windows install. The installed desktop app forks its terminal daemon +# from the relocated %LOCALAPPDATA%\Orca\daemon-host; `orca serve` then serves the same profile +# on orcad and must adopt that daemon, and the desktop must reattach the terminal afterwards and +# still write its settings under orcad's data-root ACL. Unpackaged e2e cannot exercise the +# relocation, which only packaged builds do. A CI runner is the only safe place to install. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - 'src/main/daemon/**' + - 'src/main/orcad/**' + - 'src/cli/runtime/**' + - 'src/shared/orcad-local-serve-selection.ts' + - 'tests/tools/win-update-e2e/**' + - '.github/workflows/win-orcad-serve-switch-e2e.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: win-orcad-serve-switch-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + serve-switch: + name: packaged Windows Electron/orcad serve switch (D7) + if: github.event_name != 'pull_request' || github.event.pull_request.draft != true + runs-on: windows-2022 + timeout-minutes: 75 + env: + NODE_OPTIONS: --max-old-space-size=4096 + ORCA_BACKGROUND_LAUNCH: '1' + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: node + + # Same key inputs as win-update-survival-e2e: a harness-only edit skips the installer build. + - name: Cache branch installer + id: cache-installer + uses: actions/cache/restore@v4 + with: + path: dist/orca-windows-setup.exe + key: serve-switch-installer-${{ hashFiles('src/**', 'config/**', 'native/**', 'resources/**', 'mobile/**', 'patches/**', 'package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml') }} + + - uses: ./.github/actions/install-mobile-dependencies + if: steps.cache-installer.outputs.cache-hit != 'true' + + # Before the template exists: packaging verifies a present template for every target. + - name: Build Windows installer (unsigned) + if: steps.cache-installer.outputs.cache-hit != 'true' + shell: bash + run: | + node config/scripts/ensure-native-runtime.mjs --runtime=electron + pnpm run build:desktop + pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never + + - name: Cache the built installer + if: steps.cache-installer.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: dist/orca-windows-setup.exe + key: ${{ steps.cache-installer.outputs.cache-primary-key }} + + # The win32-x64 template the harness drops into the install, which is where serve looks. + - uses: ./.github/actions/prepare-orcad-prebuilds + with: + resolve-windows-cache: 'true' + restore-windows-cache: 'true' + - name: Build the win32-x64 orcad template + shell: bash + run: | + node config/scripts/ensure-native-runtime.mjs --runtime=node + pnpm build:orcad + node config/scripts/build-windows-process-tree-relay-addon.mjs --arch=x64 + node config/scripts/build-orcad-template.mjs --targets win32-x64 + + - name: Switch serve hosts on the installed app's profile + shell: bash + run: | + mkdir -p artifacts + node tests/tools/win-update-e2e/serve-switch.mjs \ + --installer dist/orca-windows-setup.exe \ + --template out/orcad-template 2>&1 | tee artifacts/serve-switch.log + exit "${PIPESTATUS[0]}" + + - name: Upload serve switch output + if: always() + uses: actions/upload-artifact@v7 + with: + name: win-orcad-serve-switch-output + path: artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/config/build-plugins/plain-node-entry-guard.ts b/config/build-plugins/plain-node-entry-guard.ts index b07b40c0478..cc0ab39bc1d 100644 --- a/config/build-plugins/plain-node-entry-guard.ts +++ b/config/build-plugins/plain-node-entry-guard.ts @@ -43,6 +43,7 @@ const PLAIN_NODE_ENTRY_NAMES = [ 'parcel-watcher-process-entry', 'computer-sidecar', 'wsl-transcript-fs-process-entry', + 'orcad/orcad-local-serve-selection-entry', ...CLI_MAIN_ENTRY_NAMES ] as const diff --git a/config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1 b/config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1 index e4bfae52d69..9d58935f03a 100644 --- a/config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1 +++ b/config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1 @@ -6,12 +6,17 @@ param( [Parameter(Mandatory=$true)][hashtable]$Context, [Parameter(Mandatory=$true)][ValidateSet('win32-arm64','win32-x64')][string]$Target, [Parameter(Mandatory=$true)][string]$ReceiptRoot, - [ValidateSet('pinned-cmd','pinned-powershell','legacy-opt-out')][string[]]$Cells=@('pinned-cmd','pinned-powershell','legacy-opt-out') + [ValidateSet('pinned-cmd','pinned-powershell','legacy-opt-out','orcad-cmd','orcad-powershell','orcad-convert','orcad-cli-managed','orcad-cli-convert','orcad-cli-relay-kept')][string[]]$Cells=@('pinned-cmd','pinned-powershell','legacy-opt-out','orcad-cmd','orcad-powershell') ) $ErrorActionPreference='Stop' if($env:GITHUB_ACTIONS -ne 'true' -or $env:ORCA_ISOLATED_SSH_CI -ne '1'){throw 'Disposable CI only'} -$shells=@{'pinned-cmd'='cmd';'pinned-powershell'='powershell';'legacy-opt-out'='cmd'} +$shells=@{'pinned-cmd'='cmd';'pinned-powershell'='powershell';'legacy-opt-out'='cmd';'orcad-cmd'='cmd';'orcad-powershell'='powershell';'orcad-convert'='cmd';'orcad-cli-managed'='cmd';'orcad-cli-convert'='cmd';'orcad-cli-relay-kept'='cmd'} +# App-level cells drive the e2e build (and the bundled CLI) against the host; each greps one tagged test. +$appCells=@{'orcad-convert'=@('tests/e2e/ssh-orcad-auto-convert.spec.ts','');'orcad-cli-managed'=@('tests/e2e/ssh-orcad-windows-cli-matrix.spec.ts','@orcad-cli-managed');'orcad-cli-convert'=@('tests/e2e/ssh-orcad-windows-cli-matrix.spec.ts','@orcad-cli-convert');'orcad-cli-relay-kept'=@('tests/e2e/ssh-orcad-windows-cli-matrix.spec.ts','@orcad-cli-relay-kept')} +$electronBuilt=$false if($Context.accounts.Count -lt $Cells.Count){throw 'Each cell needs its own private account'} +$seenApp=$false +foreach($id in $Cells){if($appCells.ContainsKey($id)){$seenApp=$true}elseif($seenApp){throw 'App cells must run last: they switch native modules to Electron'}} if(-not $Context.forbiddenToolLog){throw 'Run the provisioning with -HiddenTools so toolchain calls are logged'} $openSshKey='HKLM:\SOFTWARE\OpenSSH' $windowsPowerShell=Join-Path $env:WINDIR 'System32\WindowsPowerShell\v1.0\powershell.exe' @@ -53,6 +58,37 @@ function Test-PrivateWmiLaunch([string]$Account) { if($match.Success){return $match.Groups[1].Value}else{return 'no-output'} } +# App-level cells (tests/e2e/ssh-orcad-auto-convert.spec.ts, ssh-orcad-windows-cli-matrix.spec.ts). Last +# in the run: they switch native modules to Electron's ABI, which the vitest cells cannot load. +function Invoke-AppCell($Account,[string]$Descriptor,[string]$Log,[string]$Spec,[string]$Grep) { + $ready=Invoke-PrivateSsh $Account.name 'git init -q orca-convert-repo && git -C orca-convert-repo -c user.name=orca -c user.email=orca@example.invalid commit -q --allow-empty -m init && echo ORCA_REPO_READY' + if($ready -notmatch 'ORCA_REPO_READY'){throw 'Could not create the convert cell repository as the account'} + # Out of the app's default lookup, so the relay phase runs without a template. + $template=Join-Path $env:RUNNER_TEMP 'orcad-convert-template' + # Why guarded: Copy-Item into an existing folder nests the copy instead of replacing it. + if(-not (Test-Path -LiteralPath $template)){Copy-Item -LiteralPath 'out\orcad-template' -Destination $template -Recurse -Force} + Rename-Item -LiteralPath 'out\orcad-template' -NewName 'orcad-template.convert-hidden' + try { + if(-not $script:electronBuilt){ + & node config/scripts/ensure-native-runtime.mjs --runtime=electron 2>&1 | Tee-Object -FilePath $Log | Out-Host + if($global:LASTEXITCODE -ne 0){Write-Host 'Switching native modules to Electron failed';return $global:LASTEXITCODE} + & pnpm exec electron-vite build --mode e2e 2>&1 | Tee-Object -FilePath $Log -Append | Out-Host + if($global:LASTEXITCODE -ne 0){Write-Host 'The e2e app build failed';return $global:LASTEXITCODE} + $script:electronBuilt=$true + } + $env:ORCA_E2E_ORCAD_CONVERT_HOST=$Descriptor;$env:ORCA_E2E_ORCAD_CONVERT_TEMPLATE=$template;$env:SKIP_BUILD='1' + $grepArgs=if($Grep){@('--grep',$Grep)}else{@()} + & pnpm exec playwright test --config tests/playwright.config.ts $Spec @grepArgs --project=electron-headless --workers=1 2>&1 | Tee-Object -FilePath $Log -Append | Out-Host + # Functions return uncaptured output, so only the exit code may reach the caller. + return $global:LASTEXITCODE + } finally { + # Screenshots, traces and error context: Playwright clears test-results on the next cell's run. + if(Test-Path -LiteralPath 'test-results'){Copy-Item -LiteralPath 'test-results' -Destination ($Log -replace '\.log$','.test-results') -Recurse -Force} + Remove-Item Env:ORCA_E2E_ORCAD_CONVERT_HOST,Env:ORCA_E2E_ORCAD_CONVERT_TEMPLATE,Env:SKIP_BUILD -ErrorAction SilentlyContinue + Rename-Item -LiteralPath 'out\orcad-template.convert-hidden' -NewName 'orcad-template' + } +} + New-Item -ItemType Directory -Force -Path $ReceiptRoot | Out-Null Push-Location $SourceRoot try { @@ -78,11 +114,18 @@ try { @{cell=$cell;target=$Target;host='127.0.0.1';port=[int]$Context.port;username=$account.name;identityFile=$Context.identityFile;home=$account.home;forbiddenToolLog=$Context.forbiddenToolLog;receipt=(Join-Path $ReceiptRoot "$cell.json")} | ConvertTo-Json | Set-Content -LiteralPath $descriptor -Encoding utf8NoBOM $env:ORCA_RUN_SSH_WINDOWS_HOST='1';$env:ORCA_SSH_WINDOWS_HOST_CELL=$descriptor Write-Host "Windows host cell $cell ($Target, DefaultShell $shell, account $($account.name))" - & node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/ssh/ssh-relay-windows-host-lane.test.ts --reporter=verbose 2>&1 | Tee-Object -FilePath (Join-Path $ReceiptRoot "$cell.log") - # Why global: under the workflow's GetNewClosure callback, bare $LASTEXITCODE reads a stale captured copy. - $code=$global:LASTEXITCODE + if($appCells.ContainsKey($cell)){ + $code=Invoke-AppCell $account $descriptor (Join-Path $ReceiptRoot "$cell.log") $appCells[$cell][0] $appCells[$cell][1] + } else { + # orcad cells deploy managed orcad instead of the relay; same account and descriptor shape. + $lane=if($cell.StartsWith('orcad-')){'src/main/ssh/orcad-windows-host-lane.test.ts'}else{'src/main/ssh/ssh-relay-windows-host-lane.test.ts'} + & node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts $lane --reporter=verbose 2>&1 | Tee-Object -FilePath (Join-Path $ReceiptRoot "$cell.log") + # Why global: under the workflow's GetNewClosure callback, bare $LASTEXITCODE reads a stale captured copy. + $code=$global:LASTEXITCODE + } # The relay's own log is the only record of why it closed a client. - foreach($log in @(Get-ChildItem -Path (Join-Path $account.home '.orca-remote\relay-*\relay*.log') -File -ErrorAction SilentlyContinue)){Copy-Item -LiteralPath $log.FullName -Destination (Join-Path $ReceiptRoot "$cell.$($log.Directory.Name).$($log.Name)")} + # orcad.log holds only the last launch on Windows; orcad.log.1 is the one before a restart. + foreach($log in @(Get-ChildItem -Path (Join-Path $account.home '.orca-remote\relay-*\relay*.log'),(Join-Path $account.home '.orca-remote\orcad-*\orcad.log'),(Join-Path $account.home '.orca-remote\orcad-*\orcad.log.1') -File -ErrorAction SilentlyContinue)){Copy-Item -LiteralPath $log.FullName -Destination (Join-Path $ReceiptRoot "$cell.$($log.Directory.Name).$($log.Name)")} if(Test-Path -LiteralPath $Context.forbiddenToolLog){Copy-Item -LiteralPath $Context.forbiddenToolLog -Destination (Join-Path $ReceiptRoot "$cell.forbidden-tool-calls.log")} $summary.Add(@{cell=$cell;shell=$shell;account=$account.name;exitCode=$code}) if($code -ne 0){$failed.Add($cell)} diff --git a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 index 10b4d9d67e1..5b86447c109 100644 --- a/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 +++ b/config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1 @@ -2,7 +2,7 @@ # -HiddenTools: the private accounts are denied every machine PATH directory holding one of these # executables, and their own PATH carries logging shims for them, so SSH sessions have no host toolchain. # -HostCellProbe receives a context hashtable (accounts, port, keys, shim log) once provisioning passes. -param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,4)][int]$Accounts=1,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe,[string]$InboxPreparationReceipt) +param([Parameter(Mandatory=$true)][string]$Receipt,[string]$Archive,[Parameter(Mandatory=$true)][ValidateSet('arm64','x64')][string]$Arch,[ValidateSet('preview','inbox')][string]$Server='preview',[scriptblock]$ProductionRouteProbe,[ValidateRange(1,6)][int]$Accounts=1,[ValidateRange(0,6)][int]$ForwardingAccounts=0,[string[]]$HiddenTools=@(),[scriptblock]$HostCellProbe,[string]$InboxPreparationReceipt) $ErrorActionPreference = 'Stop' . (Join-Path $PSScriptRoot 'windows-ssh-capability.ps1') $target=@{arm64=@{os='Arm64';folder='OpenSSH-ARM64';machine='0xAA64';archive='698c6aec31c1dd0fb996206e8741f4531a97355686b5431ef347d531b07fcd42'};x64=@{os='X64';folder='OpenSSH-Win64';machine='0x8664';archive='23f50f3458c4c5d0b12217c6a5ddfde0137210a30fa870e98b29827f7b43aba5'}}[$Arch] @@ -246,6 +246,7 @@ PermitTunnel no PermitTTY no Subsystem sftp "$sftpServerPosix" LogLevel DEBUG1 +$(@($accountNames | Select-Object -Last $ForwardingAccounts | ForEach-Object {"Match User $_`n AllowTcpForwarding local"}) -join "`n") "@ | Set-Content -LiteralPath $config -Encoding ascii Write-Stage 'server-config-validate-start' Invoke-Bounded $sshd @('-t','-f',$config) | Out-Null diff --git a/config/scripts/adhoc-build-pinned-commit.test.mjs b/config/scripts/adhoc-build-pinned-commit.test.mjs new file mode 100644 index 00000000000..b2062f9b05c --- /dev/null +++ b/config/scripts/adhoc-build-pinned-commit.test.mjs @@ -0,0 +1,93 @@ +// An adhoc run builds one commit: every job checks out what resolve-ref pinned at dispatch, so a +// push mid-run can never mix slot lanes from one commit with a template merge from the next. +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { runProcess } from '../../src/shared/child-process/run-process' + +const workflow = parse(readFileSync('.github/workflows/adhoc-mac-build.yml', 'utf8')) +const PINNED = '${{ needs.resolve-ref.outputs.sha }}' +const resolveStep = workflow.jobs['resolve-ref'].steps.find((step) => step.id === 'resolve') +const directory = mkdtempSync(join(tmpdir(), 'adhoc-pinned-commit-')) +const repository = join(directory, 'remote.git') +const identity = { + ...process.env, + GIT_AUTHOR_NAME: 'Ref test', + GIT_AUTHOR_EMAIL: 'ref-test@example.com', + GIT_COMMITTER_NAME: 'Ref test', + GIT_COMMITTER_EMAIL: 'ref-test@example.com' +} +let branchTip, tagged + +async function git(args) { + const result = await runProcess({ program: 'git', args, env: identity }) + expect(result.code, result.stderr).toBe(0) + return result.stdout.trim() +} + +async function resolve(ref) { + const scratch = mkdtempSync(join(directory, 'attempt-')) + const script = join(scratch, 'resolve.sh') + writeFileSync(script, resolveStep.run) + const output = join(scratch, 'output') + writeFileSync(output, '') + const result = await runProcess({ + program: 'bash', + args: [script], + env: { + ...identity, + REPO_URL: pathToFileURL(repository).href, + GITHUB_OUTPUT: output, + REQUESTED_REF: ref + } + }) + return { code: result.code, output: readFileSync(output, 'utf8').trim() } +} + +beforeAll(async () => { + await git(['init', '--bare', repository]) + const tree = await git(['-C', repository, 'mktree']) + tagged = await git(['-C', repository, 'commit-tree', tree, '-m', 'tagged']) + branchTip = await git(['-C', repository, 'commit-tree', tree, '-p', tagged, '-m', 'tip']) + await git(['-C', repository, 'update-ref', 'refs/heads/feature/x', branchTip]) + await git(['-C', repository, 'tag', '-a', 'v1', tagged, '-m', 'annotated']) +}) + +afterAll(() => rmSync(directory, { recursive: true, force: true })) + +describe('adhoc build pins one commit for the whole run', () => { + it('checks out the pinned commit in every job that builds from the repo', () => { + const { jobs } = workflow + expect(jobs['relay-windows-process-tree'].with.ref).toBe(PINNED) + expect(jobs['orcad-template'].with.ref).toBe(PINNED) + const support = jobs['orcad-template-support'].steps.find((step) => + step.uses?.startsWith('actions/checkout@') + ) + expect(support.with.ref).toBe(PINNED) + const vet = jobs['build-adhoc-mac'].steps.find((step) => step.id === 'vetted') + expect(vet.env.REQUESTED_REF).toBe(PINNED) + for (const name of [ + 'relay-windows-process-tree', + 'orcad-template-support', + 'orcad-template', + 'build-adhoc-mac' + ]) { + expect([jobs[name].needs].flat(), name).toContain('resolve-ref') + } + }) + + it('resolves a branch, an annotated tag to its commit, and passes a full SHA through', async () => { + expect(await resolve('feature/x')).toEqual({ code: 0, output: `sha=${branchTip}` }) + expect(await resolve('v1')).toEqual({ code: 0, output: `sha=${tagged}` }) + expect(await resolve(tagged)).toEqual({ code: 0, output: `sha=${tagged}` }) + }) + + it('refuses PR refs and names it cannot resolve', async () => { + for (const ref of ['refs/pull/1/head', 'pull/1/head', 'missing', tagged.slice(0, 12)]) { + expect(await resolve(ref), ref).toEqual({ code: 1, output: '' }) + } + }) +}) diff --git a/config/scripts/build-orcad-prebuilds.mjs b/config/scripts/build-orcad-prebuilds.mjs index 73dbab5201e..4145edf1b2d 100644 --- a/config/scripts/build-orcad-prebuilds.mjs +++ b/config/scripts/build-orcad-prebuilds.mjs @@ -3,7 +3,8 @@ * Build one node-pty prebuilt for the CURRENT platform/arch/libc and file it in orcad's * prebuilds matrix, so a deployment target needs no C/C++ toolchain. * - * node-pty is the only ABI-sensitive native module orcad requires. It is also PATCHED in + * node-pty is the only ABI-sensitive native module every slot builds; a compat slot also + * builds the addons in COMPAT_SLOT_ADDONS (orcad-prebuild-compat-addons.mjs). node-pty is PATCHED in * this repo (config/patches/node-pty@1.1.0.patch), and that patch is the glibc-floor fix: * `.symver` pins on openpty/forkpty/pthread_sigmask plus the `--no-as-needed` ldflags that * keep libutil/libpthread in DT_NEEDED. An upstream prebuilt has none of it and reproduces @@ -40,13 +41,14 @@ import { highestGlibcNeed, isCompatSlot, mergeManifest, - prebuildCompileGypi, readManifest, sha256Of, slotGlibcFloor, slotSourceFiles, SLOT_NAPI_VERSION } from './orcad-prebuild-slot-contents.mjs' +import { compileCompatAddons } from './orcad-prebuild-compat-addons.mjs' +import { nodeGypRebuild, stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs' import { ensurePinnedNodeExecutable, preparePinnedNodeDir } from './pinned-node-downloads.mjs' export { readManifest } @@ -203,44 +205,23 @@ async function compileNodePty(sourceDir, slot) { ) const ptySourcePath = join(stagedDir, 'src', 'unix', 'pty.cc') writeFileSync(ptySourcePath, ptySourceForLibc(readFileSync(ptySourcePath, 'utf8'), libc)) - const addonApiDir = dirname( - require.resolve('node-addon-api/package.json', { paths: [sourceDir] }) - ) - cpSync(addonApiDir, join(stagedDir, 'node_modules', 'node-addon-api'), { - recursive: true, - dereference: true - }) + stageNodeAddonApi(sourceDir, stagedDir) if (process.platform === 'win32') { require('./node-pty-job-ownership.cjs').assertNodePtySourceDeniesMsysBreakaway({ nodePtyDir: stagedDir }) } - const compileGypi = join(workDir, 'prebuild-compile.gypi') - writeFileSync(compileGypi, prebuildCompileGypi({ staticCxxRuntime: isCompatSlot(slot) })) const nodeDir = await preparePinnedNodeDir({ target: slot, workDir: join(workDir, 'nodedir') }) console.log( `[orcad-prebuilds] compiling patched node-pty for ${slot} against Node ${NODE_RUNTIME_PIN.version} headers, N-API ${SLOT_NAPI_VERSION} ...` ) - const { runProcessSync } = await import('./script-child-process.mjs') - const result = runProcessSync({ - program: process.execPath, - args: [ - join(ROOT, 'node_modules', 'node-gyp', 'bin', 'node-gyp.js'), - 'rebuild', - `--nodedir=${nodeDir}`, - '--', - '-I', - compileGypi - ], - cwd: stagedDir, - stdio: 'inherit', - timeoutMs: null + const buildDir = await nodeGypRebuild({ + stagedDir, + workDir, + nodeDir, + staticCxxRuntime: isCompatSlot(slot) }) - if (result.code !== 0) { - throw new Error(`[orcad-prebuilds] node-gyp rebuild failed (status ${result.code})`) - } - const buildDir = join(stagedDir, 'build', 'Release') if (process.platform === 'win32') { require('./node-pty-job-ownership.cjs').assertRebuiltConptyDeniesMsysBreakaway({ nodePtyDir: stagedDir, @@ -248,7 +229,7 @@ async function compileNodePty(sourceDir, slot) { crossHost: false }) } - return buildDir + return { buildDir, nodeDir } } function requireSlots(slots) { @@ -310,16 +291,22 @@ async function build() { const slot = slotName() assertCompatSlotHost(slot, { platform: process.platform, arch: process.arch, libc: detectLibc() }) const slotDir = join(PREBUILDS_DIR, slot) - const buildDir = await compileNodePty(sourceDir, slot) + const { buildDir, nodeDir } = await compileNodePty(sourceDir, slot) + const compatAddons = isCompatSlot(slot) + ? await compileCompatAddons({ slot, workDir: join(WORK_DIR, slot), nodeDir }) + : [] rmSync(slotDir, { recursive: true, force: true }) const files = {} - for (const [relative, source] of slotSourceFiles({ - platform: process.platform, - arch: process.arch, - buildDir, - nodePtyDir: sourceDir - })) { + for (const [relative, source] of [ + ...slotSourceFiles({ + platform: process.platform, + arch: process.arch, + buildDir, + nodePtyDir: sourceDir + }), + ...compatAddons + ]) { if (!existsSync(source)) { throw new Error(`[orcad-prebuilds] ${slot} needs ${relative}, but ${source} is missing`) } diff --git a/config/scripts/build-orcad-template.mjs b/config/scripts/build-orcad-template.mjs index 355f2e2ba95..899116c07f8 100644 --- a/config/scripts/build-orcad-template.mjs +++ b/config/scripts/build-orcad-template.mjs @@ -29,7 +29,12 @@ import { pinnedNodeRuntimeAsset } from '../../src/shared/node-runtime-pin.ts' import { ORCAD_PREBUILDS_DIR } from './build-orcad-prebuilds.mjs' -import { findSlotProblems, readManifest } from './orcad-prebuild-slot-contents.mjs' +import { + COMPAT_SLOT_ADDONS, + findCompatAddonGaps, + findSlotProblems, + readManifest +} from './orcad-prebuild-slot-contents.mjs' import { runProcessSync } from './script-child-process.mjs' import { verifyPackagedOrcadTemplate } from './verify-packaged-orcad-template.cjs' @@ -121,8 +126,8 @@ export function requestedTemplateTargets(argv = process.argv) { } /** - * A compat target (design D6 rung B) is its base target's package with the compat node-pty - * slot and runtime marker swapped in; everything else is target-independent or libc-static. + * A compat target (design D6 rung B) is its base target's package with the compat slot's addons + * and runtime marker swapped in; everything else is target-independent or libc-static. * Omitted, not failed, when this build has no compat slot: rung B then refuses as unavailable. */ function stageCompatTarget(compat, basePackageDir) { @@ -136,12 +141,21 @@ function stageCompatTarget(compat, basePackageDir) { return null } const destination = join(outputDir, ORCAD_TEMPLATE_TARGETS_DIR, compat) - const slotFiles = new Map( - orcadNodePtySlotFiles(compat).map((file) => [ + const slotFiles = new Map([ + ...orcadNodePtySlotFiles(compat).map((file) => [ `${ORCAD_NODE_PTY_DIR}/build/Release/${file}`, join(ORCAD_PREBUILDS_DIR, compat, ...file.split('/')) + ]), + ...Object.entries(COMPAT_SLOT_ADDONS).map(([file, shipped]) => [ + shipped, + join(ORCAD_PREBUILDS_DIR, compat, ...file.split('/')) ]) - ) + ]) + // Why fatal: the base binary would pass every check here and fail only on a compat host. + const gaps = findCompatAddonGaps(orcadTemplateTargetFilenames(compat), slotFiles) + if (gaps.length > 0) { + throw new Error(`compat target ${compat} would ship base-target addons: ${gaps.join(', ')}`) + } const files = {} for (const filename of orcadTemplateTargetFilenames(compat)) { const staged = join(destination, ...filename.split('/')) diff --git a/config/scripts/build-orcad.mjs b/config/scripts/build-orcad.mjs index 66c8f535a5b..cf1c744d3b4 100644 --- a/config/scripts/build-orcad.mjs +++ b/config/scripts/build-orcad.mjs @@ -29,6 +29,7 @@ import { stageOrcadWindowsProcessTree } from './orcad-windows-process-tree.mjs' import { ORCAD_EMOJI_SHORTCODE_DATASET, ORCAD_FOREIGN_SQLITE_READER_ENTRY, + ORCAD_PORT_SCAN_COMMAND_WORKER_ENTRY, ORCAD_NODE_PTY_DIR, ORCAD_NODE_PTY_JS_ARTIFACTS, ORCAD_NODE_RUNTIME_MARKER_FILENAME, @@ -62,6 +63,10 @@ const DAEMON_OUT_FILE = join(OUT_DIR, 'daemon-entry.js') // start this worker from the module dir, since orcad has no Electron resources tree. const FOREIGN_SQLITE_READER_ENTRY = join(ROOT, ORCAD_CHILD_ENTRY_POINTS.foreignSqliteReader) const FOREIGN_SQLITE_READER_OUT_FILE = join(OUT_DIR, ORCAD_FOREIGN_SQLITE_READER_ENTRY) +// Why beside orcad.js: workspace port detection runs its probe commands on this worker thread, +// and `resolveWorkerEntryPath` looks for it next to the running bundle. +const PORT_SCAN_WORKER_ENTRY = join(ROOT, ORCAD_CHILD_ENTRY_POINTS.portScanCommandWorker) +const PORT_SCAN_WORKER_OUT_FILE = join(OUT_DIR, ORCAD_PORT_SCAN_COMMAND_WORKER_ENTRY) const OUT_FILE = join(OUT_DIR, 'orcad.js') const BUILD_TARGET = process.env.ORCAD_BUILD_TARGET if (!BUILD_TARGET) { @@ -212,6 +217,7 @@ const childResults = await Promise.all([ buildForkedChild(WATCHER_ENTRY, WATCHER_OUT_FILE), buildForkedChild(DAEMON_ENTRY, DAEMON_OUT_FILE), buildForkedChild(FOREIGN_SQLITE_READER_ENTRY, FOREIGN_SQLITE_READER_OUT_FILE), + buildForkedChild(PORT_SCAN_WORKER_ENTRY, PORT_SCAN_WORKER_OUT_FILE), ...['writer', 'backup'].map((role) => buildForkedChild( join(ROOT, ORCAD_CHILD_ENTRY_POINTS[role]), diff --git a/config/scripts/ci-e2e-job-selection.mjs b/config/scripts/ci-e2e-job-selection.mjs index cb9d07aec25..e2e207ce58b 100644 --- a/config/scripts/ci-e2e-job-selection.mjs +++ b/config/scripts/ci-e2e-job-selection.mjs @@ -38,11 +38,23 @@ export const NODE_NETWORK_E2E_SPEC = 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts' export const LOCALHOST_SSH_E2E_SPEC = 'tests/e2e/ssh-localhost.spec.ts' export const NATIVE_IME_E2E_SPEC = 'tests/e2e/terminal-ibus-hangul-native.spec.ts' +// Needs the packaged orcad slot, which only its own job builds. +export const ORCAD_SERVE_MODE_SWITCH_E2E_SPEC = 'tests/e2e/orcad-serve-mode-switch.spec.ts' +// Needs the orcad template for its host's target, which only its own job builds. +export const ORCAD_AUTO_CONVERT_E2E_SPEC = 'tests/e2e/ssh-orcad-auto-convert.spec.ts' +// Windows-only; its own job runs it on a Windows runner. +export const WINDOWS_MISSING_APPDATA_E2E_SPEC = 'tests/e2e/windows-missing-appdata-startup.spec.ts' +// Runs in the auto-convert job, which builds the template it needs. +export const ORCAD_IDLE_EXIT_E2E_SPEC = 'tests/e2e/ssh-orcad-idle-exit.spec.ts' export const DEDICATED_E2E_SPECS = [ ...DOCKER_SSH_E2E_SPECS, NODE_NETWORK_E2E_SPEC, LOCALHOST_SSH_E2E_SPEC, - NATIVE_IME_E2E_SPEC + NATIVE_IME_E2E_SPEC, + ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, + ORCAD_AUTO_CONVERT_E2E_SPEC, + WINDOWS_MISSING_APPDATA_E2E_SPEC, + ORCAD_IDLE_EXIT_E2E_SPEC ] const dedicatedSpecs = new Set(DEDICATED_E2E_SPECS) const dockerSpecs = new Set(DOCKER_SSH_E2E_SPECS) @@ -77,7 +89,14 @@ export function classifyE2eJobs(input, sshSourceChanged = 'false') { e2e_needs_build: runChanged || sshSourceChanged !== 'false' || - specs.some((spec) => dockerSpecs.has(spec) || spec === LOCALHOST_SSH_E2E_SPEC) + specs.some( + (spec) => + dockerSpecs.has(spec) || + spec === LOCALHOST_SSH_E2E_SPEC || + spec === ORCAD_SERVE_MODE_SWITCH_E2E_SPEC || + spec === ORCAD_AUTO_CONVERT_E2E_SPEC || + spec === ORCAD_IDLE_EXIT_E2E_SPEC + ) } } diff --git a/config/scripts/merge-orcad-prebuilds.test.mjs b/config/scripts/merge-orcad-prebuilds.test.mjs index 430cf1bca1f..4a937810aa8 100644 --- a/config/scripts/merge-orcad-prebuilds.test.mjs +++ b/config/scripts/merge-orcad-prebuilds.test.mjs @@ -1,9 +1,15 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { mergeOrcadPrebuildTrees } from './merge-orcad-prebuilds.mjs' -import { findSlotProblems, mergeManifest, sha256Of } from './orcad-prebuild-slot-contents.mjs' +import { + COMPAT_SLOT_ADDONS, + findSlotProblems, + isCompatSlot, + mergeManifest, + sha256Of +} from './orcad-prebuild-slot-contents.mjs' const dirs = [] function temp() { @@ -20,15 +26,20 @@ afterEach(() => { /** One CI lane's `out/orcad-prebuilds`: a single slot plus its manifest. */ function laneTree(slot, { version = '1.1.0', nodeHeaders = '24.21.0', bytes = slot } = {}) { const dir = temp() - mkdirSync(join(dir, slot), { recursive: true }) - const binary = join(dir, slot, 'pty.node') - writeFileSync(binary, bytes) + const files = {} + // A compat slot also carries its own addons. + for (const file of ['pty.node', ...(isCompatSlot(slot) ? Object.keys(COMPAT_SLOT_ADDONS) : [])]) { + const binary = join(dir, slot, ...file.split('/')) + mkdirSync(dirname(binary), { recursive: true }) + writeFileSync(binary, bytes) + files[file] = sha256Of(binary) + } const manifest = mergeManifest(null, { slot, version, napi: 8, nodeHeaders, - entry: { napi: 8, files: { 'pty.node': sha256Of(binary) } } + entry: { napi: 8, files } }) writeFileSync(join(dir, 'manifest.json'), JSON.stringify(manifest)) return dir diff --git a/config/scripts/node-server-test-paths.mjs b/config/scripts/node-server-test-paths.mjs index a93c30869ca..8d9e8eb6710 100644 --- a/config/scripts/node-server-test-paths.mjs +++ b/config/scripts/node-server-test-paths.mjs @@ -14,6 +14,12 @@ export function nodeServerTestPaths({ artifact = false, crossRuntime = false } = 'src/main/sqlite', 'src/main/orcad/orcad-entry.test.ts', 'src/main/orcad/orcad-push-startup.test.ts', + // The orcad server's identity and stop path, which Windows SSH hosts rely on (W2). + 'src/main/orcad/orcad-instance-lock.test.ts', + 'src/main/orcad/orcad-process-start-time.test.ts', + 'src/main/orcad/orcad-stop-request-listener.test.ts', + 'src/main/orcad/orcad-managed-stop.test.ts', + 'src/main/orcad/orcad-managed-stop-cancellation.test.ts', // The directory, not a prefix: its siblings are POSIX-host unit tests pr.yml already runs. 'src/main/daemon/pty-subprocess/', 'src/main/daemon/pty-subprocess-spawn-file-foreground.test.ts', @@ -25,6 +31,9 @@ export function nodeServerTestPaths({ artifact = false, crossRuntime = false } = 'src/main/orcad/orcad-packaged-node-pty.integration.test.ts', 'src/main/providers/agent-foreground-process-git-bash.win32.test.ts', 'src/main/orcad/orcad-node-launcher.integration.test.ts', + 'src/main/orcad/orcad-stop-request-shutdown.integration.test.ts', + 'src/main/orcad/orcad-windows-conpty-breakaway.integration.test.ts', + 'src/main/orcad/orcad-serve-parity.integration.test.ts', 'config/scripts/zip-extractor-command.test.mjs' ] : []), diff --git a/config/scripts/orcad-daemon-protocol-crossing.test.mjs b/config/scripts/orcad-daemon-protocol-crossing.test.mjs new file mode 100644 index 00000000000..131c1d54c57 --- /dev/null +++ b/config/scripts/orcad-daemon-protocol-crossing.test.mjs @@ -0,0 +1,100 @@ +// D7: orcad's update and rollback planning must agree with the CI protocol-crossing facts. +import { readFileSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + DAEMON_PROTOCOL_SOURCE_PATH, + canAttach, + parseDaemonProtocolFacts +} from './daemon-protocol-facts.mjs' +import { CURRENT_ORCAD_DAEMON_PROTOCOL } from '../../src/main/ssh/orcad-daemon-protocol-crossing' +import { assessOrcadRollback, planOrcadUpdate } from '../../src/main/ssh/orcad-update-plan' + +const projectDir = resolve(import.meta.dirname, '../..') +const current = parseDaemonProtocolFacts( + readFileSync(join(projectDir, DAEMON_PROTOCOL_SOURCE_PATH), 'utf8') +) +// The release before the newest protocol bump: it speaks one version lower and cannot list ours. +const older = { + protocolVersion: current.protocolVersion - 1, + previousProtocolVersions: current.previousProtocolVersions.filter( + (version) => version < current.protocolVersion - 1 + ) +} +const record = { + schemaVersion: 1, + active: '0.3.0+new', + previous: '0.2.0+old', + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: { + dirName: 'pre-0.3.0+new-1', + takenBeforeVersion: '0.3.0+new', + readableByVersion: '0.2.0+old', + takenAt: '2026-01-01T00:00:00.000Z' + } +} +const live = (daemonProtocolVersion) => ({ + liveSessions: 2, + startedSinceActivation: 0, + daemonProtocolVersion +}) + +function rollback(target, daemonProtocolVersion) { + return assessOrcadRollback({ + record, + snapshotPresent: true, + census: live(daemonProtocolVersion), + targetDaemonProtocol: target, + stateWritesSinceActivation: false + }) +} + +describe('orcad daemon protocol crossing', () => { + it('deploys exactly the protocol the working tree declares', () => { + expect({ + protocolVersion: CURRENT_ORCAD_DAEMON_PROTOCOL.protocolVersion, + previousProtocolVersions: [...CURRENT_ORCAD_DAEMON_PROTOCOL.previousProtocolVersions] + }).toEqual(current) + }) + + it('keeps terminals on rollback only when the old build lists the new protocol', () => { + expect(canAttach(older, current)).toBe(false) + expect(rollback(older, current.protocolVersion)).toMatchObject({ + safety: 'unsafe', + code: 'orcad_rollback_strands_live_terminals' + }) + // A daemon preserved from before the activation still speaks the old build's protocol. + expect(canAttach(older, older)).toBe(true) + expect(rollback(older, older.protocolVersion)).toMatchObject({ safety: 'clean' }) + const listing = { ...older, previousProtocolVersions: [...older.previousProtocolVersions] } + listing.previousProtocolVersions.push(current.protocolVersion + 1) + expect(canAttach(listing, { ...current, protocolVersion: current.protocolVersion + 1 })).toBe( + true + ) + expect(rollback(listing, current.protocolVersion + 1)).toMatchObject({ safety: 'clean' }) + }) + + it('updates over live terminals only when the candidate can attach their daemon', () => { + const plan = (daemonProtocolVersion) => + planOrcadUpdate({ + record, + candidateVersion: '0.4.0+next', + census: live(daemonProtocolVersion), + candidateDaemonProtocol: current, + force: true + }) + expect(canAttach(current, older)).toBe(true) + expect(plan(older.protocolVersion)).toMatchObject({ + action: 'proceed', + preservesLiveDaemon: true + }) + const dropped = current.protocolVersion + 1 + expect(canAttach(current, { protocolVersion: dropped, previousProtocolVersions: [] })).toBe( + false + ) + expect(plan(dropped)).toMatchObject({ + action: 'defer', + code: 'orcad_update_strands_live_terminals' + }) + }) +}) diff --git a/config/scripts/orcad-e2e-routing.test.mjs b/config/scripts/orcad-e2e-routing.test.mjs new file mode 100644 index 00000000000..a133e151efa --- /dev/null +++ b/config/scripts/orcad-e2e-routing.test.mjs @@ -0,0 +1,102 @@ +import { existsSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { expect, it } from 'vitest' +import { parse } from 'yaml' +import { + classifyE2eJobs, + ORCAD_AUTO_CONVERT_E2E_SPEC, + ORCAD_IDLE_EXIT_E2E_SPEC, + ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, + WINDOWS_MISSING_APPDATA_E2E_SPEC +} from './ci-e2e-job-selection.mjs' +import { selectPrE2eSpecs, shouldRunReusablePrE2e } from './pr-e2e-source-routing.mjs' + +const root = resolve(import.meta.dirname, '../..') +const jobs = parse(readFileSync(resolve(root, '.github/workflows/e2e.yml'), 'utf8')).jobs + +function expectRouted(files, spec) { + for (const file of files) { + expect(existsSync(resolve(root, file)), file).toBe(true) + expect(selectPrE2eSpecs([file]), file).toContain(spec) + expect(shouldRunReusablePrE2e([file]), file).toBe(true) + } +} + +it('routes the mode-switch spec from its serve sources and harness', () => { + expectRouted( + [ + 'src/main/orcad/orcad-lifecycle.ts', + 'src/main/daemon/daemon-spawner.ts', + 'tests/e2e/helpers/orca-serve-cli-host.ts', + 'tests/e2e/helpers/headless-paired-runtime-host.ts' + ], + ORCAD_SERVE_MODE_SWITCH_E2E_SPEC + ) +}) + +it('routes the missing-AppData spec from its startup sources and harness', () => { + expectRouted( + ['src/main/startup/windows-app-data-path.ts', 'tests/e2e/helpers/orca-serve-cli-host.ts'], + WINDOWS_MISSING_APPDATA_E2E_SPEC + ) +}) + +it('routes the auto-convert spec from its conversion sources and harness', () => { + expectRouted( + [ + 'src/main/ssh/orcad-runtime-conversion.ts', + 'tests/e2e/helpers/orcad-convert-flow.ts', + 'tests/e2e/helpers/orcad-convert-host.ts', + 'tests/e2e/helpers/orcad-template-variant.ts', + 'tests/e2e/helpers/orcad-upgrade-profile.ts' + ], + ORCAD_AUTO_CONVERT_E2E_SPEC + ) + expect(selectPrE2eSpecs(['src/main/ssh/orcad-runtime-conversion.test.ts'])).not.toContain( + ORCAD_AUTO_CONVERT_E2E_SPEC + ) +}) + +it('routes the idle-exit spec from its idle sources and the shared convert harness', () => { + expectRouted( + [ + 'src/shared/orcad-idle-exit.ts', + 'tests/e2e/helpers/orcad-convert-flow.ts', + 'tests/e2e/helpers/orcad-convert-host.ts' + ], + ORCAD_IDLE_EXIT_E2E_SPEC + ) +}) + +it('builds the e2e app when only a build-dependent orcad spec is requested', () => { + for (const spec of [ + ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, + ORCAD_AUTO_CONVERT_E2E_SPEC, + ORCAD_IDLE_EXIT_E2E_SPEC + ]) { + expect(classifyE2eJobs(JSON.stringify([spec])), spec).toEqual({ + e2e_run_changed: false, + e2e_needs_build: true + }) + } + expect(jobs['orcad-auto-convert-docker'].needs).toEqual(['build', 'prepare-native-cache']) +}) + +it('runs the auto-convert lane only when routed, not on every SSH source change', () => { + const condition = jobs['orcad-auto-convert-docker'].if + for (const spec of [ORCAD_AUTO_CONVERT_E2E_SPEC, ORCAD_IDLE_EXIT_E2E_SPEC]) { + expect(condition).toContain(`contains(inputs.test_files, '${spec}')`) + } + expect(condition).not.toContain('ssh_source_changed') +}) + +it('runs both Windows serve specs on one runner, each only when requested', () => { + expect(jobs['windows-missing-appdata-startup']).toBeUndefined() + const job = jobs['orcad-serve-mode-switch-windows'] + for (const spec of [ORCAD_SERVE_MODE_SWITCH_E2E_SPEC, WINDOWS_MISSING_APPDATA_E2E_SPEC]) { + expect(job.if, spec).toContain(`contains(inputs.test_files, '${spec}')`) + const step = job.steps.find((candidate) => candidate.run?.includes(spec)) + expect(step.if, spec).toContain(`contains(inputs.test_files, '${spec}')`) + expect(step.env.ORCA_STARTUP_DIAGNOSTICS, spec).toBeUndefined() + } +}) diff --git a/config/scripts/orcad-entry-build.mjs b/config/scripts/orcad-entry-build.mjs index 8ee6b243b3d..90767764d21 100644 --- a/config/scripts/orcad-entry-build.mjs +++ b/config/scripts/orcad-entry-build.mjs @@ -9,7 +9,8 @@ export const ORCAD_CHILD_ENTRY_POINTS = { daemon: 'src/main/daemon/daemon-entry.ts', writer: 'src/main/persistence/profile-state/profile-state-writer-worker-entry.ts', backup: 'src/main/persistence/profile-state/profile-state-backup-worker-entry.ts', - foreignSqliteReader: 'src/main/foreign-sqlite-readers/foreign-sqlite-reader-entry.ts' + foreignSqliteReader: 'src/main/foreign-sqlite-readers/foreign-sqlite-reader-entry.ts', + portScanCommandWorker: 'src/main/ports/port-scan-command-worker-entry.ts' } export const ORCAD_EXTERNAL_MODULES = ['electron', 'node-pty', '@parcel/watcher', 'fsevents'] diff --git a/config/scripts/orcad-prebuild-compat-addons.mjs b/config/scripts/orcad-prebuild-compat-addons.mjs new file mode 100644 index 00000000000..b7e35e4e991 --- /dev/null +++ b/config/scripts/orcad-prebuild-compat-addons.mjs @@ -0,0 +1,52 @@ +/** + * The addons a compat slot builds beside node-pty (design D6 rung B). The default slots take + * @parcel/watcher's upstream prebuild, which needs a newer libstdc++ than a glibc 2.17 host has, + * so the compat slot compiles it from the package's own sources with the C++ runtime static. + */ +import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import { NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts' +import { nodeGypRebuild, stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs' +import { COMPAT_SLOT_ADDONS, SLOT_NAPI_VERSION } from './orcad-prebuild-slot-contents.mjs' + +const require = createRequire(import.meta.url) + +const BUILDERS = { + 'parcel-watcher/watcher.node': compileParcelWatcher +} + +/** `[slot-relative path, built file]` for every compat addon, compiled under `workDir`. */ +export async function compileCompatAddons({ slot, workDir, nodeDir }) { + const built = [] + for (const relative of Object.keys(COMPAT_SLOT_ADDONS)) { + const builder = BUILDERS[relative] + if (!builder) { + throw new Error(`[orcad-prebuilds] no builder for compat addon ${relative}`) + } + built.push([relative, await builder({ slot, workDir, nodeDir })]) + } + return built +} + +async function compileParcelWatcher({ slot, workDir, nodeDir }) { + const sourceDir = dirname(require.resolve('@parcel/watcher/package.json')) + const addonWorkDir = join(workDir, 'parcel-watcher') + const stagedDir = join(addonWorkDir, 'watcher') + rmSync(addonWorkDir, { recursive: true, force: true }) + mkdirSync(stagedDir, { recursive: true }) + for (const entry of ['package.json', 'binding.gyp', 'src']) { + cpSync(join(sourceDir, entry), join(stagedDir, entry), { recursive: true }) + } + stageNodeAddonApi(sourceDir, stagedDir) + console.log( + `[orcad-prebuilds] compiling @parcel/watcher for ${slot} against Node ${NODE_RUNTIME_PIN.version} headers, N-API ${SLOT_NAPI_VERSION} ...` + ) + const buildDir = await nodeGypRebuild({ + stagedDir, + workDir: addonWorkDir, + nodeDir, + staticCxxRuntime: true + }) + return join(buildDir, 'watcher.node') +} diff --git a/config/scripts/orcad-prebuild-node-gyp.mjs b/config/scripts/orcad-prebuild-node-gyp.mjs new file mode 100644 index 00000000000..4391b93bbee --- /dev/null +++ b/config/scripts/orcad-prebuild-node-gyp.mjs @@ -0,0 +1,44 @@ +// node-gyp rebuild of one staged addon against the pinned Node headers, shared by every slot addon. +import { cpSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { prebuildCompileGypi } from './orcad-prebuild-slot-contents.mjs' + +const require = createRequire(import.meta.url) +const ROOT = join(import.meta.dirname, '..', '..') + +/** Copies node-addon-api beside a staged addon, since scratch copies leave the pnpm tree behind. */ +export function stageNodeAddonApi(sourceDir, stagedDir) { + const addonApiDir = dirname( + require.resolve('node-addon-api/package.json', { paths: [sourceDir] }) + ) + cpSync(addonApiDir, join(stagedDir, 'node_modules', 'node-addon-api'), { + recursive: true, + dereference: true + }) +} + +export async function nodeGypRebuild({ stagedDir, workDir, nodeDir, staticCxxRuntime }) { + const compileGypi = join(workDir, 'prebuild-compile.gypi') + writeFileSync(compileGypi, prebuildCompileGypi({ staticCxxRuntime })) + const { runProcessSync } = await import('./script-child-process.mjs') + const result = runProcessSync({ + program: process.execPath, + args: [ + join(ROOT, 'node_modules', 'node-gyp', 'bin', 'node-gyp.js'), + 'rebuild', + `--nodedir=${nodeDir}`, + '--', + '-I', + compileGypi + ], + cwd: stagedDir, + stdio: 'inherit', + timeoutMs: null + }) + if (result.code !== 0) { + throw new Error(`[orcad-prebuilds] node-gyp rebuild failed (status ${result.code})`) + } + return join(stagedDir, 'build', 'Release') +} diff --git a/config/scripts/orcad-prebuild-slot-contents.mjs b/config/scripts/orcad-prebuild-slot-contents.mjs index 3f817da251b..4084d9fcc20 100644 --- a/config/scripts/orcad-prebuild-slot-contents.mjs +++ b/config/scripts/orcad-prebuild-slot-contents.mjs @@ -1,5 +1,6 @@ /** - * What goes into one orcad node-pty prebuild slot, and the manifest that records it. + * What goes into one orcad node-pty prebuild slot (plus a compat slot's own addons), and the + * manifest that records it. * * The manifest is the loader's contract (src/main/orcad/node-pty-prebuilt-slot.ts): per-slot * N-API level, libc, the highest glibc symbol version the binaries need, and a sha256 per @@ -33,6 +34,23 @@ export const COMPAT_SLOTS = Object.freeze({ 'linux-x64-glibc217': Object.freeze({ platform: 'linux', arch: 'x64', libc: 'glibc' }) }) +/** + * Native addons a compat slot builds beside node-pty, by slot-relative path, mapped to where an + * orcad slot ships them. A compat target ships no native file without a compat build. + */ +export const COMPAT_SLOT_ADDONS = Object.freeze({ + 'parcel-watcher/watcher.node': 'node_modules/@parcel/watcher/watcher.node' +}) + +/** + * The native files of a compat target with no compat build behind them: each would be the base + * target's binary, built for a newer glibc and libstdc++ than the compat host has. + * `compatSources` maps orcad slot paths to the compat slot files that fill them. + */ +export function findCompatAddonGaps(targetFilenames, compatSources) { + return targetFilenames.filter((file) => file.endsWith('.node') && !compatSources.has(file)) +} + export function isCompatSlot(slot) { return Object.hasOwn(COMPAT_SLOTS, slot) } @@ -217,6 +235,13 @@ export function findSlotProblems(manifest, prebuildsDir, requiredSlots) { problems.push(`${slot}: not built`) continue } + if (isCompatSlot(slot)) { + for (const addon of Object.keys(COMPAT_SLOT_ADDONS)) { + if (!Object.hasOwn(entry.files ?? {}, addon)) { + problems.push(`${slot}/${addon}: not built`) + } + } + } for (const [file, expected] of Object.entries(entry.files ?? {})) { const path = join(prebuildsDir, slot, ...file.split('/')) if (!existsSync(path)) { diff --git a/config/scripts/orcad-prebuild-slot-contents.test.mjs b/config/scripts/orcad-prebuild-slot-contents.test.mjs index a60ddbe817d..720a0436c3e 100644 --- a/config/scripts/orcad-prebuild-slot-contents.test.mjs +++ b/config/scripts/orcad-prebuild-slot-contents.test.mjs @@ -5,7 +5,9 @@ import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { assertCompatSlotHost, + COMPAT_SLOT_ADDONS, COMPAT_SLOTS, + findCompatAddonGaps, findPostBaselineNodeApiNames, findSharedCxxRuntimeNeeds, findSlotProblems, @@ -19,7 +21,10 @@ import { SLOT_NAPI_VERSION, windowsConptyRuntimeDir } from './orcad-prebuild-slot-contents.mjs' -import { ORCAD_ADDON_NAPI_VERSION } from '../../src/shared/orcad-artifacts.ts' +import { + ORCAD_ADDON_NAPI_VERSION, + orcadTemplateTargetFilenames +} from '../../src/shared/orcad-artifacts.ts' const floors = createRequire(import.meta.url)('./verify-linux-glibc-floor.cjs') const dirs = [] @@ -232,4 +237,37 @@ describe('findSlotProblems', () => { 'manifest.json is missing or not schema 2' ]) }) + + it('refuses a compat slot missing one of its own addons', () => { + const dir = temp() + const slot = 'linux-x64-glibc217' + mkdirSync(join(dir, slot)) + writeFileSync(join(dir, slot, 'pty.node'), 'binary') + const manifest = mergeManifest( + null, + next(slot, { entry: entry({ 'pty.node': sha256Of(join(dir, slot, 'pty.node')) }) }) + ) + expect(findSlotProblems(manifest, dir, [slot])).toEqual([ + `${slot}/parcel-watcher/watcher.node: not built` + ]) + }) +}) + +describe('compat addon coverage', () => { + const nodePtySources = (target) => + orcadTemplateTargetFilenames(target).filter((file) => file.includes('node-pty/build/Release/')) + + it('gives every native addon a compat target ships a compat build', () => { + for (const compat of Object.keys(COMPAT_SLOTS)) { + const sources = new Set([...nodePtySources(compat), ...Object.values(COMPAT_SLOT_ADDONS)]) + expect(findCompatAddonGaps(orcadTemplateTargetFilenames(compat), sources)).toEqual([]) + } + }) + + it('names a native file that would ship as the base target build', () => { + const target = 'linux-x64-glibc217' + expect( + findCompatAddonGaps(orcadTemplateTargetFilenames(target), new Set(nodePtySources(target))) + ).toEqual(['node_modules/@parcel/watcher/watcher.node']) + }) }) diff --git a/config/scripts/orcad-prebuild-smoke-child.cjs b/config/scripts/orcad-prebuild-smoke-child.cjs index 67f784eaf69..2fbb2e45e45 100644 --- a/config/scripts/orcad-prebuild-smoke-child.cjs +++ b/config/scripts/orcad-prebuild-smoke-child.cjs @@ -2,7 +2,7 @@ const { join } = require('node:path') const { tmpdir } = require('node:os') -const [nodePtyDir, expectedVersion] = process.argv.slice(2) +const [nodePtyDir, expectedVersion, ...addons] = process.argv.slice(2) if (!nodePtyDir || !expectedVersion) { throw new Error('usage: orcad-prebuild-smoke-child.cjs ') } @@ -11,6 +11,10 @@ if (process.version !== `v${expectedVersion}`) { } const pty = require(nodePtyDir) +// A compat slot's own addons: loading proves their glibc and C++ runtime needs resolve here. +for (const addon of addons) { + require(addon) +} if (process.platform === 'win32') { // Loaded only by the non-DLL kill path; prove the shipped module still loads under this Node. const { loadNativeModule } = require(join(nodePtyDir, 'lib', 'utils')) diff --git a/config/scripts/orcad-prebuild-smoke.mjs b/config/scripts/orcad-prebuild-smoke.mjs index 363e3ea7d62..52987a15eb6 100644 --- a/config/scripts/orcad-prebuild-smoke.mjs +++ b/config/scripts/orcad-prebuild-smoke.mjs @@ -3,7 +3,12 @@ import { chmodSync, cpSync, existsSync, mkdirSync, rmSync } from 'node:fs' import { createRequire } from 'node:module' import { dirname, join } from 'node:path' import { NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts' -import { findSlotProblems, readManifest } from './orcad-prebuild-slot-contents.mjs' +import { + COMPAT_SLOT_ADDONS, + findSlotProblems, + isCompatSlot, + readManifest +} from './orcad-prebuild-slot-contents.mjs' import { ensurePinnedNodeExecutable } from './pinned-node-downloads.mjs' import { runProcessSync } from './script-child-process.mjs' @@ -28,6 +33,15 @@ export function stageSmokeNodePty({ slotDir, stageDir }) { return nodePtyDir } +/** stageSmokeNodePty copies the whole slot into build/Release, compat addons included. */ +function compatAddonPaths(slot, nodePtyDir) { + return isCompatSlot(slot) + ? Object.keys(COMPAT_SLOT_ADDONS).map((file) => + join(nodePtyDir, 'build', 'Release', ...file.split('/')) + ) + : [] +} + export async function runOrcadPrebuildSmoke({ slot, prebuildsDir }) { const problems = findSlotProblems(readManifest(prebuildsDir), prebuildsDir, [slot]) if (problems.length > 0) { @@ -43,7 +57,8 @@ export async function runOrcadPrebuildSmoke({ slot, prebuildsDir }) { args: [ join(import.meta.dirname, 'orcad-prebuild-smoke-child.cjs'), nodePtyDir, - NODE_RUNTIME_PIN.version + NODE_RUNTIME_PIN.version, + ...compatAddonPaths(slot, nodePtyDir) ], timeoutMs: 60_000 }) diff --git a/config/scripts/orcad-windows-prebuild-workflow.test.mjs b/config/scripts/orcad-windows-prebuild-workflow.test.mjs index 3e6c1199e3c..d8402ca0c40 100644 --- a/config/scripts/orcad-windows-prebuild-workflow.test.mjs +++ b/config/scripts/orcad-windows-prebuild-workflow.test.mjs @@ -205,7 +205,9 @@ describe('SSH Windows consumers of qualified server slots', () => { expect(template).toBeLessThan(hosts) expect(sshSteps[template].env.ORCA_REQUIRE_RELAY_NATIVE_ADDONS).toBe('${{ matrix.arch }}') expect(sshSteps[template].run).toContain('--require-slots "win32-${{ matrix.arch }}"') - expect(sshSteps[hosts].run).toContain("@('pinned-cmd','pinned-powershell','legacy-opt-out')") + expect(sshSteps[hosts].run).toContain( + "@('pinned-cmd','pinned-powershell','legacy-opt-out','orcad-cmd','orcad-powershell')" + ) for (const workflowPaths of [ workflow.on.pull_request.paths, sshWorkflow.on.pull_request.paths diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index f8737ad6ea6..f080b28cb8d 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -13,6 +13,48 @@ const NATIVE_IME_HARNESS = /^(?:config\/scripts\/focus-nested-wayland-terminal\.sh$|config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/ export const PR_E2E_SOURCE_ROUTES = [ + { + id: 'serve.orcad-mode-switch', + specs: ['tests/e2e/orcad-serve-mode-switch.spec.ts'], + matches: (file) => + /^tests\/e2e\/helpers\/(?:orca-serve-cli-host|headless-paired-runtime-host)\.ts$/.test( + file + ) || + (isProductSource(file) && + /^src\/(?:cli\/runtime\/(?:launch|serve-)|main\/orcad\/(?:main|orcad-entry|orcad-instance-lock|orcad-command-arguments|orcad-lifecycle)\.ts$|main\/startup\/desktop-profile-instance-lock\.ts$|main\/daemon\/daemon-(?:spawner|endpoint-adoption|init)|main\/server\/serve-)/.test( + file + )) + }, + { + id: 'startup.windows-missing-appdata', + specs: ['tests/e2e/windows-missing-appdata-startup.spec.ts'], + matches: (file) => + file === 'tests/e2e/helpers/orca-serve-cli-host.ts' || + (isProductSource(file) && + /^src\/main\/startup\/(?:windows-app-data-path|main-process-preflight)\.ts$/.test(file)) + }, + { + id: 'ssh.orcad-auto-convert', + specs: ['tests/e2e/ssh-orcad-auto-convert.spec.ts'], + matches: (file) => + /^tests\/e2e\/helpers\/(?:orcad-convert-(?:flow|host)|orcad-template-variant|orcad-upgrade-profile)\.ts$/.test( + file + ) || + (isProductSource(file) && + /^src\/main\/(?:ipc\/ssh-host-server-|ssh\/(?:ssh-host-server-|orcad-runtime-conversion|orcad-migration-|orcad-retained-source|orcad-runtime-deployment))/.test( + file + )) + }, + { + id: 'ssh.orcad-idle-exit', + specs: ['tests/e2e/ssh-orcad-idle-exit.spec.ts'], + matches: (file) => + /^tests\/e2e\/helpers\/orcad-convert-(?:flow|host)\.ts$/.test(file) || + (isProductSource(file) && + /^src\/(?:main\/(?:orcad\/orcad-(?:idle-|managed-idle-)|ssh\/orcad-(?:managed-wake|managed-tunnel|recovery-slot|remote-launch))|shared\/orcad-idle-exit)/.test( + file + )) + }, { id: 'ssh.localhost-agent-hooks', specs: ['tests/e2e/ssh-localhost.spec.ts'], diff --git a/config/scripts/release-cut-token-permissions.test.mjs b/config/scripts/release-cut-token-permissions.test.mjs index 7b70e2c47d0..d4a6d36bc4e 100644 --- a/config/scripts/release-cut-token-permissions.test.mjs +++ b/config/scripts/release-cut-token-permissions.test.mjs @@ -11,6 +11,9 @@ const EXPECTED_MATRIX = { '.github/workflows/e2e.yml#build': { contents: 'read' }, '.github/workflows/e2e.yml#changed-e2e': { contents: 'read' }, '.github/workflows/e2e.yml#e2e': { contents: 'read' }, + '.github/workflows/e2e.yml#orcad-auto-convert-docker': { contents: 'read' }, + '.github/workflows/e2e.yml#orcad-serve-mode-switch': { contents: 'read' }, + '.github/workflows/e2e.yml#orcad-serve-mode-switch-windows': { contents: 'read' }, '.github/workflows/e2e.yml#prepare-native-cache': { contents: 'read' }, '.github/workflows/e2e.yml#ssh-browser-network-route': { contents: 'read' }, '.github/workflows/e2e.yml#ssh-localhost': { contents: 'read' }, diff --git a/config/scripts/ssh-windows-hosts-workflow.test.mjs b/config/scripts/ssh-windows-hosts-workflow.test.mjs index 3daef7cd4e2..80ae3c055af 100644 --- a/config/scripts/ssh-windows-hosts-workflow.test.mjs +++ b/config/scripts/ssh-windows-hosts-workflow.test.mjs @@ -4,7 +4,10 @@ import { describe, expect, it } from 'vitest' import { parse } from 'yaml' import { WINDOWS_FORBIDDEN_TOOLS, - WINDOWS_HOST_CELL_IDS + WINDOWS_HOST_CELL_IDS, + WINDOWS_CLI_MATRIX_CELL_IDS, + WINDOWS_CONVERT_CELL_ID, + WINDOWS_ORCAD_CELL_IDS } from '../../src/main/ssh/ssh-windows-host-cells.ts' const projectDir = resolve(import.meta.dirname, '../..') @@ -41,6 +44,10 @@ describe('SSH Windows-host workflow', () => { expect(paths.indexOf('src/main/ssh/ssh-relay-windows-host-lane.test.ts')).toBeGreaterThan( paths.indexOf('!src/**/*.test.ts') ) + // A later glob would re-include unit tests the Windows lane never runs. + for (const glob of paths.slice(paths.indexOf('!src/**/*.test.ts') + 1)) { + expect(glob.startsWith('src/') ? glob.endsWith('.test.ts') : true, glob).toBe(true) + } expect(job.if).toContain('github.event.pull_request.draft != true') }) @@ -54,10 +61,18 @@ describe('SSH Windows-host workflow', () => { 'x64/windows-2022/inbox', 'x64/windows-2022/preview' ]) - expect(job.env).toMatchObject({ ORCA_BACKGROUND_LAUNCH: '1', ORCA_ISOLATED_SSH_CI: '1' }) + expect(job.env).toMatchObject({ + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_ISOLATED_SSH_CI: '1' + }) expect(job.strategy['fail-fast']).toBe(false) - expect(job['timeout-minutes']).toBe(75) - expect(runStep['timeout-minutes']).toBe(50) + // The CLI matrix cells, dispatched by name only, get a longer budget. + expect(job['timeout-minutes']).toBe( + "${{ contains(github.event.inputs.cells || '', 'orcad-cli') && 160 || 75 }}" + ) + expect(runStep['timeout-minutes']).toBe( + "${{ contains(github.event.inputs.cells || '', 'orcad-cli') && 130 || 50 }}" + ) }) it('overlaps only guarded ARM inbox capability preparation with the existing builds', () => { @@ -102,7 +117,10 @@ describe('SSH Windows-host workflow', () => { "if('${{ matrix.server }}' -eq 'inbox' -and '${{ matrix.arch }}' -eq 'arm64'){$preparation=" ) expect(runStep.background).toBeUndefined() - expect(job.steps.at(-1)).toMatchObject({ if: 'always()', uses: 'actions/upload-artifact@v7' }) + expect(job.steps.at(-1)).toMatchObject({ + if: 'always()', + uses: 'actions/upload-artifact@v7' + }) }) it('shares one capability installer without bypassing native verification or private cleanup', () => { @@ -171,14 +189,42 @@ describe('SSH Windows-host workflow', () => { it('defaults to every cell the TypeScript lane knows', () => { const defaults = /\{\$cells=@\(([^)]*)\)\}/.exec(runStep.run)?.[1] expect(defaults?.split(',').map((id) => id.trim().replaceAll("'", ''))).toEqual([ - ...WINDOWS_HOST_CELL_IDS + ...WINDOWS_HOST_CELL_IDS, + ...WINDOWS_ORCAD_CELL_IDS ]) const invoker = readFileSync( join(projectDir, 'config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1'), 'utf8' ) - for (const id of WINDOWS_HOST_CELL_IDS) { + for (const id of [...WINDOWS_HOST_CELL_IDS, ...WINDOWS_ORCAD_CELL_IDS]) { expect(invoker).toContain(`'${id}'`) } + expect(invoker).toContain('src/main/ssh/orcad-windows-host-lane.test.ts') + }) + + it('provisions one private account for every cell, convert cell included', () => { + expect(runStep.run).toContain(`$cells+='${WINDOWS_CONVERT_CELL_ID}'`) + // Only app cells reach a managed server, through an SSH local forward; they run last. + const appCells = /\$appCellIds=@\(([^)]*)\)/.exec(runStep.run)?.[1] + expect(appCells?.split(',').map((id) => id.trim().replaceAll("'", ''))).toEqual([ + WINDOWS_CONVERT_CELL_ID, + ...WINDOWS_CLI_MATRIX_CELL_IDS + ]) + expect(runStep.run).toContain( + '$forwarding=@($cells | Where-Object {$appCellIds -contains $_}).Count' + ) + expect(runStep.run).toContain('-ForwardingAccounts $forwarding') + // A dispatched list may name them anywhere; they still run last. + expect(runStep.run).toContain( + '$cells=@($cells | Where-Object {$appCellIds -notcontains $_})+@($cells | Where-Object {$appCellIds -contains $_})' + ) + const provisioner = readFileSync( + join(projectDir, 'config/ci/windows-ssh-provider/preview-ssh/prove-preview-openssh.ps1'), + 'utf8' + ) + const max = Number(/\[ValidateRange\(1,(\d+)\)\]\[int\]\$Accounts/.exec(provisioner)?.[1]) + expect(max).toBeGreaterThanOrEqual( + WINDOWS_HOST_CELL_IDS.length + WINDOWS_ORCAD_CELL_IDS.length + 1 + ) }) }) diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 69df486ddc8..0edefd1c8e2 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -33,6 +33,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/codex/codex-structured-question-order.test.ts', 'src/main/ipc/pty/ipc/spawn-commit-ssh-lease-cardinality.test.ts', 'src/main/ipc/ssh-host-partition-session-export.test.ts', + 'src/main/ipc/ssh-host-server-on-connect-wiring.test.ts', + 'src/main/ipc/ssh-managed-server-move-conversion.test.ts', 'src/main/native-chat/agent-session-wire/agent-session-history-byte-accounting.test.ts', 'src/main/native-chat/agent-session-wire/agent-session-history-conversation-window.test.ts', 'src/main/native-chat/agent-session-wire/agent-session-history-forward-read-budget.test.ts', @@ -142,6 +144,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-wire-admission.test.ts', 'src/main/native-chat/agent-session-wire/structured-conversation-command.test.ts', + 'src/main/orcad/orcad-automations.test.ts', 'src/main/runtime/agent-session-conversation-clear-commit.test.ts', 'src/main/runtime/agent-session-conversation-name-store.test.ts', 'src/main/runtime/agent-session-death-evidence-persistence.test.ts', @@ -161,6 +164,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/orchestration/orchestration-party-location.test.ts', 'src/main/runtime/orchestration/structured-worker-journal-page.test.ts', 'src/main/runtime/rpc/methods/agent-launch-caller-selection.test.ts', + 'src/main/runtime/rpc/methods/agent-launch-instant-tab.test.ts', 'src/main/runtime/rpc/methods/agent-launch-pane-reservation.test.ts', 'src/main/runtime/rpc/methods/agent-launch-prestart-failure.test.ts', 'src/main/runtime/rpc/methods/agent-launch-replay.test.ts', @@ -186,6 +190,14 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/structured-claude-pending-rewind.test.ts', 'src/main/ssh-expired-lease-pane-readoption.test.ts', 'src/main/ssh-reattach-pane-cardinality.test.ts', + 'src/main/ssh/orcad-migration-cutover-coordinator.test.ts', + 'src/main/ssh/orcad-migration-delta-move.test.ts', + 'src/main/ssh/orcad-migration-delta-snapshot.test.ts', + 'src/main/ssh/orcad-migration-snapshot-resume.test.ts', + 'src/main/ssh/orcad-migration-source-fence.test.ts', + 'src/main/ssh/orcad-runtime-conversion.test.ts', + 'src/main/ssh/orcad-unreachable-setup-release.test.ts', + 'src/main/ssh/ssh-target-orcad-preflight.test.ts', 'src/main/worktree-identity-persistence.test.ts', 'src/main/worktree-removal-close-records.test.ts', 'src/main/worktree-removal-session-partition-fencing.test.ts', diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index af4d5651ea0..be0414e9611 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -102,6 +102,7 @@ "../src/main/windows-process-tree-kill.ts", "../src/main/windows-pty-root-identity.ts", "../src/main/windows/windows-process-table-cim-scan.ts", + "../src/main/windows/windows-process-table-timeout-error.ts", "../src/main/daemon/daemon-process-start-time.ts", "../src/main/daemon/daemon-process-identity-query.ts", "../src/main/startup/startup-diagnostics.ts", diff --git a/docs/reference/orcad-operations.md b/docs/reference/orcad-operations.md index adc18586c21..961b7fa1e00 100644 --- a/docs/reference/orcad-operations.md +++ b/docs/reference/orcad-operations.md @@ -66,6 +66,25 @@ the daemon shares the service cgroup and a combined-unit stop ends live terminal it is populated from `/proc/self/cgroup`, so it reports the isolation the daemon actually has rather than what the launcher intended. +## `orca serve` on this machine + +`orca serve` runs on the local orcad slot by default. The CLI asks the app's +`out/main/orcad/orcad-local-serve-selection-entry.js` (run as plain Node on the app's executable) +which host to use. Any reason orcad cannot serve falls back to Electron serve with one +`[serve] using Electron serve: ` line on stderr. Those reasons are: no slot for this host, +no template in the install, the pinned Node could not be fetched, or a failed native preflight. + +- `ORCA_SERVE_RUNTIME=electron` keeps Electron serve and skips the question. `orcad` (or unset) is + the default, and any other value falls back with a reason. +- Packaged macOS stays on Electron: only Electron serve, supervised by the CLI, can take a remote + app update there, and orcad has no updater. Recipe-JSON serve has no handoff and uses orcad. +- Windows serves on orcad too. Both hosts share `\daemon`, so the daemon pipe name + (hashed from that path) is the same, and the relocated Electron daemon host changes only the + executable, not the pipe. The `orcad-serve-mode-switch-windows` e2e job checks D7 there, in the + daily run and on PRs routed to it; it does not block merges. + +The slot and its pinned Node live under the desktop's `/orcad-artifacts`. + ## Bind policy `--bind `, **default `127.0.0.1`**. @@ -84,6 +103,16 @@ nothing can reach. Under the shipping design a client reaches a remote orcad over an SSH local port-forward, so loopback is the correct default and the pairing credential travels over SSH. +A host whose sshd refuses forwarding (`AllowTcpForwarding no`) is reached through the stdio +bridge instead: the client keeps the same local port, and each connection to it opens one SSH +exec channel running a small script on the host's pinned Node that dials orcad's loopback port. +Windows hosts run it as the host script's `stdio-bridge` op and frame bytes as base64 lines, +because a PowerShell DefaultShell re-decodes native output. Bridges are capped below OpenSSH's +default `MaxSessions` of 10 per connection; further connections wait for a free one. The choice +is made each time the tunnel starts (`orcad-managed-tunnel-transport.ts`), so nothing is +recorded per host, and only a host where even the bridge cannot run keeps the relay, recorded as +`ssh_tunnel_unavailable`. + ## Data root and the instance lock The data root is `$ORCA_USER_DATA`, else `$XDG_DATA_HOME/Orca`, else `~/.orca`. @@ -102,11 +131,16 @@ It refuses to start when: A root that is merely too permissive and that we own is tightened to `0700` rather than refused — orcad stores credentials there unsealed (no OS keyring on this host), so the goal is a private root, and refusing when we could just fix it helps nobody. We refuse when the -permissions are not ours to fix. Windows is exempt from the owner and mode checks: ACLs are -not expressible as a POSIX mode, and `statSync().mode` there reports a synthesized one. +permissions are not ours to fix. Windows has no owner or mode check, because ACLs are not +expressible as a POSIX mode and `statSync().mode` there reports a synthesized one. Instead +orcad restricts the root's ACL to its own user with `icacls` (the same verified restriction +`secure-file.ts` applies to credential files) and refuses with `orcad_data_root_shared` when +that cannot be applied. A dead holder's record is reclaimed (PID plus process start time, so a recycled PID does not -read as alive). A record belonging to a different identity is never reclaimed. +read as alive). On Windows the start time is the kernel creation time read through the +process-tree addon the slot stages; without the addon it is null and the PID alone fences, +which errs toward "held". A record belonging to a different identity is never reclaimed. **The lock scopes one role — who is the runtime.** It deliberately says nothing about the daemon, which lives under `/daemon` and fences its own endpoint with its own PID @@ -150,6 +184,47 @@ An external supervisor (systemd, launchd, a process manager). orcad conforms to exits with code 1 if teardown stalls. The bundled runtime also stops gracefully if its launcher's IPC channel closes. On POSIX, both the launcher and runtime ignore `SIGHUP`, so terminal hangups do not stop a headless host. Use `SIGTERM` or `SIGINT` to stop it. +- **Stop requests.** A file stops orcad the same way `SIGTERM` does, without a PID that may + since have been reused by another process: + - `.orcad-stop-request` beside `orcad.js` in the running slot. orcad deletes it and stops. + - An instance-bound request in the data root, named + `.orcad-managed-stop-request.`. orcad stops only when it + names this orcad's version, runtime ID, PID, start time and lock nonce, and while the + instance lock still holds that record. The file is kept as evidence. + - `orcad --complete-managed-stop ''` writes that request, waits for the + instance to exit, and prints one JSON line whose `verdict` is `live`, `unverifiable` or + `exited`. `exited` needs proof: no process with that PID, or a PID whose start time shows + it now belongs to another process. On `exited` it writes + `/orcad-stop-receipts/.json`. It exits 0 whenever it printed a + verdict, 64 for a malformed invocation, and 1 for a failure before any verdict, which is + never evidence of exit. + - A request with `retireIdleDaemon: true` asks orcad to retire the terminal daemon too. This + is best effort and never blocks or fails the stop: + - The daemon is retired only when it proves it owns no live session across every + generation. + - A busy daemon (`live`) or one whose state cannot be proven (`unverifiable`) stays up with + its terminals, and orcad reopens new-terminal admission before exiting. + - The completed-stop receipt records `retirement` as `retired`, `live` or `unverifiable`. + If orcad exits without recording an outcome, the receipt says `unverifiable`. + - `orcad --cancel-managed-stop ''` withdraws a request orcad has not acted on. + orcad and the canceller each try to create `.decision.json` exclusively, + so exactly one wins. `canceled` means orcad keeps running and the request file is removed; + `dispatched` means orcad already began stopping, and only the completion can say how it + ended. + - A build advertises all of the above with `health.stopRequests: 1` in its readiness line. + Clients stop such a build through the slot request file and older builds with `SIGTERM`, + after corroborating the PID with readiness either way. A launch clears a slot request + that the previous process never consumed. +- **Decommissioning a managed slot.** An Orca client decommissions through the same activation + journal and fence as deploy and rollback. It refuses while the terminal census reports live + or uncounted terminals, stops the instance with a managed request that also asks to retire + the daemon, and records that no version is active only after `exited` is proven. A stop + that did not finish is cancelled; if orcad already acted on it, or the host cannot answer, + the fence stays for recovery. +- **Instance lock.** `/orcad.lock` names the running orcad. A record that is + unreadable, malformed or over 64 KiB is never reclaimed: orcad exits 78 until an operator + removes it. A shutdown whose teardown failed keeps the lock until the process exits, so a + second orcad cannot start beside a writer that may still be running. - **Exit codes.** | Code | Meaning | Supervisor should | @@ -202,6 +277,65 @@ above, stop orcad, then stop the daemon named by `health.terminalDaemon.pid`. Only report it `exited` after verification on the execution host; loss of contact is `unverifiable`. +### Windows hosts + +What differs on a Windows SSH host, and what deliberately does not: + +- **Stop path.** A signal is TerminateProcess on Windows: no flush, no lock release. The + slot's `.orcad-stop-request` file (and the managed, instance-bound request) is therefore the + only graceful stop. A detached orcad receives no console control events, so the listener + (`fs.watch` plus a one-second poll) is what stops it; the packaged-slot test proves it exits + cleanly within the 15 s shutdown deadline on every server lane, Windows included. +- **Exit proof.** `--complete-managed-stop` proves a reused PID by the addon's creation time. + Without the addon a live PID stays `live` or `unverifiable`, never `exited`. +- **Daemon endpoint.** The terminal daemon listens on a named pipe + (`\\?\pipe\orca-terminal-host-v-`), not a socket under the data root. +- **Leaving sshd's job.** orcad is started outside the SSH session's kill-on-close job, so the + daemon it forks inherits no such job and outlives the connection the same way. +- **Per-PTY jobs.** Each ConPTY child gets its own job (`windows-pty-job.ts`), and Git Bash / + MSYS panes follow [`windows-msys-job-breakaway.md`](./windows-msys-job-breakaway.md) + unchanged. A ConPTY smoke test runs inside a process started exactly that way (breakaway, + no window) on the Windows server lanes. +- **No daemon-host relocation.** The desktop copies its runtime to `%LOCALAPPDATA%` because + the NSIS updater deletes the install directory under a running daemon + ([`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md)). orcad slots are + versioned directories that nothing deletes while a process runs from them: Windows refuses + to delete a running image, and GC treats an in-use slot as live. + +## Idle exit (client-managed orcad only) + +An orcad that a desktop client launched over SSH stops itself, like the relay, once its host has +been unused for 15 minutes. The client's launch sets `ORCA_ORCAD_MANAGED_ACTIVATION_ROOT`; an +orcad started by hand, by a supervisor, or as a paired server never carries it and never idles +out. + +"Unused" means every one of these held on every check for the whole period: + +- no client socket open and no RPC request running; +- no terminal in the PTY provider, and the daemon answered with zero live sessions (a daemon + that does not answer keeps orcad up); +- no agent reporting `working`; +- no staged migration into this server; +- no enabled automation and no automation run still in flight (nothing on the host would start + orcad again for the next scheduled run, so a server with an enabled automation never idles out); +- no activation fence on the host (an update, rollback, decommission or recovery in flight). + +The stop is the ordinary graceful shutdown, which disconnects from the daemon and never shuts it +down, so it cannot kill a terminal. It then asks the daemon to retire only if the daemon itself +proves it holds no session. Before stopping, orcad writes `/orcad-idle-stop.json`; +the next start reports it once as `health.previousIdleStop` and removes it, so a later crash is +never read as an idle stop. A managed start with no record reports `previousIdleStop: null`. + +The client starts a stopped server again, whatever stopped it (an idle stop, a kill, a host +reboot): on every connect, on every fresh tunnel (including after the client wakes from sleep), +and before a call through an environment the client restored at launch. A server that does not +answer is checked on the host; only a proven exit starts the activated slot, under the activation +fence, and the status line shows "Starting managed server…". A daemon that survived is adopted +with its terminals; after a reboot both start fresh. A process that is live or cannot be proven +gone is left alone, and a start that fails keeps the host managed with the reason and orcad.log's +tail, never as a verdict about its terminals. `ORCA_E2E_ORCAD_IDLE_TIMEOUT_MS` shortens the idle +period for tests; the client forwards it to the servers it launches. + ## Health The readiness payload carries a `health` object: diff --git a/docs/reference/ssh-execution-boundary.md b/docs/reference/ssh-execution-boundary.md index 12aeea89753..62ecb83838d 100644 --- a/docs/reference/ssh-execution-boundary.md +++ b/docs/reference/ssh-execution-boundary.md @@ -42,9 +42,9 @@ Reconnect re-attaches to the same live PTYs and replays a bounded buffer (`REPLA ## Updating Orca strands relay-backed terminals -There is a third outcome that is neither of the two above, and the vocabulary matters: the work does not stop, it becomes permanently unreachable. +There is a third outcome that is neither of the two above, and the vocabulary matters: the work does not stop, it becomes unreachable through the new build's own relay. -The relay's install directory — and therefore its socket path — is namespaced by a content hash of the relay bundle (`computeRemoteRelayDir` in `src/main/ssh/ssh-relay-versioned-install.ts`, consumed by `resolveRemoteInstallState` in `src/main/ssh/ssh-relay-deploy.ts`), and the daemon refuses any client whose bundle hash differs (`handleDaemonHandshakeFrame` in `src/relay/relay-handshake.ts`, exit `EXIT_CODE_VERSION_MISMATCH` 42). Two builds whose relay protocol is byte-identical still refuse each other. So the first reconnect after an app update deploys a new relay at a path the incumbent was never listening on, and cannot reach it even in principle. Every PTY the incumbent owns is `unverifiable` — running, unreachable, and never `exited`. The client's leases are attempted against a relay that never minted their ids, expired on the not-found answer (`handlePtyReattachFailure` in `src/main/ssh/ssh-relay-session.ts`), and the pane falls back to a cold-restore agent resume — or to a bare shell when no resumable provider session was captured for it. The old relay keeps its directory pinned against GC, because its socket really is live (`hasLiveRelaySocket` in `src/main/ssh/remote-install-gc.ts`). See #13852. +The relay's install directory — and therefore its socket path — is namespaced by a content hash of the relay bundle (`computeRemoteRelayDir` in `src/main/ssh/ssh-relay-versioned-install.ts`, consumed by `resolveRemoteInstallState` in `src/main/ssh/ssh-relay-deploy.ts`), and the daemon refuses any client whose bundle hash differs (`handleDaemonHandshakeFrame` in `src/relay/relay-handshake.ts`, exit `EXIT_CODE_VERSION_MISMATCH` 42). Two builds whose relay protocol is byte-identical still refuse each other. So the first reconnect after an app update deploys a new relay at a path the incumbent was never listening on, and this build's own bridge cannot reach the incumbent. The incumbent's own bridge can: its version directory still holds its `relay.js` and `.version`, so `relay.js --connect` run from there presents its own hash by construction. Each deploy takes a census of this target's older endpoints (`startPreviousRelayCensus` in `src/main/ssh/ssh-previous-relay-terminals.ts`); while one is live or unverifiable, a not-found reattach keeps its lease and answers `SSH_PTY_HELD_BY_PREVIOUS_RELAY` instead of expiring. On POSIX hosts the pane is then reattached through that older bridge (`SshLegacyRelayRoute` in `src/main/ssh/ssh-legacy-relay-route.ts`), which takes the PTY owner role without output flow control, and every later operation on that PTY id is routed there (`src/main/providers/ssh-pty-legacy-relay-delegation.ts`). When the last pane it serves exits, the route hangs up and the incumbent's own grace retires it. On Windows the census probes each older version directory's pipe for the target (`src/main/ssh/ssh-previous-relay-windows-census.ts`), and a census that cannot run or does not know the host platform counts as `unverifiable`, never as "no older relay". The connect-time host census, which decides whether a host may convert to a managed server before any relay session exists, instead lists every `orca-relay-*` pipe on a Windows machine and maps each to the relay instance that owns it through that instance's credential file or pipe marker, whichever desktop launched it (`src/main/ssh/ssh-host-relay-windows-inventory.ts`); a pipe no directory accounts for counts as `unverifiable` unless the host proves it another account's. The migration terminal gate (`assessOrcadMigrationTerminals`) answers `exited` only from a complete inventory: every relay session answered, or, with no session, a host census proved the endpoints idle. A missing or failed inventory is `unverifiable` even when this desktop leases nothing. Where no route can be opened — Windows named pipes, a socket relocated under the short `/tmp` base, a bridge that fails — the PTY stays `unverifiable`: the pane says it is still running under the previous Orca version, and is never respawned. The old relay keeps its directory pinned against GC while its socket is live (`hasLiveRelaySocket` in `src/main/ssh/remote-install-gc.ts`). See #13852: on POSIX hosts that is no longer a dead end, because a terminal held by a previous relay version now resumes in its pane through that relay's own bridge. The peer model does not have this failure, and that is the concrete reason behind "One host, one model" below. The daemon's endpoint is namespaced by a **semantic protocol version** rather than a build (`daemon-v.sock`, from `getDaemonSocketPath` in `src/main/daemon/daemon-spawner.ts`), every earlier protocol version stays attachable (`PROTOCOL_VERSION` in `src/main/daemon/daemon-protocol-version.ts`), and a daemon holding live sessions is preserved across a version change instead of replaced (`shouldPreserveDaemonWithLiveSessions` in `src/main/daemon/daemon-replacement-preflight.ts`). diff --git a/docs/reference/windows-edr-posture.md b/docs/reference/windows-edr-posture.md index d1845ec5cbe..39bb20388e8 100644 --- a/docs/reference/windows-edr-posture.md +++ b/docs/reference/windows-edr-posture.md @@ -396,6 +396,16 @@ built without the addon or a job that refuses breakaway, and a refusal there is reported as `ORCA_RELAY_LAUNCH_REFUSED`. The Windows SSH-host lanes run with no WMI grant and assert the breakaway route. +`orcad.js` exposes the same launcher (`src/shared/windows-breakaway-launcher.ts`) +with **no** WMI fallback: a host that cannot break away refuses the orcad launch. +Windows orcad operations start **no PowerShell**: sshd's DefaultShell runs the +pinned `node.exe` directly with plain path arguments, against one content-addressed +host script staged beside the slots (`src/main/ssh/orcad-windows-host-script.ts`). +Only a `node.exe` path that itself needs quoting (a profile name with a space) +falls back to one unencoded `powershell.exe -Command`. The launch waits for +readiness host-side, one exec per 20 s at most, rather than re-running an exec +every 500 ms. + ## Signing is not the gate The most useful calibration in the whole incident set came from the reporter's diff --git a/docs/site/content/docs/remote-servers.mdx b/docs/site/content/docs/remote-servers.mdx index f23e3d94a6a..25ad464f002 100644 --- a/docs/site/content/docs/remote-servers.mdx +++ b/docs/site/content/docs/remote-servers.mdx @@ -167,6 +167,8 @@ orca-ide serve --port 6768 --pairing-address 100.64.1.20 Use only one host mode at a time. If the Orca desktop app is already sharing that computer, do not start a second `orca serve` process for the same setup. +`orca serve` runs on Orca's own Node server (orcad) by default. If orcad can't serve on this computer, it uses the desktop app's server instead and prints one line saying why. On a packaged macOS app it currently always uses the desktop app's server, so paired clients can still update it. To always use the desktop app's server, set `ORCA_SERVE_RUNTIME=electron`. + ### Mobile from a headless server For the Orca mobile app, request a mobile-scoped QR code and link: diff --git a/docs/site/content/docs/telemetry.mdx b/docs/site/content/docs/telemetry.mdx index a8b8715812a..305304c951d 100644 --- a/docs/site/content/docs/telemetry.mdx +++ b/docs/site/content/docs/telemetry.mdx @@ -25,7 +25,7 @@ The categories of behavior we observe: - **Agent errors** — a coarse error category and which agent kind was involved. We never see raw error messages or stack traces; per-incident detail stays in a local diagnostic trace file on your machine and only reaches Orca if you explicitly share a diagnostic bundle. - **Settings** — when you toggle one of a small whitelisted set of feature-flag or UX preferences. We record which preference changed and whether it's a boolean or an enum, never the raw value of any free-form setting. - **Privacy controls** — when you opt in or out of telemetry, so we can tell from aggregate data whether our consent UI is working. -- **SSH remote runtime** — once per SSH host per app session, which runtime Orca used to run on that host and why: coarse host facts (operating system, CPU architecture, C library family and glibc minor version, and the host's Node.js major version when Orca runs on it), the category of any refusal, whether a runtime was uploaded, and a coarse duration bucket. Never the hostname, user name, paths, or raw error text. +- **SSH remote runtime** — once per SSH host and outcome per app session, which runtime Orca used to run on that host and why, or that its runtime check could not be confirmed or failed: coarse host facts (operating system, CPU architecture, C library family and glibc minor version, and the host's Node.js major version when Orca runs on it), the category of any refusal (including security software blocking the runtime on Windows), whether a runtime was uploaded, and a coarse duration bucket. Never the hostname, user name, paths, or raw error text. Fields include fixed enum values, version strings, numeric counts and revisions, and random local IDs. Conversation IDs allow repeated token summaries for the same conversation to be associated; they are pseudonymous. No free-form strings from any UI input ever leave your machine. diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 8dba09bce1a..263268833ba 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -10,6 +10,7 @@ import { CLI_MAIN_ENTRY_NAMES, createPlainNodeEntryGuardPlugin } from './config/build-plugins/plain-node-entry-guard' +import { ORCAD_LOCAL_SERVE_SELECTION_ENTRY } from './src/shared/orcad-local-serve-selection' import packageJson from './package.json' with { type: 'json' } const BUNDLED_MAIN_DEPENDENCIES = new Set([ @@ -269,6 +270,10 @@ export const electronViteConfig: UserConfig = { // Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults // can't take down the main process (issue #7547). 'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'), + // Why: `orca serve` runs it under ELECTRON_RUN_AS_NODE so the CLI never bundles orcad prep. + [ORCAD_LOCAL_SERVE_SELECTION_ENTRY]: resolve( + 'src/main/orcad/orcad-local-serve-selection-entry.ts' + ), // Why: a worker thread survives the macOS 26 AppKit main-thread deadlock // without paying for another Electron process. 'main-thread-hang-watchdog-entry': resolve( diff --git a/src/cli/command-scoped-flag-help.ts b/src/cli/command-scoped-flag-help.ts index d4b3abcfaf9..0750e3d5768 100644 --- a/src/cli/command-scoped-flag-help.ts +++ b/src/cli/command-scoped-flag-help.ts @@ -14,6 +14,17 @@ const COMMAND_SCOPED_FLAG_HELP: Record> = { reference: '--reference Print one bundled reference by name', references: '--references List the bundled reference names for a topic' }, + 'environment update': { + force: '--force Restart over running terminals instead of deferring the update' + }, + 'environment recover': { + 'accept-changed-state': + '--accept-changed-state Restore the prelaunch snapshot over state a rejected build changed', + yes: '--yes Confirm discarding what the rejected build changed' + }, + 'environment stop': { + yes: '--yes Confirm stopping the server and unlinking it from this machine' + }, 'file open': { focus: FILE_OPEN_FOCUS_HELP }, diff --git a/src/cli/handler-group-manifest.ts b/src/cli/handler-group-manifest.ts index b4f0473ee1e..e4bff998549 100644 --- a/src/cli/handler-group-manifest.ts +++ b/src/cli/handler-group-manifest.ts @@ -217,6 +217,18 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ ], load: async () => (await import('./handlers/environment.js')).ENVIRONMENT_HANDLERS }, + { + name: 'managed-server', + keys: [ + 'environment status', + 'environment update', + 'environment rollback', + 'environment recover', + 'environment stop', + 'environment cancel-stop' + ], + load: async () => (await import('./handlers/managed-server.js')).MANAGED_SERVER_HANDLERS + }, { name: 'linear', keys: [ diff --git a/src/cli/handlers/managed-server-format.ts b/src/cli/handlers/managed-server-format.ts new file mode 100644 index 00000000000..594a47ef79c --- /dev/null +++ b/src/cli/handlers/managed-server-format.ts @@ -0,0 +1,27 @@ +import type { OrcadManagedRuntimeStatus } from '../../shared/orcad-managed-runtime' + +function count(value: number | null): string { + return value === null ? 'unverifiable' : String(value) +} + +export function formatManagedServerStatus(status: OrcadManagedRuntimeStatus): string { + const lines = [ + `Active version: ${status.activeVersion ?? 'none'}`, + `Previous version: ${status.previousVersion ?? 'none'}${status.rollbackAvailable ? ' (rollback available)' : ''}`, + `Live terminals: ${count(status.terminals.liveSessions)}` + ] + if (status.recovery) { + lines.push( + `Interrupted ${status.recovery.operation} of ${status.recovery.version} (${status.recovery.phase}); run \`orca environment recover\`.` + ) + } + if (status.migration) { + lines.push(`Migration into this server: ${status.migration.phase}`) + } + if (status.deferredUpdate) { + lines.push( + `Deferred update to ${status.deferredUpdate.candidateVersion}: ${status.deferredUpdate.reason}` + ) + } + return lines.join('\n') +} diff --git a/src/cli/handlers/managed-server-reconnect.ts b/src/cli/handlers/managed-server-reconnect.ts new file mode 100644 index 00000000000..b9e31d82413 --- /dev/null +++ b/src/cli/handlers/managed-server-reconnect.ts @@ -0,0 +1,50 @@ +/** + * An update or rollback restarts the managed server, and a CLI talking to that runtime then sees + * its connection close mid-call. That is expected, not a failure: read the host's status once the + * runtime answers again and report what the action left behind. + */ +import type { OrcadManagedRuntimeStatus } from '../../shared/orcad-managed-runtime' +import type { HandlerContext } from '../dispatch' +import { printResult } from '../format' +import { RuntimeClientError, type RuntimeRpcSuccess } from '../runtime-client' + +const STATUS_ATTEMPTS = 15 +const STATUS_RETRY_MS = 2_000 + +export async function reportAfterClosedConnection( + { client, json }: HandlerContext, + selector: { selector: string }, + sleep: (ms: number) => Promise = (ms) => new Promise((done) => setTimeout(done, ms)) +): Promise { + for (let attempt = 0; attempt < STATUS_ATTEMPTS; attempt++) { + await sleep(STATUS_RETRY_MS) + let response: RuntimeRpcSuccess + try { + response = await client.call('managedServer.status', selector) + } catch { + continue + } + const status = response.result + if (status.recovery) { + throw new RuntimeClientError( + 'managed_server_interrupted', + `The ${status.recovery.operation} of ${status.recovery.version} was interrupted. Run \`orca environment recover\`.`, + status + ) + } + if (status.deferredUpdate) { + throw new RuntimeClientError('managed_server_deferred', status.deferredUpdate.reason, status) + } + printResult( + response, + json, + (value) => + `The server restarted during the action and now runs ${value.activeVersion ?? 'no version'}.` + ) + return + } + throw new RuntimeClientError( + 'managed_server_in_progress', + 'The connection closed while the server restarted, and it has not answered since. Check `orca environment status`.' + ) +} diff --git a/src/cli/handlers/managed-server.test.ts b/src/cli/handlers/managed-server.test.ts new file mode 100644 index 00000000000..3ec02a79961 --- /dev/null +++ b/src/cli/handlers/managed-server.test.ts @@ -0,0 +1,289 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCAD_RECOVERY_CHANGED_STATE_CODE } from '../../shared/orcad-managed-runtime' +import { MANAGED_SERVER_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import { RuntimeClientError } from '../runtime-client' +import { MANAGED_SERVER_ACTION_TIMEOUT_MS, MANAGED_SERVER_HANDLERS } from './managed-server' + +function envelope(result: unknown) { + return { id: 'r', ok: true, result, _meta: { runtimeId: 'runtime-1' } } +} + +function client(result: unknown, capabilities = [MANAGED_SERVER_RUNTIME_CAPABILITY]) { + return vi.fn(async (method: string) => + method === 'status.get' ? envelope({ capabilities }) : envelope(result) + ) +} + +async function run( + command: string, + call: ReturnType, + flags: [string, string | boolean][] +) { + const handler = MANAGED_SERVER_HANDLERS[command] + if (!handler) { + throw new Error(`no handler for ${command}`) + } + await handler({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the handlers only call client.call. + client: { call } as never, + cwd: '/tmp', + flags: new Map([['environment', 'build-box'], ...flags]), + json: false + }) +} + +afterEach(() => vi.restoreAllMocks()) + +describe('managed server CLI verbs', () => { + it('refuses on a runtime that does not advertise managed servers, before calling the action', async () => { + const call = client({ outcome: 'none' }, []) + await expect(run('environment recover', call, [])).rejects.toMatchObject({ + code: 'incompatible_runtime' + }) + expect(call).toHaveBeenCalledTimes(1) + }) + + it('reads an older runtime’s method_not_found as the same refusal', async () => { + const call = vi.fn(async (method: string) => { + if (method === 'status.get') { + return envelope({ capabilities: [MANAGED_SERVER_RUNTIME_CAPABILITY] }) + } + throw new RuntimeClientError('method_not_found', 'Unknown method') + }) + await expect(run('environment status', call, [])).rejects.toMatchObject({ + code: 'incompatible_runtime' + }) + }) + + it('needs --yes before stopping, and then calls the same stop the settings use', async () => { + const stopped = { + outcome: 'unlinked', + verdict: 'exited', + environmentId: 'env-1', + sshTargetId: 'ssh-1', + stoppedVersion: '1.0.0', + retirement: null + } + const unconfirmed = client(stopped) + await expect(run('environment stop', unconfirmed, [])).rejects.toMatchObject({ + code: 'confirmation_required' + }) + expect(unconfirmed).not.toHaveBeenCalled() + + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + const confirmed = client(stopped) + await run('environment stop', confirmed, [['yes', true]]) + expect(confirmed).toHaveBeenCalledWith( + 'managedServer.stop', + { selector: 'build-box' }, + { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + expect(log).toHaveBeenCalledWith('Stopped build-box and unlinked it from this machine.') + }) + + it('fails the command with the refusal, so scripts see a non-zero exit', async () => { + const refusal = { + outcome: 'refused', + verdict: 'live', + code: 'orcad_stop_active_environment', + reason: 'Choose another Active Server in Advanced before stopping this server.' + } + await expect(run('environment stop', client(refusal), [['yes', true]])).rejects.toMatchObject({ + code: 'managed_server_refused', + message: refusal.reason, + data: refusal + }) + }) + + it('passes --force to an update, and reports a deferred one as unsettled', async () => { + const call = client({ + outcome: 'deferred', + candidateVersion: '2.0.0', + code: 'orcad_update_terminals_running', + reason: 'Terminals are running.' + }) + await expect(run('environment update', call, [])).rejects.toMatchObject({ + code: 'managed_server_deferred' + }) + expect(call).toHaveBeenCalledWith( + 'managedServer.update', + { selector: 'build-box', force: false }, + { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + + vi.spyOn(console, 'log').mockImplementation(() => undefined) + const forced = client({ + outcome: 'updated', + environment: { name: 'build-box' }, + activeVersion: '2.0.0' + }) + await run('environment update', forced, [['force', true]]) + expect(forced).toHaveBeenCalledWith( + 'managedServer.update', + { selector: 'build-box', force: true }, + { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + }) + + it('fails on an outcome a newer desktop added instead of printing undefined', async () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + await expect( + run('environment cancel-stop', client({ outcome: 'future-outcome' }), []) + ).rejects.toMatchObject({ code: 'managed_server_future-outcome' }) + expect(log).not.toHaveBeenCalled() + }) + + it('prints a readable status', async () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + await run( + 'environment status', + client({ + activeVersion: '1.0.0', + previousVersion: '0.9.0', + rollbackAvailable: true, + recovery: null, + terminals: { liveSessions: null }, + migration: null, + deferredUpdate: null + }), + [] + ) + expect(log.mock.calls[0]?.[0]).toBe( + 'Active version: 1.0.0\nPrevious version: 0.9.0 (rollback available)\nLive terminals: unverifiable' + ) + }) + + it('gives mutating actions a budget past the desktop’s own deadlines, and status the default', async () => { + const stopped = { + outcome: 'unlinked', + verdict: 'exited', + environmentId: 'env-1', + sshTargetId: 'ssh-1', + stoppedVersion: '1.0.0', + retirement: null + } + vi.spyOn(console, 'log').mockImplementation(() => undefined) + const stop = client(stopped) + await run('environment stop', stop, [['yes', true]]) + expect(stop).toHaveBeenCalledWith( + 'managedServer.stop', + { selector: 'build-box' }, + { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + const status = client({ + activeVersion: '1.0.0', + previousVersion: null, + rollbackAvailable: false, + recovery: null, + terminals: { liveSessions: 0 }, + migration: null, + deferredUpdate: null + }) + await run('environment status', status, []) + expect(status).toHaveBeenCalledWith( + 'managedServer.status', + { selector: 'build-box' }, + undefined + ) + }) + + it('reports a timed-out action as possibly still running, never as a failure of the action', async () => { + const call = vi.fn(async (method: string) => { + if (method === 'status.get') { + return envelope({ capabilities: [MANAGED_SERVER_RUNTIME_CAPABILITY] }) + } + throw new RuntimeClientError( + 'runtime_timeout', + 'Timed out waiting for the Orca runtime to respond.' + ) + }) + await expect(run('environment update', call, [])).rejects.toMatchObject({ + code: 'managed_server_in_progress', + message: expect.stringContaining('orca environment status') + }) + }) + + // BUG-21: an update restarts the server, so the call's connection closes before it answers. + it('reports what a restarting update left behind instead of failing on the closed connection', async () => { + vi.useFakeTimers() + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + let updated = false + const call = vi.fn(async (method: string) => { + if (method === 'status.get') { + return envelope({ capabilities: [MANAGED_SERVER_RUNTIME_CAPABILITY] }) + } + if (method === 'managedServer.update') { + updated = true + throw new RuntimeClientError( + 'runtime_unavailable', + 'The Orca runtime closed the connection before responding.' + ) + } + return envelope({ activeVersion: '0.2.0+bb01', recovery: null, deferredUpdate: null }) + }) + try { + const done = run('environment update', call, []) + await vi.advanceTimersByTimeAsync(2_000) + await done + expect(updated).toBe(true) + expect(log).toHaveBeenCalledWith(expect.stringContaining('now runs 0.2.0+bb01')) + } finally { + vi.useRealTimers() + } + }) + + it('reports an update the restart left interrupted as needing recover', async () => { + vi.useFakeTimers() + const call = vi.fn(async (method: string) => { + if (method === 'status.get') { + return envelope({ capabilities: [MANAGED_SERVER_RUNTIME_CAPABILITY] }) + } + if (method === 'managedServer.update') { + throw new RuntimeClientError('runtime_unavailable', 'closed') + } + return envelope({ + activeVersion: '0.1.0+aa01', + recovery: { operation: 'activate', version: '0.2.0+bb01', phase: 'snapshot-captured' }, + deferredUpdate: null + }) + }) + try { + const done = run('environment update', call, []).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(2_000) + expect(await done).toMatchObject({ code: 'managed_server_interrupted' }) + } finally { + vi.useRealTimers() + } + }) + + it('restores changed state only with --accept-changed-state --yes, and its refusal names the flags', async () => { + const refusal = { + outcome: 'refused', + verdict: 'unverifiable', + code: ORCAD_RECOVERY_CHANGED_STATE_CODE, + reason: 'The launched build changed profile state. Recover to restore the prelaunch snapshot.' + } + await expect(run('environment recover', client(refusal), [])).rejects.toMatchObject({ + code: 'managed_server_refused', + message: expect.stringContaining('--accept-changed-state --yes') + }) + + const unconfirmed = client({ outcome: 'none' }) + await expect( + run('environment recover', unconfirmed, [['accept-changed-state', true]]) + ).rejects.toMatchObject({ code: 'confirmation_required' }) + expect(unconfirmed).not.toHaveBeenCalled() + + vi.spyOn(console, 'log').mockImplementation(() => undefined) + const confirmed = client({ outcome: 'none' }) + await run('environment recover', confirmed, [ + ['accept-changed-state', true], + ['yes', true] + ]) + expect(confirmed).toHaveBeenCalledWith( + 'managedServer.recover', + { selector: 'build-box', acceptChangedState: true }, + { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + }) +}) diff --git a/src/cli/handlers/managed-server.ts b/src/cli/handlers/managed-server.ts new file mode 100644 index 00000000000..07c529f2042 --- /dev/null +++ b/src/cli/handlers/managed-server.ts @@ -0,0 +1,227 @@ +/** + * `orca environment status|update|rollback|recover|stop|cancel-stop`: the Managed servers + * actions over runtime RPC. Each call is gated on the runtime's managedServer.v1 capability, and + * an older runtime's method_not_found reads the same as a missing capability. + */ +import type { + OrcadManagedCancelStopResult, + OrcadManagedDeployResult, + OrcadManagedRecoveryResult, + OrcadManagedRollbackResult, + OrcadManagedRuntimeStatus, + OrcadManagedStopResult +} from '../../shared/orcad-managed-runtime' +import { ORCAD_RECOVERY_CHANGED_STATE_CODE } from '../../shared/orcad-managed-runtime' +import { MANAGED_SERVER_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import type { RuntimeStatus } from '../../shared/runtime-types' +import type { CommandHandler, HandlerContext } from '../dispatch' +import { getRequiredStringFlag } from '../flags' +import { printResult } from '../format' +import { RuntimeClientError, type RuntimeRpcSuccess } from '../runtime-client' +import { formatManagedServerStatus } from './managed-server-format' +import { reportAfterClosedConnection } from './managed-server-reconnect' + +const UNSUPPORTED_MESSAGE = + 'This Orca runtime cannot manage servers over SSH. Run this on the computer whose Orca desktop app deployed the server, after updating Orca there.' + +// Why 20 minutes: the desktop runs the whole action inline, and a Windows runtime promotion (5 min) +// plus a readiness wait (5 min) on a slow host already outlast the 60 s RPC default. +export const MANAGED_SERVER_ACTION_TIMEOUT_MS = 20 * 60_000 +const READ_ONLY_METHODS = new Set(['managedServer.status']) + +function unsupported(): RuntimeClientError { + return new RuntimeClientError('incompatible_runtime', UNSUPPORTED_MESSAGE) +} + +async function callManagedServer( + { client }: HandlerContext, + method: string, + params: Record +): Promise> { + const status = await client.call('status.get') + if (!status.result.capabilities?.includes(MANAGED_SERVER_RUNTIME_CAPABILITY)) { + throw unsupported() + } + const readOnly = READ_ONLY_METHODS.has(method) + try { + return await client.call( + method, + params, + readOnly ? undefined : { timeoutMs: MANAGED_SERVER_ACTION_TIMEOUT_MS } + ) + } catch (error) { + if (error instanceof RuntimeClientError && error.code === 'method_not_found') { + throw unsupported() + } + // Why not a failure: the desktop keeps running the action after this client stops waiting. + if (!readOnly && error instanceof RuntimeClientError && error.code === 'runtime_timeout') { + throw new RuntimeClientError( + 'managed_server_in_progress', + 'Stopped waiting, but the desktop may still be running this action. Check `orca environment status` before retrying.' + ) + } + throw error + } +} + +/** Null once the restart's outcome was reported from status instead. */ +async function callExpectingRestart( + context: HandlerContext, + method: string, + params: { selector: string } & Record +): Promise | null> { + try { + return await callManagedServer(context, method, params) + } catch (error) { + if (!(error instanceof RuntimeClientError) || error.code !== 'runtime_unavailable') { + throw error + } + await reportAfterClosedConnection(context, { selector: params.selector }) + return null + } +} + +function selectorOf(context: HandlerContext): { selector: string } { + return { selector: getRequiredStringFlag(context.flags, 'environment') } +} + +type Outcome = { outcome: string; code?: string; reason?: string } + +/** + * Prints a result whose outcome is in `settled`; throws any other so scripts see a non-zero exit. + * Why an allow-list: an outcome a newer desktop adds must fail loudly, not print `undefined`. + */ +function report( + response: RuntimeRpcSuccess, + json: boolean, + settled: readonly TSettled[], + done: (result: Extract) => string, + nextStep?: (result: TResult) => string | null +): void { + const result = response.result + if (!isSettled(result, settled)) { + const reason = result.reason ?? `The managed Orca server action was ${result.outcome}.` + const step = nextStep?.(result) + throw new RuntimeClientError( + `managed_server_${result.outcome}`, + step ? `${reason} ${step}` : reason, + result + ) + } + printResult({ ...response, result }, json, done) +} + +function isSettled( + result: TResult, + settled: readonly TSettled[] +): result is Extract { + return settled.some((outcome) => outcome === result.outcome) +} + +export const MANAGED_SERVER_HANDLERS: Record = { + 'environment status': async (context) => { + const response = await callManagedServer( + context, + 'managedServer.status', + selectorOf(context) + ) + printResult(response, context.json, formatManagedServerStatus) + }, + 'environment update': async (context) => { + const params = { ...selectorOf(context), force: context.flags.get('force') === true } + const response = await callExpectingRestart( + context, + 'managedServer.update', + params + ) + if (!response) { + return + } + report(response, context.json, ['created', 'updated', 'already-current'], (result) => + result.outcome === 'already-current' + ? `Already on ${result.activeVersion}.` + : `Updated ${result.environment.name} to ${result.activeVersion}.` + ) + }, + 'environment rollback': async (context) => { + const response = await callExpectingRestart( + context, + 'managedServer.rollback', + selectorOf(context) + ) + if (!response) { + return + } + report( + response, + context.json, + ['rolled-back'], + (result) => `Rolled ${result.environment.name} back to ${result.activeVersion}.` + ) + }, + 'environment recover': async (context) => { + const params = selectorOf(context) + const acceptChangedState = context.flags.get('accept-changed-state') === true + if (acceptChangedState && context.flags.get('yes') !== true) { + throw new RuntimeClientError( + 'confirmation_required', + `Restoring ${params.selector}'s prelaunch snapshot discards what the rejected build changed. Re-run with --yes to confirm.` + ) + } + const response = await callManagedServer( + context, + 'managedServer.recover', + acceptChangedState ? { ...params, acceptChangedState } : params + ) + report( + response, + context.json, + ['recovered', 'none'], + (result) => + result.outcome === 'recovered' + ? `Recovered ${result.environment.name} (${result.resolution}); active version ${result.activeVersion ?? 'none'}.` + : 'Nothing to recover.', + (result) => + !acceptChangedState && 'code' in result && result.code === ORCAD_RECOVERY_CHANGED_STATE_CODE + ? 'To restore it from the CLI, re-run with --accept-changed-state --yes.' + : null + ) + }, + 'environment stop': async (context) => { + const params = selectorOf(context) + if (context.flags.get('yes') !== true) { + throw new RuntimeClientError( + 'confirmation_required', + `Stopping ${params.selector} ends its terminals and unlinks it from this machine. Re-run with --yes to confirm.` + ) + } + const response = await callManagedServer( + context, + 'managedServer.stop', + params + ) + report( + response, + context.json, + ['unlinked'], + () => `Stopped ${params.selector} and unlinked it from this machine.` + ) + }, + 'environment cancel-stop': async (context) => { + const response = await callManagedServer( + context, + 'managedServer.cancelStop', + selectorOf(context) + ) + report(response, context.json, ['canceled', 'already-stopped', 'none'], (result) => { + switch (result.outcome) { + case 'canceled': + return `Stop withdrawn; the server keeps serving ${result.activeVersion}.` + case 'already-stopped': + return 'orcad had already exited. Run `orca environment stop --yes` to unlink it.' + case 'none': + return 'No stop is pending.' + } + }) + } +} diff --git a/src/cli/handlers/worktree.ts b/src/cli/handlers/worktree.ts index f4980263ed9..32f37e253bb 100644 --- a/src/cli/handlers/worktree.ts +++ b/src/cli/handlers/worktree.ts @@ -26,6 +26,7 @@ import { import { isTuiAgent } from '../../shared/tui-agent-config' import { isWorkspaceKey, worktreeWorkspaceKey } from '../../shared/workspace-scope' import { printLineageSummary } from './worktree-lineage-summary' +import { projectWorktreePsTerminalVerdict } from '../worktree-ps-terminal-verdict' import { assertWorkspaceTargetFlagsCompatible, hasWorkspaceProjectTarget, @@ -143,7 +144,10 @@ export const WORKTREE_HANDLERS: Record = { { limit: getOptionalPositiveIntegerFlag(flags, 'limit') } ) await annotateOmittedHostScope(client, result.result) - printResult(result, json, formatWorktreePs) + const worktrees = result.result.worktrees.map(projectWorktreePsTerminalVerdict) + printResult({ ...result, result: { ...result.result, worktrees } }, json, () => + formatWorktreePs(result.result) + ) }, 'worktree list': async ({ flags, client, json }) => { const result = await client.call>( diff --git a/src/cli/runtime/launch-serve-runtime.test.ts b/src/cli/runtime/launch-serve-runtime.test.ts new file mode 100644 index 00000000000..9d1fc96802a --- /dev/null +++ b/src/cli/runtime/launch-serve-runtime.test.ts @@ -0,0 +1,93 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { spawnMock, resolveLocalServeRuntimeMock, serveWithOrcadMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), + resolveLocalServeRuntimeMock: vi.fn(), + serveWithOrcadMock: vi.fn() +})) + +vi.mock('child_process', () => ({ spawn: spawnMock, spawnSync: vi.fn() })) +vi.mock('./serve-orcad-launch', () => ({ + resolveLocalServeRuntime: resolveLocalServeRuntimeMock, + serveWithOrcad: serveWithOrcadMock +})) + +import { serveOrcaApp } from './launch' + +class FakeChildProcess extends EventEmitter { + stdout = new EventEmitter() + kill = vi.fn() + unref = vi.fn() + pid = 4101 +} + +/** Electron serve that exits cleanly once spawned, whenever selection gets to spawning it. */ +function electronChild(): void { + spawnMock.mockImplementation(() => { + const child = new FakeChildProcess() + setTimeout(() => child.emit('exit', 0, null), 0) + return child + }) +} + +describe('orca serve host selection', () => { + let stderr: string[] + + beforeEach(() => { + spawnMock.mockReset() + resolveLocalServeRuntimeMock.mockReset() + serveWithOrcadMock.mockReset() + process.env.ORCA_APP_EXECUTABLE = '/opt/orca/orca-ide' + stderr = [] + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + stderr.push(String(chunk)) + return true + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + delete process.env.ORCA_APP_EXECUTABLE + delete process.env.ORCA_SERVE_RUNTIME + }) + + it('serves on orcad by default', async () => { + const selection = { kind: 'orcad', runtime: '/node', entry: '/slot/orcad.js', version: '1' } + resolveLocalServeRuntimeMock.mockResolvedValue(selection) + serveWithOrcadMock.mockResolvedValue(0) + + await expect(serveOrcaApp({ json: true })).resolves.toBe(0) + expect(serveWithOrcadMock).toHaveBeenCalledWith( + selection, + { json: true }, + expect.any(String), + expect.not.objectContaining({ ELECTRON_RUN_AS_NODE: expect.anything() }) + ) + expect(spawnMock).not.toHaveBeenCalled() + expect(stderr.join('')).toContain('[serve] running on orcad 1') + }) + + it('falls back to Electron and prints why when orcad cannot serve', async () => { + resolveLocalServeRuntimeMock.mockResolvedValue({ kind: 'electron', reason: 'no template' }) + electronChild() + + await expect(serveOrcaApp({ json: true })).resolves.toBe(0) + expect(spawnMock).toHaveBeenCalledWith( + '/opt/orca/orca-ide', + expect.arrayContaining(['--serve', '--serve-json']), + expect.any(Object) + ) + expect(stderr.join('')).toContain('[serve] using Electron serve: no template') + }) + + it('keeps Electron without asking orcad when ORCA_SERVE_RUNTIME=electron', async () => { + process.env.ORCA_SERVE_RUNTIME = 'electron' + electronChild() + + await expect(serveOrcaApp({ json: true })).resolves.toBe(0) + expect(resolveLocalServeRuntimeMock).not.toHaveBeenCalled() + expect(spawnMock).toHaveBeenCalledOnce() + expect(stderr.join('')).not.toContain('[serve]') + }) +}) diff --git a/src/cli/runtime/launch.test.ts b/src/cli/runtime/launch.test.ts index 7931e489e3d..584ab18e503 100644 --- a/src/cli/runtime/launch.test.ts +++ b/src/cli/runtime/launch.test.ts @@ -90,10 +90,13 @@ describe('serveOrcaApp', () => { spawnMock.mockReset() spawnSyncMock.mockReset() process.env.ORCA_APP_EXECUTABLE = '/Applications/Orca.app/Contents/MacOS/Orca' + // These cover Electron serve itself; the orcad default is in launch-serve-runtime.test.ts. + process.env.ORCA_SERVE_RUNTIME = 'electron' }) afterEach(() => { vi.restoreAllMocks() + delete process.env.ORCA_SERVE_RUNTIME delete process.env.ORCA_APP_EXECUTABLE delete process.env.ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT delete process.env.ORCA_USER_DATA_PATH diff --git a/src/cli/runtime/launch.ts b/src/cli/runtime/launch.ts index ebf10c2aaa9..afce6264154 100644 --- a/src/cli/runtime/launch.ts +++ b/src/cli/runtime/launch.ts @@ -1,16 +1,11 @@ import { spawn as spawnProcess, type SpawnOptions } from 'node:child_process' import { existsSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' -import { StringDecoder } from 'node:string_decoder' import { runProcessSync } from '../../shared/child-process/run-process' import { SERVE_UPDATE_HANDOFF_PATH_ENV, getServeUpdateHandoffPath } from '../../shared/serve-update-handoff' -import { - getEphemeralVmRecipeResultConnection, - parseEphemeralVmRecipeResult -} from '../../shared/ephemeral-vm-recipes' import { getDefaultUserDataPath } from './metadata' import { getMacAppBundlePath } from './mac-app-update-bundle' import { @@ -19,8 +14,14 @@ import { superviseForegroundServe } from './serve-update-supervisor' import { RuntimeClientError } from './types' +import { SERVE_RUNTIME_ELECTRON, SERVE_RUNTIME_ENV } from '../../shared/orcad-local-serve-selection' +import { + resolveLocalServeRuntime, + serveWithOrcad, + type ServeOrcaAppArgs +} from './serve-orcad-launch' +import { waitForRecipeJson } from './serve-recipe-json' -const IGNORED_NON_RECIPE_STDOUT = '[serve] ignored non-recipe stdout' const USER_NAMESPACE_PROBE_TIMEOUT_MS = 2_000 export function launchOrcaApp(): void { @@ -77,18 +78,44 @@ function spawnDetached(command: string, args: string[], options: SpawnOptions): child.unref() } -export function serveOrcaApp( - args: { - json?: boolean - port?: string | null - pairingAddress?: string | null - noPairing?: boolean - mobilePairing?: boolean - recipeJson?: boolean - projectRoot?: string | null - } = {} -): Promise { +export function serveOrcaApp(args: ServeOrcaAppArgs = {}): Promise { const executable = resolveForegroundOrcaExecutable() + if (args.recipeJson && !args.projectRoot) { + throw new RuntimeClientError('invalid_argument', 'Recipe JSON output requires --project-root.') + } + // Why synchronous on the opt-out: it must spawn Electron exactly as before, without asking. + if (process.env[SERVE_RUNTIME_ENV] === SERVE_RUNTIME_ELECTRON) { + return serveWithElectron(executable, args) + } + return serveWithSelectedRuntime(executable, args) +} + +async function serveWithSelectedRuntime( + executable: string, + args: ServeOrcaAppArgs +): Promise { + const selection = await resolveLocalServeRuntime({ + executable, + appRoot: resolveAppRoot(), + userDataPath: getDefaultUserDataPath(), + usesMacUpdateHandoff: args.recipeJson !== true && getMacAppBundlePath(executable) !== null + }) + if (selection.kind === 'orcad') { + process.stderr.write(`[serve] running on orcad ${selection.version}\n`) + return serveWithOrcad( + selection, + args, + getDefaultUserDataPath(), + stripElectronRunAsNode(process.env) + ) + } + if (selection.reason) { + process.stderr.write(`[serve] using Electron serve: ${selection.reason}\n`) + } + return serveWithElectron(executable, args) +} + +function serveWithElectron(executable: string, args: ServeOrcaAppArgs): Promise { const childArgs = [...getExecutableAppArgs(executable)] childArgs.push('--serve') if (args.json) { @@ -106,13 +133,7 @@ export function serveOrcaApp( if (args.mobilePairing) { childArgs.push('--serve-mobile-pairing') } - if (args.recipeJson) { - if (!args.projectRoot) { - throw new RuntimeClientError( - 'invalid_argument', - 'Recipe JSON output requires --project-root.' - ) - } + if (args.recipeJson && args.projectRoot) { childArgs.push('--serve-recipe-json', '--serve-project-root', args.projectRoot) } @@ -164,97 +185,6 @@ export function serveOrcaApp( }) } -function waitForRecipeJson(child: ReturnType): Promise { - return new Promise((resolve, reject) => { - let output = '' - let settled = false - const timeout = setTimeout(() => { - finish(new RuntimeClientError('runtime_serve_failed', 'Timed out waiting for recipe JSON.')) - child.kill('SIGTERM') - }, 60000) - const finish = (error?: Error): void => { - if (settled) { - return - } - settled = true - clearTimeout(timeout) - child.stdout?.off('data', onData) - child.off('error', onError) - child.off('close', onClose) - if (error) { - reject(error) - return - } - child.stdout?.destroy?.() - child.unref() - resolve(0) - } - const writeIgnoredRecipeStdout = (): void => { - // Why: non-readiness child stdout is untrusted and cannot be safely - // redacted, including schema-valid results with arbitrary user data. - process.stderr.write(`${IGNORED_NON_RECIPE_STDOUT}\n`) - } - const processRecipeOutputLine = (line: string): void => { - const normalizedLine = line.endsWith('\r') ? line.slice(0, -1) : line - if (!normalizedLine.trim()) { - return - } - const parsed = parseEphemeralVmRecipeResult(normalizedLine) - if (!parsed.ok) { - writeIgnoredRecipeStdout() - return - } - if (getEphemeralVmRecipeResultConnection(parsed.result).type !== 'orca-server') { - writeIgnoredRecipeStdout() - return - } - process.stdout.write(`${normalizedLine.trim()}\n`) - finish() - } - const stdoutDecoder = new StringDecoder('utf8') - const onData = (chunk: Buffer | string): void => { - output += typeof chunk === 'string' ? chunk : stdoutDecoder.write(chunk) - while (!settled) { - const newlineIndex = output.indexOf('\n') - if (newlineIndex === -1) { - return - } - const line = output.slice(0, newlineIndex) - output = output.slice(newlineIndex + 1) - processRecipeOutputLine(line) - } - } - const onError = (error: Error): void => { - finish(error) - } - const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { - if (settled) { - return - } - output += stdoutDecoder.end() - if (output.trim()) { - processRecipeOutputLine(output) - } - if (settled) { - return - } - finish( - new RuntimeClientError( - 'runtime_serve_failed', - typeof code === 'number' - ? `Orca serve exited before printing valid recipe JSON with code ${code}.` - : `Orca serve exited before printing valid recipe JSON via ${signal}.` - ) - ) - } - child.stdout?.on('data', onData) - child.once('error', onError) - // Why: `exit` can precede the final piped stdout data. `close` waits until - // stdio closes so a last recipe chunk is not mistaken for missing output. - child.once('close', onClose) - }) -} - export function getExecutableAppArgs(executable: string): string[] { const args = process.env.ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT === '1' ? [resolveAppRoot()] : [] if (shouldDisableExtractedAppImageSandbox(executable)) { diff --git a/src/cli/runtime/serve-orcad-launch.test.ts b/src/cli/runtime/serve-orcad-launch.test.ts new file mode 100644 index 00000000000..2f4161c0f09 --- /dev/null +++ b/src/cli/runtime/serve-orcad-launch.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from 'vitest' +import { formatServeRuntimeSelection } from '../../shared/orcad-local-serve-selection' +import type { runProcess } from '../../shared/child-process/run-process' +import { orcadServeArgs, resolveLocalServeRuntime } from './serve-orcad-launch' + +type RunProcess = typeof runProcess + +function answering(stdout: string, code = 0): RunProcess { + return vi.fn(async () => ({ + code, + signal: null, + stdout, + stderr: '', + timedOut: false + })) +} + +const options = { + executable: '/Applications/Orca.app/Contents/MacOS/Orca', + appRoot: '/Applications/Orca.app/Contents/Resources/app.asar', + userDataPath: '/Users/u/Library/Application Support/orca', + usesMacUpdateHandoff: false +} + +describe('orca serve asking the app which host to run', () => { + it("runs the app's own selection entry as plain Node and reads its answer", async () => { + const selection = { + kind: 'orcad' as const, + runtime: '/rt/node', + entry: '/slot/orcad.js', + version: '1' + } + const run = answering(`noise\n${formatServeRuntimeSelection(selection)}\n`) + expect(await resolveLocalServeRuntime(options, run)).toEqual(selection) + expect(run).toHaveBeenCalledWith( + expect.objectContaining({ + program: options.executable, + args: [ + `${options.appRoot}/out/main/orcad/orcad-local-serve-selection-entry.js`, + '--user-data', + options.userDataPath, + '--app-root', + options.appRoot + ], + env: expect.objectContaining({ ELECTRON_RUN_AS_NODE: '1' }), + stdio: ['ignore', 'pipe', 'inherit'] + }) + ) + }) + + it('keeps packaged macOS on Electron without starting the app to ask', async () => { + const run = answering('') + expect(await resolveLocalServeRuntime({ ...options, usesMacUpdateHandoff: true }, run)).toEqual( + { + kind: 'electron', + reason: expect.stringContaining('packaged macOS') + } + ) + expect(run).not.toHaveBeenCalled() + }) + + it('serves on Electron, and says why, when the app gives no answer', async () => { + expect(await resolveLocalServeRuntime(options, answering('', 1))).toEqual({ + kind: 'electron', + reason: expect.stringContaining('did not answer') + }) + const failing = vi.fn(async () => { + throw new Error('spawn ENOENT') + }) + expect(await resolveLocalServeRuntime(options, failing)).toEqual({ + kind: 'electron', + reason: expect.stringContaining('spawn ENOENT') + }) + }) + + it('forwards every desktop serve flag, binding wide as Electron serve does', () => { + expect( + orcadServeArgs({ + json: true, + port: '6768', + pairingAddress: '10.0.0.5', + noPairing: true, + mobilePairing: true, + recipeJson: true, + projectRoot: '/work/app' + }) + ).toEqual([ + '--bind', + '0.0.0.0', + '--json', + '--port', + '6768', + '--pairing-address', + '10.0.0.5', + '--no-pairing', + '--mobile-pairing', + '--recipe-json', + '--project-root', + '/work/app' + ]) + expect(orcadServeArgs({})).toEqual(['--bind', '0.0.0.0']) + }) +}) diff --git a/src/cli/runtime/serve-orcad-launch.ts b/src/cli/runtime/serve-orcad-launch.ts new file mode 100644 index 00000000000..3379518d829 --- /dev/null +++ b/src/cli/runtime/serve-orcad-launch.ts @@ -0,0 +1,135 @@ +/** + * `orca serve` on this machine's orcad slot. Whether to (and the slot itself) is decided + * app-side by `src/main/orcad/orcad-local-serve-selection.ts`; the CLI only asks and runs. + */ +import { dirname, join } from 'node:path' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { + ORCAD_LOCAL_SERVE_SELECTION_ENTRY, + ORCAD_LOCAL_SERVE_SELECTION_FLAGS as FLAGS, + parseServeRuntimeSelection, + type ServeRuntimeSelection +} from '../../shared/orcad-local-serve-selection' +import { waitForRecipeJson } from './serve-recipe-json' +import { superviseForegroundServe } from './serve-update-supervisor' + +type SupervisorArgs = Parameters[0] + +export type ServeOrcaAppArgs = { + json?: boolean + port?: string | null + pairingAddress?: string | null + noPairing?: boolean + mobilePairing?: boolean + recipeJson?: boolean + projectRoot?: string | null +} + +/** A first run may download and verify the pinned Node; bound it well past that. */ +const SELECTION_TIMEOUT_MS = 10 * 60_000 + +/** Asks the app's own entry, run on the app's executable as plain Node, which host to serve on. */ +export async function resolveLocalServeRuntime( + options: { + executable: string + appRoot: string + userDataPath: string + usesMacUpdateHandoff: boolean + }, + run: typeof runProcess = runProcess +): Promise { + // Why: only packaged macOS serve can take a remote app update, through Electron's updater and + // this CLI's supervisor; orcad has no updater, so switching would drop that. + if (options.usesMacUpdateHandoff) { + return { + kind: 'electron', + reason: + 'packaged macOS serve stays on Electron so paired clients can still update it (orcad has no app updater)' + } + } + const entry = join(options.appRoot, 'out', 'main', `${ORCAD_LOCAL_SERVE_SELECTION_ENTRY}.js`) + try { + const result = await run({ + program: options.executable, + args: [entry, FLAGS.userData, options.userDataPath, FLAGS.appRoot, options.appRoot], + env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, + // Why inherit stderr: a first run may download the pinned Node, and that progress is the + // only sign `orca serve` is not hung. + stdio: ['ignore', 'pipe', 'inherit'], + timeoutMs: SELECTION_TIMEOUT_MS + }) + return ( + parseServeRuntimeSelection(result.stdout) ?? { + kind: 'electron', + reason: `the app did not answer which serve host to use (exit ${String(result.code)})` + } + ) + } catch (error) { + return { + kind: 'electron', + reason: `the app could not check orcad: ${error instanceof Error ? error.message : String(error)}` + } + } +} + +/** The shared spawn chokepoint (windowsHide, no shell) in the supervisor's spawn shape. */ +const spawnThroughChokepoint: SupervisorArgs['spawnChild'] = (program, args, options) => + spawnProcess({ + program, + args, + cwd: typeof options.cwd === 'string' ? options.cwd : undefined, + env: options.env, + stdio: options.stdio, + detached: options.detached + }) + +/** Electron serve binds every interface (`exposeNetworkByDefault`); orcad does it on request. */ +export function serveWithOrcad( + selection: Extract, + args: ServeOrcaAppArgs, + userDataPath: string, + /** The caller's environment without `ELECTRON_RUN_AS_NODE`. */ + baseEnv: NodeJS.ProcessEnv, + spawnChild: SupervisorArgs['spawnChild'] = spawnThroughChokepoint +): Promise { + const childArgs = [selection.entry, ...orcadServeArgs(args)] + const spawnOptions: SupervisorArgs['spawnOptions'] = { + detached: args.recipeJson === true, + cwd: dirname(selection.entry), + stdio: args.recipeJson === true ? ['ignore', 'pipe', 'inherit'] : 'inherit', + env: { + ...baseEnv, + // The desktop's profile: its instance lock makes the two refuse each other. + ORCA_USER_DATA: userDataPath, + ORCA_VERSION: selection.version + } + } + const child = spawnChild(selection.runtime, childArgs, spawnOptions) + if (args.recipeJson) { + return waitForRecipeJson(child) + } + return superviseForegroundServe({ + executable: selection.runtime, + childArgs, + spawnOptions, + spawnChild, + child, + handoffPath: null, + expectedHandoff: null + }) +} + +export function orcadServeArgs(args: ServeOrcaAppArgs): string[] { + return [ + '--bind', + '0.0.0.0', + ...(args.json ? ['--json'] : []), + ...(args.port ? ['--port', args.port] : []), + ...(args.pairingAddress ? ['--pairing-address', args.pairingAddress] : []), + ...(args.noPairing ? ['--no-pairing'] : []), + ...(args.mobilePairing ? ['--mobile-pairing'] : []), + ...(args.recipeJson && args.projectRoot + ? ['--recipe-json', '--project-root', args.projectRoot] + : []) + ] +} diff --git a/src/cli/runtime/serve-recipe-json.ts b/src/cli/runtime/serve-recipe-json.ts new file mode 100644 index 00000000000..70899c8f31f --- /dev/null +++ b/src/cli/runtime/serve-recipe-json.ts @@ -0,0 +1,101 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { StringDecoder } from 'node:string_decoder' +import { + getEphemeralVmRecipeResultConnection, + parseEphemeralVmRecipeResult +} from '../../shared/ephemeral-vm-recipes' +import { RuntimeClientError } from './types' + +const IGNORED_NON_RECIPE_STDOUT = '[serve] ignored non-recipe stdout' + +/** Relays the one recipe line a detached serve child prints, then lets the CLI exit. */ +export function waitForRecipeJson(child: ChildProcessHandle): Promise { + return new Promise((resolve, reject) => { + let output = '' + let settled = false + const timeout = setTimeout(() => { + finish(new RuntimeClientError('runtime_serve_failed', 'Timed out waiting for recipe JSON.')) + child.kill('SIGTERM') + }, 60000) + const finish = (error?: Error): void => { + if (settled) { + return + } + settled = true + clearTimeout(timeout) + child.stdout?.off('data', onData) + child.off('error', onError) + child.off('close', onClose) + if (error) { + reject(error) + return + } + child.stdout?.destroy?.() + child.unref() + resolve(0) + } + const writeIgnoredRecipeStdout = (): void => { + // Why: non-readiness child stdout is untrusted and cannot be safely + // redacted, including schema-valid results with arbitrary user data. + process.stderr.write(`${IGNORED_NON_RECIPE_STDOUT}\n`) + } + const processRecipeOutputLine = (line: string): void => { + const normalizedLine = line.endsWith('\r') ? line.slice(0, -1) : line + if (!normalizedLine.trim()) { + return + } + const parsed = parseEphemeralVmRecipeResult(normalizedLine) + if (!parsed.ok) { + writeIgnoredRecipeStdout() + return + } + if (getEphemeralVmRecipeResultConnection(parsed.result).type !== 'orca-server') { + writeIgnoredRecipeStdout() + return + } + process.stdout.write(`${normalizedLine.trim()}\n`) + finish() + } + const stdoutDecoder = new StringDecoder('utf8') + const onData = (chunk: Buffer | string): void => { + output += typeof chunk === 'string' ? chunk : stdoutDecoder.write(chunk) + while (!settled) { + const newlineIndex = output.indexOf('\n') + if (newlineIndex === -1) { + return + } + const line = output.slice(0, newlineIndex) + output = output.slice(newlineIndex + 1) + processRecipeOutputLine(line) + } + } + const onError = (error: Error): void => { + finish(error) + } + const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { + if (settled) { + return + } + output += stdoutDecoder.end() + if (output.trim()) { + processRecipeOutputLine(output) + } + if (settled) { + return + } + finish( + new RuntimeClientError( + 'runtime_serve_failed', + typeof code === 'number' + ? `Orca serve exited before printing valid recipe JSON with code ${code}.` + : `Orca serve exited before printing valid recipe JSON via ${signal}.` + ) + ) + } + child.stdout?.on('data', onData) + child.once('error', onError) + // Why: `exit` can precede the final piped stdout data. `close` waits until + // stdio closes so a last recipe chunk is not mistaken for missing output. + child.once('close', onClose) + }) +} diff --git a/src/cli/runtime/serve-update-supervisor.ts b/src/cli/runtime/serve-update-supervisor.ts index f791ef2ae6e..6c5ee735f9a 100644 --- a/src/cli/runtime/serve-update-supervisor.ts +++ b/src/cli/runtime/serve-update-supervisor.ts @@ -1,4 +1,4 @@ -import type { ChildProcess, SpawnOptions, spawn } from 'node:child_process' +import type { ChildProcess, SpawnOptions } from 'node:child_process' import { readFileSync } from 'node:fs' import { readFile, rename, unlink, writeFile } from 'node:fs/promises' import { @@ -27,7 +27,7 @@ type ServeSupervisorArgs = { executable: string childArgs: string[] spawnOptions: SpawnOptions - spawnChild: typeof spawn + spawnChild: (program: string, args: string[], options: SpawnOptions) => ChildProcess handoffPath: string | null } diff --git a/src/cli/specs/index.ts b/src/cli/specs/index.ts index b817749f425..8d72b3b53b9 100644 --- a/src/cli/specs/index.ts +++ b/src/cli/specs/index.ts @@ -9,6 +9,7 @@ import { PROJECT_COMMAND_SPECS } from './project' import { ORCHESTRATION_COMMAND_SPECS } from './orchestration' import { COMPUTER_COMMAND_SPECS } from './computer' import { ENVIRONMENT_COMMAND_SPECS } from './environment' +import { MANAGED_SERVER_COMMAND_SPECS } from './managed-server' import { AGENT_HOOK_COMMAND_SPECS } from './agent-hooks' import { DIAGNOSTICS_COMMAND_SPECS } from './diagnostics' import { EMULATOR_COMMAND_SPECS } from './emulator' @@ -35,6 +36,7 @@ export const COMMAND_SPECS: CommandSpec[] = [ ...DIAGNOSTICS_COMMAND_SPECS, ...INTROSPECTION_COMMAND_SPECS, ...ENVIRONMENT_COMMAND_SPECS, + ...MANAGED_SERVER_COMMAND_SPECS, ...LINEAR_COMMAND_SPECS, ...VM_COMMAND_SPECS, ...EMULATOR_COMMAND_SPECS, diff --git a/src/cli/specs/managed-server.ts b/src/cli/specs/managed-server.ts new file mode 100644 index 00000000000..89a592402d7 --- /dev/null +++ b/src/cli/specs/managed-server.ts @@ -0,0 +1,75 @@ +import type { CommandSpec } from '../args' +import { GLOBAL_FLAGS } from '../args' + +const SELECTOR_NOTE = + '--environment names a managed Orca server this machine deployed over SSH (see `orca environment list`); it is a selector here, not a routing flag.' +const DESKTOP_NOTE = + 'Runs on this machine’s Orca desktop app, the same action as Settings > Managed servers. A runtime without the SSH registry (headless `orca serve`) refuses it.' + +export const MANAGED_SERVER_COMMAND_SPECS: CommandSpec[] = [ + { + path: ['environment', 'status'], + summary: 'Show a managed Orca server’s active version, terminals and pending work', + usage: 'orca environment status --environment [--json]', + allowedFlags: [...GLOBAL_FLAGS], + notes: [SELECTOR_NOTE, DESKTOP_NOTE], + examples: ['orca environment status --environment build-box'] + }, + { + path: ['environment', 'update'], + summary: 'Update a managed Orca server to this build’s version', + usage: 'orca environment update --environment [--force] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'force'], + notes: [ + 'Without --force, an update that would restart over running terminals is deferred and reported, not applied.', + SELECTOR_NOTE, + DESKTOP_NOTE + ], + examples: ['orca environment update --environment build-box'] + }, + { + path: ['environment', 'rollback'], + summary: 'Roll a managed Orca server back to its previous version', + usage: 'orca environment rollback --environment [--json]', + allowedFlags: [...GLOBAL_FLAGS], + notes: [SELECTOR_NOTE, DESKTOP_NOTE] + }, + { + path: ['environment', 'recover'], + summary: 'Finish or undo a managed Orca server’s interrupted update, rollback or stop', + usage: + 'orca environment recover --environment [--accept-changed-state --yes] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'accept-changed-state', 'yes'], + notes: [ + 'When a rejected build changed profile state, recover refuses rather than restart the previous build over it. --accept-changed-state --yes restores the prelaunch snapshot instead, the same as Restore in Settings > Managed servers; what the rejected build changed is discarded.', + SELECTOR_NOTE, + DESKTOP_NOTE + ], + examples: [ + 'orca environment recover --environment build-box', + 'orca environment recover --environment build-box --accept-changed-state --yes' + ] + }, + { + path: ['environment', 'stop'], + destructive: true, + summary: 'Stop a managed Orca server and unlink it from this machine (decommission)', + usage: 'orca environment stop --environment --yes [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'yes'], + notes: [ + 'Stops orcad on the SSH host and, once the host proves it exited, removes the server from this machine. Its terminals end. Requires --yes.', + 'If the host cannot prove orcad exited, the server stays linked and the refusal says why; nothing is removed on a guess.', + 'Pick another Active Server first if this one is active. `orca environment cancel-stop` withdraws a stop orcad has not acted on yet.', + SELECTOR_NOTE, + DESKTOP_NOTE + ], + examples: ['orca environment stop --environment build-box --yes'] + }, + { + path: ['environment', 'cancel-stop'], + summary: 'Withdraw a managed Orca server stop that orcad has not acted on yet', + usage: 'orca environment cancel-stop --environment [--json]', + allowedFlags: [...GLOBAL_FLAGS], + notes: [SELECTOR_NOTE, DESKTOP_NOTE] + } +] diff --git a/src/cli/workspace-format.ts b/src/cli/workspace-format.ts index 51a0369978b..fc7a9bc0acb 100644 --- a/src/cli/workspace-format.ts +++ b/src/cli/workspace-format.ts @@ -8,6 +8,7 @@ import type { } from '../shared/runtime-types' import type { MemorySnapshot, WorktreeMemory } from '../shared/process-stats-types' import { formatListingHostScope, type WithAnnotatedHostScope } from './omitted-host-scope-selectors' +import { formatWorktreePsTerminalFields } from './worktree-ps-terminal-verdict' export function formatMemorySnapshot(snapshot: MemorySnapshot): string { const topWorktrees = [...snapshot.worktrees].sort((a, b) => b.memory - a.memory).slice(0, 10) @@ -139,7 +140,7 @@ export function formatWorktreePs(result: WithAnnotatedHostScope - `${worktree.repo} ${worktree.branch} host=${worktree.hostId ?? 'unverifiable'} live:${worktree.liveTerminalCount} pty:${worktree.hasAttachedPty ? 'yes' : 'no'} unread:${worktree.unread ? 'yes' : 'no'}\n${worktree.path}${worktree.preview ? `\npreview: ${worktree.preview}` : ''}` + `${worktree.repo} ${worktree.branch} host=${worktree.hostId ?? 'unverifiable'} ${formatWorktreePsTerminalFields(worktree)} unread:${worktree.unread ? 'yes' : 'no'}\n${worktree.path}${worktree.preview ? `\npreview: ${worktree.preview}` : ''}` ) .join('\n\n') const bodyWithScope = `${body}\n\n${scope}` diff --git a/src/cli/worktree-ps-terminal-verdict.test.ts b/src/cli/worktree-ps-terminal-verdict.test.ts new file mode 100644 index 00000000000..af0e5bf19e2 --- /dev/null +++ b/src/cli/worktree-ps-terminal-verdict.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { + formatWorktreePsTerminalFields, + projectWorktreePsTerminalVerdict +} from './worktree-ps-terminal-verdict' + +describe('worktree ps terminal verdict', () => { + it('prints counts for a reachable host', () => { + const row = { liveTerminalCount: 2, hasAttachedPty: true, unverifiableTerminalCount: 0 } + expect(formatWorktreePsTerminalFields(row)).toBe('live:2 pty:yes') + expect(projectWorktreePsTerminalVerdict(row)).toEqual({ ...row, terminalVerdict: 'live' }) + }) + + it('prints unverifiable, never zero or no, for an unreachable host', () => { + const row = { liveTerminalCount: 0, hasAttachedPty: false, unverifiableTerminalCount: 1 } + expect(formatWorktreePsTerminalFields(row)).toBe('live:unverifiable pty:unverifiable') + expect(projectWorktreePsTerminalVerdict(row), 'count fields keep their JSON types').toEqual({ + ...row, + terminalVerdict: 'unverifiable' + }) + }) + + it('keeps verified terminals alongside unverifiable ones', () => { + const row = { liveTerminalCount: 1, hasAttachedPty: true, unverifiableTerminalCount: 2 } + expect(formatWorktreePsTerminalFields(row)).toBe('live:1+2 unverifiable pty:yes') + expect( + formatWorktreePsTerminalFields({ ...row, hasAttachedPty: false }), + 'an unverifiable terminal may hold the pty' + ).toBe('live:1+2 unverifiable pty:unverifiable') + expect(projectWorktreePsTerminalVerdict(row)).toEqual({ ...row, terminalVerdict: 'live' }) + }) + + it('reports none, not exited, for a row with no terminals', () => { + const row = { liveTerminalCount: 0, hasAttachedPty: false, unverifiableTerminalCount: 0 } + expect(formatWorktreePsTerminalFields(row)).toBe('live:0 pty:no') + expect(projectWorktreePsTerminalVerdict(row)).toEqual({ ...row, terminalVerdict: 'none' }) + }) + + it('reports unverifiable for an idle row from a host that predates the count', () => { + const row = { liveTerminalCount: 0, hasAttachedPty: false } + expect(formatWorktreePsTerminalFields(row)).toBe('live:unverifiable pty:unverifiable') + expect(projectWorktreePsTerminalVerdict(row)).toEqual({ + ...row, + terminalVerdict: 'unverifiable' + }) + }) +}) diff --git a/src/cli/worktree-ps-terminal-verdict.ts b/src/cli/worktree-ps-terminal-verdict.ts new file mode 100644 index 00000000000..2d79796cec4 --- /dev/null +++ b/src/cli/worktree-ps-terminal-verdict.ts @@ -0,0 +1,42 @@ +import type { RuntimeWorktreePsSummary } from '../shared/runtime-types' + +type TerminalCounts = Pick< + RuntimeWorktreePsSummary, + 'liveTerminalCount' | 'hasAttachedPty' | 'unverifiableTerminalCount' +> + +/** `none`: the host reports no terminal for the row. Never `exited`, which needs proof of an exit. */ +type TerminalVerdict = 'live' | 'unverifiable' | 'none' + +function terminalVerdict(row: TerminalCounts): TerminalVerdict { + if (row.liveTerminalCount > 0) { + return 'live' + } + // Absent count: a host that predates it, which cannot tell no terminals from lost contact. + if (row.unverifiableTerminalCount === undefined || row.unverifiableTerminalCount > 0) { + return 'unverifiable' + } + return 'none' +} + +/** `live:` and `pty:` words for one row; lost contact never reads as zero or no. */ +export function formatWorktreePsTerminalFields(row: TerminalCounts): string { + if (terminalVerdict(row) === 'unverifiable') { + return 'live:unverifiable pty:unverifiable' + } + const unverifiable = row.unverifiableTerminalCount ?? 0 + if (unverifiable === 0) { + return `live:${row.liveTerminalCount} pty:${row.hasAttachedPty ? 'yes' : 'no'}` + } + return `live:${row.liveTerminalCount}+${unverifiable} unverifiable pty:${row.hasAttachedPty ? 'yes' : 'unverifiable'}` +} + +/** + * JSON counterpart. The count fields keep their number/boolean types for existing scripts, so + * `terminalVerdict` is what says a 0/false came from a host that could not be asked. + */ +export function projectWorktreePsTerminalVerdict( + row: TRow +): TRow & { terminalVerdict: TerminalVerdict } { + return { ...row, terminalVerdict: terminalVerdict(row) } +} diff --git a/src/main/automations/automation-run-agent-evidence.ts b/src/main/automations/automation-run-agent-evidence.ts new file mode 100644 index 00000000000..3874e07e4bc --- /dev/null +++ b/src/main/automations/automation-run-agent-evidence.ts @@ -0,0 +1,78 @@ +/** + * What an idle pane says about a run. A ready shell prompt satisfies `tui-idle` too, so a prompt + * typed at a shell whose agent is not installed ("command not found") would read as finished. + * The agent's own status for the run's pane, reported since the run started, completes it as on + * the desktop. Not every agent reports status (no hooks on the host, no recognised title), so for + * those an idle pane still means done, but only after the agent had time to start and only when + * no shell refused its command. + */ +import type { AutomationRun } from '../../shared/automations-types' + +/** Covers agent spin-up over SSH before an idle pane without agent status is believed. */ +export const AGENT_START_GRACE_MS = 2 * 60 * 1000 +/** Only output after the prompt: a refusal further up predates this run. */ +const MISSING_COMMAND_TAIL_LINES = 6 + +// bash, zsh, dash/sh, fish, PowerShell and cmd.exe refusing a command that does not exist. +const MISSING_COMMAND_PATTERNS = [ + /command not found/i, + /^\S+: \d+: \S+: not found$/i, + /unknown command/i, + /is not recognized as (?:the name of a cmdlet|an internal or external command)/i +] + +export type AutomationRunAgentEvidence = { + /** Agent status rows for a pane, from hooks, OSC and titles alike. */ + getAgentStatusRowsForPane(paneKey: string): readonly { receivedAt: number }[] + /** The names the run's agent command may run under, to tell its refusal from its output. */ + agentCommandsForRun(run: AutomationRun): readonly string[] +} + +export type IdleRunVerdict = + | { kind: 'completed' } + | { kind: 'failed'; error: string } + /** An idle shell that may not have started the agent yet. */ + | { kind: 'wait' } + +/** A shell refusing one of the agent's own commands; an agent's tool output never matches. */ +export function findMissingCommandLine( + tail: readonly string[], + commands: readonly string[] +): string | null { + const names = commands.map((command) => command.split(/\s+/)[0]).filter(Boolean) + const recent = tail + .map((line) => line.trim()) + .filter(Boolean) + .slice(-MISSING_COMMAND_TAIL_LINES) + return ( + recent.find( + (line) => + MISSING_COMMAND_PATTERNS.some((pattern) => pattern.test(line)) && + names.some((name) => line.includes(name)) + ) ?? null + ) +} + +export function judgeIdleRun( + evidence: AutomationRunAgentEvidence, + run: AutomationRun, + tail: readonly string[], + runStartedAt: number, + now: number +): IdleRunVerdict { + const paneKey = run.terminalPaneKey + if ( + paneKey && + evidence.getAgentStatusRowsForPane(paneKey).some((row) => row.receivedAt >= runStartedAt) + ) { + return { kind: 'completed' } + } + const missing = findMissingCommandLine(tail, evidence.agentCommandsForRun(run)) + if (missing) { + return { + kind: 'failed', + error: `Automation agent did not start; this host could not run its command (${missing}).` + } + } + return now - runStartedAt >= AGENT_START_GRACE_MS ? { kind: 'completed' } : { kind: 'wait' } +} diff --git a/src/main/automations/external-automation-owner-guard.ts b/src/main/automations/external-automation-owner-guard.ts index db69ff5a7a3..c8b28f90c85 100644 --- a/src/main/automations/external-automation-owner-guard.ts +++ b/src/main/automations/external-automation-owner-guard.ts @@ -23,7 +23,7 @@ import type { ExternalAutomationTarget } from '../../shared/automations-types' import type { SshTarget } from '../../shared/ssh-types' -import { isRuntimeOwnedSshTarget } from '../ssh/ssh-connection-store' +import { isManagedOrcadSshTarget, isRuntimeOwnedSshTarget } from '../ssh/ssh-connection-store' /** Current SSH registrations, hidden ones included so the guard can reject them itself. */ export type DesktopSshTargetRegistry = { @@ -50,7 +50,7 @@ function resolveSshScope( throw externalAutomationTargetRemovedError() } // Why: checked before the generation compare so a hidden target reveals nothing about its registration. - if (isRuntimeOwnedSshTarget(target)) { + if (isRuntimeOwnedSshTarget(target) || isManagedOrcadSshTarget(target)) { throw new ExternalAutomationScopeError(EXTERNAL_AUTOMATION_SCOPE_CODES.targetHidden) } const current = sanitizeSshTargetGeneration(target.generation) diff --git a/src/main/automations/headless-dispatch-runner.ts b/src/main/automations/headless-dispatch-runner.ts index 5a0ed60582c..7315e33c390 100644 --- a/src/main/automations/headless-dispatch-runner.ts +++ b/src/main/automations/headless-dispatch-runner.ts @@ -67,6 +67,8 @@ export async function runHeadlessAutomationDispatch( runId: run.id, status: 'dispatched', ...launchRunTarget, + // Kept here: a watched run's completion is written without it. + ...(precheckResult ? { precheckResult } : {}), error: null }) // Observe the launched agent even while persistence is stalled or rejects its acknowledgement. diff --git a/src/main/automations/headless-run-terminal-retention.test.ts b/src/main/automations/headless-run-terminal-retention.test.ts new file mode 100644 index 00000000000..9304b453a12 --- /dev/null +++ b/src/main/automations/headless-run-terminal-retention.test.ts @@ -0,0 +1,228 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AutomationRun, AutomationRunStatus } from '../../shared/automations-types' +import { + createHeadlessRunTerminalRetention, + RUN_TERMINAL_GRACE_MS, + RUN_TERMINALS_KEPT_PER_AUTOMATION +} from './headless-run-terminal-retention' + +function makeRun( + id: string, + dispatchedAt: number, + status: AutomationRunStatus = 'completed', + automationId = 'nightly' +): AutomationRun { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: retention reads only these run fields. + return { + id, + automationId, + status, + error: status === 'dispatch_failed' ? 'agent missing' : null, + terminalPaneKey: `tab-${id}:1`, + dispatchedAt, + startedAt: dispatchedAt, + createdAt: dispatchedAt + } as AutomationRun +} + +function harness(runs: AutomationRun[]) { + const closed: string[] = [] + const forgotten: AutomationRun[] = [] + const use = new Map() + // Runs whose shell is proven alone at its prompt; any other is unproven, as a live agent reads. + const idleShell = new Set() + const dead = new Set() + const retention = createHeadlessRunTerminalRetention({ + listRuns: () => runs.filter((run) => !forgotten.some((gone) => gone.id === run.id)), + terminalClientUse: (run) => use.get(run.id) ?? 'unused', + runTerminalAlive: (run) => !dead.has(run.id), + shellAloneAtPrompt: async (run) => idleShell.has(run.id), + closeRunTerminal: async (run) => { + closed.push(run.terminalPaneKey ?? '') + return true + }, + forgetRunTerminal: async (run) => { + forgotten.push(run) + } + }) + return { retention, closed, forgotten, use, idleShell, dead } +} + +beforeEach(() => { + vi.useFakeTimers() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('headless run terminal retention', () => { + it('closes finished run terminals past the grace period, keeping the newest few viewable', async () => { + // Six hourly runs of one automation, all finished. + const runs = [0, 1, 2, 3, 4, 5].map((hour) => makeRun(`r${hour}`, hour * 3_600_000)) + const h = harness(runs) + + await h.retention.sweep() + expect(h.closed).toEqual([]) + + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS) + await h.retention.sweep() + expect(h.closed.toSorted()).toEqual(['tab-r0:1', 'tab-r1:1', 'tab-r2:1']) + expect(6 - h.closed.length).toBe(RUN_TERMINALS_KEPT_PER_AUTOMATION) + }) + + it('never closes a run that has not finished, however old', async () => { + const runs = [ + makeRun('working', 0, 'dispatched'), + makeRun('starting', 1, 'dispatching'), + ...[2, 3, 4, 5].map((n) => makeRun(`done${n}`, n)) + ] + const h = harness(runs) + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS * 10) + await h.retention.sweep() + expect(h.closed).toEqual(['tab-done2:1']) + }) + + it.each([ + ['a client typed into since dispatch', 'used'], + ['a client is attached to or viewing', 'used'], + ['this host cannot tell whether a client used', 'unknown'] + ] as const)('never closes a terminal %s', async (_case, verdict) => { + const runs = [0, 1, 2, 3, 4].map((n) => makeRun(`r${n}`, n)) + const h = harness(runs) + h.use.set('r0', verdict) + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS * 10) + await h.retention.sweep() + expect(h.closed).toEqual(['tab-r1:1']) + }) + + it.each(['dispatch_failed', 'skipped_precheck'] as const)( + 'keeps a %s run whose shell is not proven alone, since its agent may still be alive', + async (status) => { + const runs = [ + ...[0, 1, 2, 3].map((n) => makeRun(`f${n}`, n, status)), + ...[4, 5, 6].map((n) => makeRun(`done${n}`, n)) + ] + const h = harness(runs) + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS * 10) + await h.retention.sweep() + expect(h.closed).toEqual([]) + } + ) + + it('keeps the newest few per automation, not across all of them', async () => { + const runs = [ + ...[0, 1, 2].map((n) => makeRun(`a${n}`, n, 'completed', 'a')), + ...[0, 1, 2].map((n) => makeRun(`b${n}`, n, 'completed', 'b')) + ] + const h = harness(runs) + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS) + await h.retention.sweep() + expect(h.closed).toEqual([]) + }) + + it('sweeps on its own once started, and stops with the service', async () => { + const runs = [0, 1, 2, 3].map((n) => makeRun(`r${n}`, n)) + const h = harness(runs) + h.retention.start() + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS + 2 * 60_000) + expect(h.closed).toEqual(['tab-r0:1']) + + h.retention.stop() + runs.push(makeRun('r4', 4), makeRun('r5', 5)) + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS * 2) + expect(h.closed).toEqual(['tab-r0:1']) + }) + + it('drains every completed, unused run terminal for an update, ignoring keep and grace', async () => { + const runs = [ + ...[0, 1, 2, 3, 4].map((n) => makeRun(`r${n}`, n)), + makeRun('typed', 5), + makeRun('adopted', 6), + makeRun('failed', 7, 'dispatch_failed'), + makeRun('working', 8, 'dispatched') + ] + const h = harness(runs) + h.use.set('typed', 'used') + h.use.set('adopted', 'unknown') + + // Fresh runs, inside the grace period: an update still releases them. + expect(await h.retention.drain()).toBe(5) + expect(h.closed.toSorted()).toEqual([ + 'tab-r0:1', + 'tab-r1:1', + 'tab-r2:1', + 'tab-r3:1', + 'tab-r4:1' + ]) + }) + + it('closes a failed run whose agent command was not found, once its shell is idle', async () => { + const runs = [ + makeRun('missing', 0, 'dispatch_failed'), + ...[1, 2, 3].map((n) => makeRun(`done${n}`, n)) + ] + const h = harness(runs) + h.idleShell.add('missing') + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS) + await h.retention.sweep() + expect(h.closed).toEqual(['tab-missing:1']) + expect(h.forgotten[0]).toMatchObject({ id: 'missing', status: 'dispatch_failed' }) + }) + + it('keeps a timed-out run whose agent still runs in its shell', async () => { + const runs = [ + makeRun('timed-out', 0, 'dispatch_failed'), + ...[1, 2, 3].map((n) => makeRun(`done${n}`, n)) + ] + const h = harness(runs) + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS * 10) + await h.retention.sweep() + expect(h.closed).toEqual([]) + }) + + it('closes a still-dispatched run whose agent exited, leaving its shell idle', async () => { + const runs = [ + makeRun('exited', 0, 'dispatched'), + ...[1, 2, 3].map((n) => makeRun(`done${n}`, n)) + ] + const h = harness(runs) + h.idleShell.add('exited') + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS) + await h.retention.sweep() + expect(h.closed).toEqual(['tab-exited:1']) + }) + + it('keeps the newest three live terminals, not counting ones already gone', async () => { + const runs = [0, 1, 2, 3, 4].map((n) => makeRun(`r${n}`, n)) + const h = harness(runs) + // The newest run's terminal was closed by hand: it must not take a keep slot. + h.dead.add('r4') + await h.retention.sweep() + vi.advanceTimersByTime(RUN_TERMINAL_GRACE_MS) + await h.retention.sweep() + expect(h.closed).toEqual(['tab-r0:1']) + expect(h.forgotten.map((run) => run.id).toSorted()).toEqual(['r0', 'r4']) + }) + + it('drains idle failed and exited runs for an update, never a live agent', async () => { + const runs = [ + makeRun('missing', 0, 'dispatch_failed'), + makeRun('exited', 1, 'dispatched'), + makeRun('timed-out', 2, 'dispatch_failed'), + makeRun('working', 3, 'dispatched') + ] + const h = harness(runs) + h.idleShell.add('missing') + h.idleShell.add('exited') + expect(await h.retention.drain()).toBe(2) + expect(h.closed.toSorted()).toEqual(['tab-exited:1', 'tab-missing:1']) + }) +}) diff --git a/src/main/automations/headless-run-terminal-retention.ts b/src/main/automations/headless-run-terminal-retention.ts new file mode 100644 index 00000000000..b3f02b9df00 --- /dev/null +++ b/src/main/automations/headless-run-terminal-retention.ts @@ -0,0 +1,150 @@ +/** + * Closing run terminals on a headless host. The desktop closes a run's terminal when the run + * completes; on orcad nobody does, so schedules leave a shell and a PTY per run until the host + * runs out. A run terminal stays open for a grace period and the newest few per automation stay + * viewable; older ones are closed. A completed run's terminal is closed as is. A failed or + * never-finishing run's is closed only once the shell is proven alone at its prompt, since a timed + * out agent may still be alive. A terminal a client typed into or is viewing, or one whose use + * this host cannot tell, is never closed: as on the desktop, that terminal is the user's. + */ +import { + isFinalAutomationRunStatus, + type AutomationRun, + type AutomationRunStatus +} from '../../shared/automations-types' + +export const RUN_TERMINAL_GRACE_MS = 10 * 60_000 +export const RUN_TERMINALS_KEPT_PER_AUTOMATION = 3 +const SWEEP_INTERVAL_MS = 60_000 + +export type HeadlessRunTerminalRetentionDeps = { + listRuns: () => readonly AutomationRun[] + /** + * Whether any client drove or is viewing the run's terminal. Like the desktop's take-over rule, + * a used terminal is the user's now; `unknown` keeps it too. + */ + terminalClientUse: (run: AutomationRun) => 'used' | 'unused' | 'unknown' + /** Whether the run's pane still holds the run's own PTY; a dead or replaced one is forgotten. */ + runTerminalAlive: (run: AutomationRun) => boolean + /** Fresh proof that only the shell runs in the run's PTY, at its prompt; false when unproven. */ + shellAloneAtPrompt: (run: AutomationRun) => Promise + /** + * Closes the run's own pane, leaving any pane a user split into that tab. False, closing + * nothing, when the pane is gone or now holds another PTY (a restart put a new one there). + */ + closeRunTerminal: (run: AutomationRun) => Promise + /** Drops the closed terminal from the run, keeping its status, error and output. */ + forgetRunTerminal: (run: AutomationRun) => Promise + now?: () => number +} + +/** Still starting: its agent is being launched, so its terminal is never a candidate. */ +const STARTING: ReadonlySet = new Set(['pending', 'dispatching']) + +export function createHeadlessRunTerminalRetention(deps: HeadlessRunTerminalRetentionDeps): { + sweep: () => Promise + /** Before an update restarts the server: no grace, no newest-N, every other rule still holds. */ + drain: () => Promise + start: () => void + stop: () => void +} { + const now = deps.now ?? Date.now + // When each run terminal was first seen as a candidate; the grace runs from there. + const firstSeenAt = new Map() + let timer: ReturnType | null = null + let sweeping: Promise | null = null + + const forget = async (run: AutomationRun): Promise => { + await deps.forgetRunTerminal(run) + firstSeenAt.delete(run.id) + } + + const mayClose = async (run: AutomationRun, graceMs: number): Promise => { + if (now() - (firstSeenAt.get(run.id) ?? now()) < graceMs) { + return false + } + if (deps.terminalClientUse(run) !== 'unused') { + return false + } + return run.status === 'completed' || (await deps.shellAloneAtPrompt(run)) + } + + const sweepOnce = async (policy: { keep: number; graceMs: number }): Promise => { + let closedCount = 0 + const byAutomation = new Map() + for (const run of deps.listRuns()) { + if (!run.terminalPaneKey || STARTING.has(run.status)) { + continue + } + if (!firstSeenAt.has(run.id)) { + firstSeenAt.set(run.id, now()) + } + byAutomation.set(run.automationId, [...(byAutomation.get(run.automationId) ?? []), run]) + } + for (const runs of byAutomation.values()) { + let kept = 0 + for (const run of runs.toSorted((a, b) => runRecency(b) - runRecency(a))) { + try { + // A terminal already gone holds no keep slot; only live ones stay viewable. + if (!deps.runTerminalAlive(run)) { + if (isFinalAutomationRunStatus(run.status)) { + await forget(run) + } + continue + } + if (kept < policy.keep) { + kept += 1 + continue + } + if (!(await mayClose(run, policy.graceMs))) { + continue + } + if (await deps.closeRunTerminal(run)) { + closedCount += 1 + } + await forget(run) + } catch (error) { + console.error('[automations] could not close a run terminal:', error) + } + } + } + return closedCount + } + + const sweep = (): Promise => { + sweeping ??= sweepOnce({ + keep: RUN_TERMINALS_KEPT_PER_AUTOMATION, + graceMs: RUN_TERMINAL_GRACE_MS + }) + .then(() => {}) + .finally(() => { + sweeping = null + }) + return sweeping + } + + const drain = async (): Promise => { + // Waits out a periodic sweep so the two never close the same terminal twice. + await sweeping?.catch(() => {}) + return sweepOnce({ keep: 0, graceMs: 0 }) + } + + return { + sweep, + drain, + start: () => { + timer ??= setInterval(() => void sweep(), SWEEP_INTERVAL_MS) + timer.unref?.() + }, + stop: () => { + if (timer) { + clearInterval(timer) + timer = null + } + } + } +} + +function runRecency(run: AutomationRun): number { + return run.dispatchedAt ?? run.startedAt ?? run.createdAt +} diff --git a/src/main/automations/precheck-runner.test.ts b/src/main/automations/precheck-runner.test.ts index bdcda1ba221..3c6405761d9 100644 --- a/src/main/automations/precheck-runner.test.ts +++ b/src/main/automations/precheck-runner.test.ts @@ -12,7 +12,7 @@ const sshManagerState = vi.hoisted(() => ({ } })) -vi.mock('../ipc/ssh', () => ({ +vi.mock('../ssh/ssh-target-registry', () => ({ getSshConnectionManager: () => sshManagerState.manager })) diff --git a/src/main/automations/precheck-runner.ts b/src/main/automations/precheck-runner.ts index ab38fd42355..d4ab63b4d75 100644 --- a/src/main/automations/precheck-runner.ts +++ b/src/main/automations/precheck-runner.ts @@ -2,7 +2,7 @@ import { spawn, type ChildProcess } from 'node:child_process' import type { ClientChannel } from 'ssh2' import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/automations-types' import { MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS } from '../../shared/automation-precheck' -import { getSshConnectionManager } from '../ipc/ssh' +import { getSshConnectionManager } from '../ssh/ssh-target-registry' import { shellEscape } from '../ssh/ssh-connection-utils' import { admitSelfInitiatedTreeKill } from '../own-chromium-tree-kill-guard' diff --git a/src/main/automations/run-completion-watcher.ts b/src/main/automations/run-completion-watcher.ts index ab53efaee58..36835d88026 100644 --- a/src/main/automations/run-completion-watcher.ts +++ b/src/main/automations/run-completion-watcher.ts @@ -18,7 +18,7 @@ export type AutomationRunTerminalObserver = { resolveRunTerminal: (run: AutomationRun) => string | null observeCompletion: ( handle: string, - options: { signal: AbortSignal } + options: { signal: AbortSignal; run?: AutomationRun } ) => Promise } @@ -118,7 +118,10 @@ export class AutomationRunCompletionWatcher { ): Promise { let observation: AutomationRunCompletionObservation try { - observation = await this.observer.observeCompletion(handle, { signal: controller.signal }) + observation = await this.observer.observeCompletion(handle, { + signal: controller.signal, + run + }) } catch (error) { if (controller.signal.aborted) { return diff --git a/src/main/automations/runtime-automation-service-retention.test.ts b/src/main/automations/runtime-automation-service-retention.test.ts new file mode 100644 index 00000000000..f3dee8abc4b --- /dev/null +++ b/src/main/automations/runtime-automation-service-retention.test.ts @@ -0,0 +1,125 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AutomationRun } from '../../shared/automations-types' +import { RUN_TERMINAL_GRACE_MS } from './headless-run-terminal-retention' + +vi.mock('./service', () => ({ + AutomationService: class { + start = vi.fn() + stop = vi.fn() + markDispatchResult = vi.fn(async () => ({})) + } +})) + +import { createRuntimeAutomationService } from './runtime-automation-service' + +function finishedRuns(count: number): AutomationRun[] { + return Array.from( + { length: count }, + (_, n) => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: retention reads only these run fields. + ({ + id: `r${n}`, + automationId: 'nightly', + status: 'completed', + error: null, + terminalPaneKey: `tab-${n}:1`, + terminalPtyId: `pty-${n}`, + dispatchedAt: n, + startedAt: n, + createdAt: n + }) as AutomationRun + ) +} + +function build(headless: boolean) { + const runtime = { + setAutomationService: vi.fn(), + notifyAutomationsChanged: vi.fn(), + getTerminalHandleForPaneKey: vi.fn((paneKey: string) => `handle:${paneKey}`), + // Each run's pane still holds its own PTY unless a test restarts it. + getTerminalPtyIdForHandle: vi.fn((handle: string) => `pty-${handle.split('tab-')[1]?.[0]}`), + // pty-1's terminal has a client on it: typed into or being viewed. + readTerminalClientUse: vi.fn((ptyId: string) => (ptyId === 'pty-1' ? 'used' : 'unused')), + closeTerminal: vi.fn(async (_handle: string) => ({})), + confirmTerminalShellAlone: vi.fn(async (_ptyId: string) => false), + closeTerminalTab: vi.fn(async (_handle: string) => ({})) + } + const store = { listAutomationRuns: vi.fn(() => finishedRuns(5)), listAutomations: () => [] } + const service = createRuntimeAutomationService({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mocked service and retention read only listAutomationRuns. + store: store as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: retention reads only the members faked above. + runtime: runtime as never, + headless + }) + return { runtime, service, store } +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('headless automation service run terminal retention', () => { + it('closes finished run terminals on orcad once the service runs, and stops with it', async () => { + vi.useFakeTimers() + const { runtime, service } = build(true) + service.start() + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS + 2 * 60_000) + + // The oldest two finished runs are past the newest three; the one a client used stays open. + // Only the run's own pane closes: a pane a user split into its tab survives. + expect(runtime.closeTerminal.mock.calls.map(([handle]) => handle)).toEqual(['handle:tab-0:1']) + expect(runtime.closeTerminalTab).not.toHaveBeenCalled() + expect(service.markDispatchResult).toHaveBeenCalledWith( + expect.objectContaining({ runId: 'r0', status: 'completed', terminalPaneKey: null }) + ) + + service.stop() + runtime.closeTerminal.mockClear() + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS * 2) + expect(runtime.closeTerminal).not.toHaveBeenCalled() + }) + + it('closes nothing in a pane a restart gave a new PTY, and still forgets the old one', async () => { + vi.useFakeTimers() + const { runtime, service } = build(true) + // The oldest run's pane was restarted (exited pane, account switch): a user's PTY lives there. + runtime.getTerminalPtyIdForHandle.mockImplementation((handle: string) => + handle === 'handle:tab-0:1' ? 'pty-user' : `pty-${handle.split('tab-')[1]?.[0]}` + ) + service.start() + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS + 2 * 60_000) + + expect(runtime.closeTerminal).not.toHaveBeenCalled() + expect(service.markDispatchResult).toHaveBeenCalledWith( + expect.objectContaining({ runId: 'r0', terminalPaneKey: null, terminalPtyId: null }) + ) + service.stop() + }) + + it('fails and closes a dispatched run whose agent exited, proven by its shell alone', async () => { + vi.useFakeTimers() + const { runtime, service, store } = build(true) + store.listAutomationRuns.mockReturnValue([ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: retention reads only these run fields. + { ...finishedRuns(1)[0], status: 'dispatched' } as AutomationRun + ]) + runtime.confirmTerminalShellAlone.mockResolvedValue(true) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mocked service exposes the drain hook it was given. + const drain = (service as unknown as { releaseFinishedRunTerminals: () => Promise }) + .releaseFinishedRunTerminals + await expect(drain()).resolves.toBe(1) + expect(runtime.confirmTerminalShellAlone).toHaveBeenCalledWith('pty-0') + expect(service.markDispatchResult).toHaveBeenCalledWith( + expect.objectContaining({ runId: 'r0', status: 'dispatch_failed', terminalPtyId: null }) + ) + }) + + it('leaves run terminals to the renderer on the desktop', async () => { + vi.useFakeTimers() + const { runtime, service } = build(false) + service.start() + await vi.advanceTimersByTimeAsync(RUN_TERMINAL_GRACE_MS * 2) + expect(runtime.closeTerminal).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/automations/runtime-automation-service.test.ts b/src/main/automations/runtime-automation-service.test.ts new file mode 100644 index 00000000000..8a0d529a355 --- /dev/null +++ b/src/main/automations/runtime-automation-service.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from 'vitest' +import type { HeadlessAutomationDispatcher } from './headless-dispatch' +import type { AutomationRunTerminalObserver } from './run-completion-watcher' + +const captured = vi.hoisted((): { dispatcher: unknown; observer: unknown } => ({ + dispatcher: null, + observer: null +})) + +vi.mock('./service', () => ({ + AutomationService: class { + start(): void {} + stop(): void {} + constructor( + _store: unknown, + opts: { headlessDispatcher?: unknown; terminalObserver?: unknown } + ) { + captured.dispatcher = opts.headlessDispatcher + captured.observer = opts.terminalObserver + } + } +})) + +import { createRuntimeAutomationService } from './runtime-automation-service' + +describe('headless automation dispatch', () => { + it('hands the launched run to its watcher instead of awaiting one tui-idle wait itself', async () => { + const runtime = { + setAutomationService: vi.fn(), + notifyAutomationsChanged: vi.fn(), + launchAgentTerminal: vi.fn(async () => ({ + handle: 'terminal-1', + tabId: 'tab-1', + paneKey: 'tab-1:pane-1', + ptyId: 'pty-1', + worktreeId: 'wt-1' + })), + showManagedWorktree: vi.fn(async () => ({ displayName: 'repo' })), + waitForTerminal: vi.fn(), + // Not resolvable by pane key yet: the launch's own handle must still reach the watcher. + getTerminalHandleForPaneKey: vi.fn(() => null), + getAgentStatusRowsForPane: vi.fn(() => []) + } + createRuntimeAutomationService({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the mocked service never reads the store. + store: {} as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the dispatcher reads only the members faked above. + runtime: runtime as never, + headless: true + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: captured from the mocked constructor above. + const dispatcher = captured.dispatcher as HeadlessAutomationDispatcher + const launch = await dispatcher({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a reuse-workspace automation; only these fields are read. + automation: { workspaceMode: 'existing', workspaceId: 'wt-1', agentId: 'goose' } as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the title is read. + run: { title: 'Nightly' } as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: unused for an existing workspace. + target: {} as never + }) + + expect(launch.completion).toBeUndefined() + expect(launch.terminalPaneKey).toBe('tab-1:pane-1') + expect(runtime.waitForTerminal).not.toHaveBeenCalled() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: captured from the mocked constructor above. + const observer = captured.observer as AutomationRunTerminalObserver + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: resolution reads only the pane key. + expect(observer.resolveRunTerminal({ terminalPaneKey: 'tab-1:pane-1' } as never)).toBe( + 'terminal-1' + ) + }) +}) diff --git a/src/main/automations/runtime-automation-service.ts b/src/main/automations/runtime-automation-service.ts new file mode 100644 index 00000000000..4eeadfb1be8 --- /dev/null +++ b/src/main/automations/runtime-automation-service.ts @@ -0,0 +1,188 @@ +/** + * The AutomationService a runtime host executes schedules with. Shared by Electron startup and + * orcad so both hosts dispatch through the same headless path. + */ +import type { ClaudeUsageStore } from '../claude-usage/store' +import type { CodexUsageStore } from '../codex-usage/store' +import type { Store } from '../persistence' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import { AutomationService } from './service' +import type { AutomationRun } from '../../shared/automations-types' +import { createHeadlessRunTerminalRetention } from './headless-run-terminal-retention' +import { + getTuiAgentDetectCommands, + isTuiAgent, + TUI_AGENT_CONFIG +} from '../../shared/tui-agent-config' +import { buildHeadlessAutomationWorktreeCreateArgs } from './headless-workspace-create' +import { createRuntimeAutomationRunTerminalObserver } from './runtime-terminal-run-observer' + +const MAX_REMEMBERED_LAUNCHES = 256 + +export function createRuntimeAutomationService(input: { + store: Store + runtime: OrcaRuntimeService + claudeUsage?: ClaudeUsageStore + codexUsage?: CodexUsageStore + /** A server process: it executes remote_host_service-owned schedules and dispatches headlessly. */ + headless: boolean +}): AutomationService { + const { store, runtime, claudeUsage, codexUsage } = input + // The handle each headless launch returned, so its watcher never depends on a pane-key lookup. + const launchedHandles = new Map() + const observer = createRuntimeAutomationRunTerminalObserver(runtime, { + getAgentStatusRowsForPane: (paneKey) => runtime.getAgentStatusRowsForPane(paneKey), + agentCommandsForRun: (run) => + automationAgentCommands( + store.listAutomations().find((entry) => entry.id === run.automationId)?.agentId + ) + }) + const service = new AutomationService(store, { + claudeUsage, + codexUsage, + terminalObserver: { + ...observer, + // This host's own launch handle first: it names the run's terminal without a lookup. + resolveRunTerminal: (run) => + (run.terminalPaneKey ? launchedHandles.get(run.terminalPaneKey) : undefined) ?? + observer.resolveRunTerminal(run) + }, + onAutomationsChanged: (payload) => runtime.notifyAutomationsChanged(payload), + allowRemoteHostScheduling: input.headless, + headlessDispatcher: input.headless + ? async ({ automation, run, target }) => { + let terminalHandle: string + let terminalSessionId: string | null = null + let terminalPaneKey: string | null = null + let terminalPtyId: string | null = null + let workspaceId: string + let workspaceDisplayName: string | null = null + if (automation.workspaceMode === 'new_per_run') { + const created = await runtime.createManagedWorktree( + buildHeadlessAutomationWorktreeCreateArgs({ automation, run, repo: target.repo }) + ) + terminalHandle = created.startupTerminal?.handle ?? '' + terminalSessionId = created.startupTerminal?.tabId ?? null + terminalPaneKey = created.startupTerminal?.paneKey ?? null + terminalPtyId = created.startupTerminal?.ptyId ?? null + workspaceId = created.worktree.id + workspaceDisplayName = created.worktree.displayName ?? null + if (!terminalHandle) { + throw new Error( + created.warning || + 'Automation workspace was created, but no agent terminal started.' + ) + } + } else { + if (!automation.workspaceId) { + throw new Error('The target workspace is no longer available.') + } + const terminal = await runtime.launchAgentTerminal(`id:${automation.workspaceId}`, { + agent: automation.agentId, + prompt: automation.prompt, + title: run.title + }) + terminalHandle = terminal.handle + terminalSessionId = terminal.tabId ?? null + terminalPaneKey = terminal.paneKey ?? null + terminalPtyId = terminal.ptyId ?? null + workspaceId = terminal.worktreeId + const worktree = await runtime.showManagedWorktree(`id:${workspaceId}`) + workspaceDisplayName = worktree.displayName ?? null + } + if (terminalPaneKey) { + launchedHandles.set(terminalPaneKey, terminalHandle) + if (launchedHandles.size > MAX_REMEMBERED_LAUNCHES) { + launchedHandles.delete(launchedHandles.keys().next().value ?? '') + } + } + // No completion: the run's watcher observes it, retrying wait timeouts until it settles. + return { + workspaceId, + workspaceDisplayName, + terminalSessionId, + terminalPaneKey, + terminalPtyId + } + } + : undefined + }) + runtime.setAutomationService(service) + if (input.headless) { + bindHeadlessRunTerminalRetention(service, store, runtime) + } + return service +} + +/** A headless host has no renderer to close finished run terminals; its service does it. */ +function bindHeadlessRunTerminalRetention( + service: AutomationService, + store: Store, + runtime: OrcaRuntimeService +): void { + const retention = createHeadlessRunTerminalRetention({ + listRuns: () => store.listAutomationRuns(), + terminalClientUse: (run) => + run.terminalPtyId ? runtime.readTerminalClientUse(run.terminalPtyId) : 'unknown', + runTerminalAlive: (run) => runOwnHandle(runtime, run) !== null, + shellAloneAtPrompt: (run) => + run.terminalPtyId + ? runtime.confirmTerminalShellAlone(run.terminalPtyId) + : Promise.resolve(false), + closeRunTerminal: async (run) => { + // Only the run's own process: a restarted pane holds a new PTY a user may be working in. + const handle = runOwnHandle(runtime, run) + if (!handle) { + return false + } + // The run's pane only: a pane a user split into the same tab is theirs. + await runtime.closeTerminal(handle) + return true + }, + forgetRunTerminal: async (run) => { + // A run still dispatched had its agent exit without a result, proven by its idle shell. + const unfinished = run.status === 'dispatched' + await service.markDispatchResult({ + runId: run.id, + status: unfinished ? 'dispatch_failed' : run.status, + error: unfinished + ? (run.error ?? 'The agent exited without reporting completion.') + : run.error, + terminalSessionId: null, + terminalPaneKey: null, + terminalPtyId: null + }) + } + }) + service.releaseFinishedRunTerminals = () => retention.drain() + const start = service.start.bind(service) + const stop = service.stop.bind(service) + service.start = () => { + start() + retention.start() + } + service.stop = () => { + retention.stop() + stop() + } +} + +function automationAgentCommands(agentId: string | undefined): string[] { + if (!isTuiAgent(agentId)) { + return [] + } + const config = TUI_AGENT_CONFIG[agentId] + return [...getTuiAgentDetectCommands(config), config.launchCmd] +} + +/** The run's pane handle while that pane still holds the run's own PTY; null otherwise. */ +function runOwnHandle(runtime: OrcaRuntimeService, run: AutomationRun): string | null { + const handle = run.terminalPaneKey + ? runtime.getTerminalHandleForPaneKey(run.terminalPaneKey) + : null + return handle && + run.terminalPtyId && + runtime.getTerminalPtyIdForHandle(handle) === run.terminalPtyId + ? handle + : null +} diff --git a/src/main/automations/runtime-terminal-run-observer-evidence.test.ts b/src/main/automations/runtime-terminal-run-observer-evidence.test.ts new file mode 100644 index 00000000000..bafa221c554 --- /dev/null +++ b/src/main/automations/runtime-terminal-run-observer-evidence.test.ts @@ -0,0 +1,173 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AutomationRun } from '../../shared/automations-types' +import { AGENT_START_GRACE_MS } from './automation-run-agent-evidence' +import { + createRuntimeAutomationRunTerminalObserver, + type AutomationRunTerminalHost +} from './runtime-terminal-run-observer' + +const HANDLE = 'terminal-1' +const PANE_KEY = 'tab-1:pane-1' +const RUNTIME_TUI_IDLE_TIMEOUT_MS = 5 * 60 * 1000 + +/** A pane where idle is idle whatever painted it: a ready shell prompt satisfies tui-idle too. */ +function createPane(initial: { idle: boolean; tail: string[]; idleEdgeOnly?: boolean }) { + let idle = initial.idle + // The real runtime may settle tui-idle once per idle edge: an already-idle shell only times out. + let edgeSpent = false + let tail = initial.tail + let rows: { receivedAt: number }[] = [] + const waiters = new Set<() => void>() + const runtime: AutomationRunTerminalHost = { + getTerminalHandleForPaneKey: () => HANDLE, + readTerminal: async () => ({ tail }), + waitForTerminal: (_handle, options) => { + if (options?.signal?.aborted) { + return Promise.reject(new Error('request_aborted')) + } + if (idle && !(initial.idleEdgeOnly && edgeSpent)) { + edgeSpent = true + return Promise.resolve({ satisfied: true }) + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + waiters.delete(wake) + reject(new Error('timeout')) + }, options?.timeoutMs ?? RUNTIME_TUI_IDLE_TIMEOUT_MS) + const wake = (): void => { + clearTimeout(timer) + resolve({ satisfied: true }) + } + waiters.add(wake) + options?.signal?.addEventListener('abort', () => { + clearTimeout(timer) + waiters.delete(wake) + reject(new Error('request_aborted')) + }) + }) + } + } + return { + runtime, + agentRows: () => rows, + report: (receivedAt = Date.now()) => (rows = [{ receivedAt }]), + setTail: (next: string[]) => (tail = next), + becomeIdle: () => { + idle = true + for (const wake of waiters) { + waiters.delete(wake) + wake() + } + } + } +} + +function observe(pane: ReturnType, controller = new AbortController()) { + const observer = createRuntimeAutomationRunTerminalObserver(pane.runtime, { + getAgentStatusRowsForPane: (paneKey) => (paneKey === PANE_KEY ? pane.agentRows() : []), + agentCommandsForRun: () => ['goose'] + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the observer reads only these run fields. + const run = { + terminalPaneKey: PANE_KEY, + startedAt: Date.now(), + dispatchedAt: null + } as AutomationRun + const settled = vi.fn() + const failed = vi.fn() + const promise = observer + .observeCompletion(HANDLE, { signal: controller.signal, run }) + .then(settled, failed) + return { settled, failed, promise, controller } +} + +beforeEach(() => { + vi.useFakeTimers() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('observing a run with agent evidence', () => { + it.each([ + ['bash', 'bash: goose: command not found'], + ['zsh', 'zsh: command not found: goose'], + ['dash', 'sh: 1: goose: not found'], + ['fish', 'fish: Unknown command: goose'], + ['PowerShell', "goose: The term 'goose' is not recognized as the name of a cmdlet"] + ])('fails, never completes, a run whose agent %s cannot find', async (_shell, refusal) => { + const run = observe(createPane({ idle: true, tail: ['$ goose run', refusal, '$'] })) + await run.promise + expect(run.settled).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'dispatch_failed', + error: expect.stringContaining(refusal) + }) + ) + }) + + it('completes once the agent itself reported for the run pane', async () => { + const pane = createPane({ idle: false, tail: ['working'] }) + const run = observe(pane) + await vi.advanceTimersByTimeAsync(1_000) + pane.report() + pane.becomeIdle() + await run.promise + expect(run.settled).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })) + }) + + it('keeps idle-means-done for an agent that never reports, after its start window', async () => { + const pane = createPane({ idle: true, tail: ['summary written', '$'] }) + pane.report(1) + const run = observe(pane) + await vi.advanceTimersByTimeAsync(10_000) + expect(run.settled).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(AGENT_START_GRACE_MS) + await run.promise + expect(run.settled).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })) + }) + + it("never reads an agent's own tool output as its command missing", async () => { + const tail = ['running tests', 'bash: pytest: command not found', 'fell back to unittest', '$'] + const run = observe(createPane({ idle: true, tail })) + await vi.advanceTimersByTimeAsync(AGENT_START_GRACE_MS + 1_000) + await run.promise + expect(run.settled).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })) + }) + + it('completes an agent that exited before the window once the window passes, not at a wait timeout', async () => { + // The stub ran, exited 0 and left an idle shell; no agent status, no new idle edge. + const pane = createPane({ idle: true, tail: ['stub done', '$'], idleEdgeOnly: true }) + const run = observe(pane) + await vi.advanceTimersByTimeAsync(AGENT_START_GRACE_MS - 1_000) + expect(run.settled).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(2_000) + expect(run.settled).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })) + expect(run.failed).not.toHaveBeenCalled() + }) + + it('keeps watching an agent past a tui-idle wait timeout and completes it later', async () => { + const pane = createPane({ idle: false, tail: ['working'] }) + const run = observe(pane) + await vi.advanceTimersByTimeAsync(RUNTIME_TUI_IDLE_TIMEOUT_MS * 2 + 1_000) + expect(run.settled).not.toHaveBeenCalled() + expect(run.failed).not.toHaveBeenCalled() + + pane.report() + pane.becomeIdle() + await run.promise + expect(run.settled).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })) + }) + + it('stops on cancellation without recording any result', async () => { + const pane = createPane({ idle: true, tail: ['$'] }) + const run = observe(pane) + await vi.advanceTimersByTimeAsync(5_000) + run.controller.abort() + await vi.advanceTimersByTimeAsync(AGENT_START_GRACE_MS) + await run.promise + expect(run.settled).not.toHaveBeenCalled() + expect(run.failed).toHaveBeenCalledWith(expect.objectContaining({ message: 'request_aborted' })) + }) +}) diff --git a/src/main/automations/runtime-terminal-run-observer.ts b/src/main/automations/runtime-terminal-run-observer.ts index 0dc6750e8de..fbc2f75022a 100644 --- a/src/main/automations/runtime-terminal-run-observer.ts +++ b/src/main/automations/runtime-terminal-run-observer.ts @@ -3,10 +3,13 @@ import type { AutomationRunCompletionObservation, AutomationRunTerminalObserver } from './run-completion-watcher' -import type { AutomationRunOutputSnapshot } from '../../shared/automations-types' +import type { AutomationRun, AutomationRunOutputSnapshot } from '../../shared/automations-types' +import { judgeIdleRun, type AutomationRunAgentEvidence } from './automation-run-agent-evidence' const TERMINAL_SNAPSHOT_LIMIT = 2_000 +/** Cadence for re-checking an idle pane whose agent has not reported yet. */ +const AGENT_EVIDENCE_POLL_INTERVAL_MS = 1_000 /** Cadence for re-probing a pane that already satisfied tui-idle at dispatch. */ const AGENT_START_POLL_INTERVAL_MS = 250 /** Kept under the runtime's 2s tui-idle fallback poll so a probe waiter is torn @@ -34,6 +37,63 @@ export type AutomationRunTerminalHost = { readTerminal(handle: string, opts?: { limit?: number }): Promise<{ tail: string[] }> } +async function readTail(runtime: AutomationRunTerminalHost, handle: string): Promise { + try { + return (await runtime.readTerminal(handle, { limit: TERMINAL_SNAPSHOT_LIMIT })).tail + } catch { + return [] + } +} + +function snapshotOf(tail: readonly string[]): AutomationRunOutputSnapshot | null { + const snapshotBuffer = createHeadlessAutomationOutputSnapshotBuffer() + snapshotBuffer.append(tail.join('\n')) + return snapshotBuffer.snapshot() +} + +/** A satisfied wait judged by the run's agent evidence; null while the agent may still start. */ +async function judgeIdleTail( + tail: readonly string[], + evidence: AutomationRunAgentEvidence, + run: AutomationRun, + runStartedAt: number +): Promise { + const verdict = judgeIdleRun(evidence, run, tail, runStartedAt, Date.now()) + if (verdict.kind === 'wait') { + return null + } + return verdict.kind === 'completed' + ? { status: 'completed', outputSnapshot: snapshotOf(tail), error: null } + : { status: 'dispatch_failed', outputSnapshot: snapshotOf(tail), error: verdict.error } +} + +/** + * After a satisfied wait inside the agent-start window. An idle shell never produces a new idle + * edge, so a fresh wait would only time out; instead the pane stays idle while its output is + * unchanged, and that state is re-judged until the window passes or the agent reports. + * Null once the pane changes: something is running, so the caller waits again. + */ +async function settleIdlePane( + runtime: AutomationRunTerminalHost, + handle: string, + judged: { evidence: AutomationRunAgentEvidence; run: AutomationRun }, + runStartedAt: number, + signal: AbortSignal +): Promise { + const idleTail = (await readTail(runtime, handle)).join('\n') + for (;;) { + const tail = await readTail(runtime, handle) + if (tail.join('\n') !== idleTail) { + return null + } + const observation = await judgeIdleTail(tail, judged.evidence, judged.run, runStartedAt) + if (observation) { + return observation + } + await sleep(AGENT_EVIDENCE_POLL_INTERVAL_MS, signal) + } +} + function isTerminalWaitTimeout(error: unknown): boolean { return error instanceof Error && error.message === 'timeout' } @@ -146,19 +206,25 @@ async function buildUnobservedObservation( } export function createRuntimeAutomationRunTerminalObserver( - runtime: AutomationRunTerminalHost + runtime: AutomationRunTerminalHost, + /** Judges idle panes by the agent's own status; without it, an idle pane after the busy edge completes. */ + evidence?: AutomationRunAgentEvidence ): AutomationRunTerminalObserver { return { resolveRunTerminal: (run) => run.terminalPaneKey ? runtime.getTerminalHandleForPaneKey(run.terminalPaneKey) : null, - observeCompletion: async (handle, { signal }) => { + observeCompletion: async (handle, { signal, run }) => { const startedAt = Date.now() + const judged = evidence && run?.terminalPaneKey ? { evidence, run } : null + // The run's own start bounds which agent status counts and when idleness is believed. + const runStartedAt = run?.startedAt ?? run?.dispatchedAt ?? startedAt // Why: tui-idle is level-triggered, so a reused pane still idle from the // PREVIOUS run satisfies it before this run's agent has typed a character. // Evidence that predates dispatch proves nothing about this run, so require // the pane to leave that state first — the busy edge the renderer's own // dispatch observer requires on reuse (requireWorkingAfterStart). - if (await isTuiIdleSatisfiedNow(runtime, handle, signal)) { + // Agent evidence already discounts a previous run's idleness, so it skips the busy edge. + if (!judged && (await isTuiIdleSatisfiedNow(runtime, handle, signal))) { const started = await waitForAgentStart( runtime, handle, @@ -177,7 +243,13 @@ export function createRuntimeAutomationRunTerminalObserver( for (;;) { try { const wait = await runtime.waitForTerminal(handle, { condition: 'tui-idle', signal }) - return await buildObservation(runtime, handle, wait) + if (!judged || !wait.satisfied) { + return await buildObservation(runtime, handle, wait) + } + const observation = await settleIdlePane(runtime, handle, judged, runStartedAt, signal) + if (observation) { + return observation + } } catch (error) { // Why: tui-idle waits expire on their own schedule; an agent still // working past that window is live, so re-arm rather than fail it. diff --git a/src/main/automations/service.ts b/src/main/automations/service.ts index 1ff8eafbd6b..107f01ae234 100644 --- a/src/main/automations/service.ts +++ b/src/main/automations/service.ts @@ -52,6 +52,8 @@ export class AutomationService { private readonly codexUsage: CodexUsageStore | null private readonly allowRemoteHostScheduling: boolean private readonly headlessDispatcher: HeadlessAutomationDispatcher | null + /** Set on a headless host: closes completed, unused run terminals now; resolves how many. */ + releaseFinishedRunTerminals: (() => Promise) | null = null private readonly publish: PublishAutomationsChanged | null private readonly runs: AutomationRunWriter private readonly completionWatcher: AutomationRunCompletionWatcher | null diff --git a/src/main/codex/codex-structured-turn-steer.test.ts b/src/main/codex/codex-structured-turn-steer.test.ts index 3e82c80b81c..dfde08edafd 100644 --- a/src/main/codex/codex-structured-turn-steer.test.ts +++ b/src/main/codex/codex-structured-turn-steer.test.ts @@ -214,9 +214,9 @@ describe('a Codex send made after Codex answered an earlier one, before it opene const sending = rig.send('client-2') expect(await settledWithin(sending)).toBe('held') const methods = () => - rig.codex.connections[0]!.calls - .map(({ method }) => method) - .filter((method) => method !== 'model/list' && method !== 'config/read') + rig.codex.connections[0]!.calls.map(({ method }) => method).filter( + (method) => method !== 'model/list' && method !== 'config/read' + ) expect(methods()).toEqual(['thread/start', 'turn/start']) return { ...rig, sending, methods } } diff --git a/src/main/daemon/daemon-health-identity.ts b/src/main/daemon/daemon-health-identity.ts new file mode 100644 index 00000000000..d9add78663f --- /dev/null +++ b/src/main/daemon/daemon-health-identity.ts @@ -0,0 +1,5 @@ +/** What a `ptySpawnHealth` reply proves on this platform. */ +export function ptySpawnHealthPlatformCoverage(): 'pty-spawn' | 'handshake' { + // Why handshake on Windows: preflightPtySpawnHealth skips the spawn probe there. + return process.platform === 'win32' ? 'handshake' : 'pty-spawn' +} diff --git a/src/main/daemon/daemon-health.test.ts b/src/main/daemon/daemon-health.test.ts index c499fadd551..d0d2d2f0334 100644 --- a/src/main/daemon/daemon-health.test.ts +++ b/src/main/daemon/daemon-health.test.ts @@ -11,6 +11,7 @@ import { getDaemonPidPath, serializeDaemonPidFile } from './daemon-spawner' import type { SocketProbeOutcome } from './daemon-endpoint-probe' import { checkDaemonHealth, + checkDaemonHealthWithCoverage, E2E_FORCE_DAEMON_HEALTH_UNREACHABLE_ENV, healthCheckDaemon } from './daemon-health' @@ -105,12 +106,61 @@ describe('daemon health', () => { try { await expect(checkDaemonHealth(socketPath, tokenPath)).resolves.toBe('healthy') await expect(healthCheckDaemon(socketPath, tokenPath)).resolves.toBe(true) - expect(ptySpawnHealthCheck).toHaveBeenCalledTimes(2) + await expect(checkDaemonHealthWithCoverage(socketPath, tokenPath)).resolves.toEqual({ + verdict: 'healthy', + coverage: process.platform === 'win32' ? 'handshake' : 'pty-spawn' + }) + expect(ptySpawnHealthCheck).toHaveBeenCalledTimes(3) } finally { await server.shutdown() } }) + it('treats missing coverage from a legacy Windows daemon as handshake-only', async () => { + writeFileSync(tokenPath, 'legacy-token') + const server = createServer((socket) => { + let pending = '' + socket.on('data', (chunk) => { + pending += chunk.toString() + for (;;) { + const newline = pending.indexOf('\n') + if (newline === -1) { + return + } + const message: unknown = JSON.parse(pending.slice(0, newline)) + const type = + typeof message === 'object' && message !== null && 'type' in message + ? message.type + : undefined + pending = pending.slice(newline + 1) + if (type === 'hello') { + socket.write(`${JSON.stringify({ type: 'hello', ok: true })}\n`) + } else if (type === 'ptySpawnHealth') { + socket.write( + `${JSON.stringify({ id: 'health-1', ok: true, payload: { healthy: true } })}\n` + ) + } + } + }) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socketPath, resolve) + }) + + const platform = Object.getOwnPropertyDescriptor(process, 'platform')! + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + try { + await expect(checkDaemonHealthWithCoverage(socketPath, tokenPath)).resolves.toEqual({ + verdict: 'healthy', + coverage: 'handshake' + }) + } finally { + Object.defineProperty(process, 'platform', platform) + await closeServer(server) + } + }) + it('fails when a protocol-healthy daemon cannot spawn PTYs', async () => { const server = new DaemonServer({ socketPath, diff --git a/src/main/daemon/daemon-health.ts b/src/main/daemon/daemon-health.ts index d3a2c8a91a9..04cf322b893 100644 --- a/src/main/daemon/daemon-health.ts +++ b/src/main/daemon/daemon-health.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync } from 'node:fs' import { connect, type Socket } from 'node:net' import { encodeNdjson } from './ndjson' +import { ptySpawnHealthPlatformCoverage } from './daemon-health-identity' import { PROTOCOL_VERSION, type HelloMessage, @@ -21,15 +22,39 @@ export const E2E_FORCE_DAEMON_HEALTH_UNREACHABLE_ENV = 'ORCA_E2E_FORCE_DAEMON_HE // also covers a live-but-wedged daemon that simply missed the RPC budget. export type DaemonHealth = 'healthy' | 'unreachable' | 'rejected' | 'pty-spawn-unhealthy' -export function checkDaemonHealth(socketPath: string, tokenPath: string): Promise { +export type DaemonHealthCheck = { + verdict: DaemonHealth + coverage: 'pty-spawn' | 'handshake' +} + +function readPtySpawnHealthCoverage( + payload: unknown, + fallback: DaemonHealthCheck['coverage'] +): DaemonHealthCheck['coverage'] { + if (typeof payload !== 'object' || payload === null) { + return fallback + } + const coverage = 'coverage' in payload ? payload.coverage : undefined + return coverage === 'pty-spawn' || coverage === 'handshake' ? coverage : fallback +} + +export function checkDaemonHealthWithCoverage( + socketPath: string, + tokenPath: string +): Promise { return new Promise((resolve) => { + // Older Windows daemons answered this RPC without spawning; an absent optional coverage + // field must preserve that weaker meaning during adoption. + const fallbackCoverage = ptySpawnHealthPlatformCoverage() + const resolveVerdict = (verdict: DaemonHealth): void => + resolve({ verdict, coverage: fallbackCoverage }) if (process.env[E2E_FORCE_DAEMON_HEALTH_UNREACHABLE_ENV] === '1') { - resolve('unreachable') + resolveVerdict('unreachable') return } if (process.platform !== 'win32' && !existsSync(socketPath)) { - resolve('unreachable') + resolveVerdict('unreachable') return } @@ -37,13 +62,13 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis try { token = readFileSync(tokenPath, 'utf8').trim() } catch { - resolve('unreachable') + resolveVerdict('unreachable') return } let settled = false let sock: Socket | null = null - const settle = (result: DaemonHealth): void => { + const settle = (result: DaemonHealthCheck): void => { if (settled) { return } @@ -58,7 +83,7 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis sock?.off('connect', onConnect) sock?.off('data', onData) } - const onError = (): void => settle('unreachable') + const onError = (): void => settle({ verdict: 'unreachable', coverage: fallbackCoverage }) const onConnect = (): void => { const hello: HelloMessage = { type: 'hello', @@ -89,13 +114,13 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis try { message = JSON.parse(line) as Record } catch { - settle('rejected') + settle({ verdict: 'rejected', coverage: fallbackCoverage }) return } if (message.type === 'hello') { if (!(message as HelloResponse).ok) { - settle('rejected') + settle({ verdict: 'rejected', coverage: fallbackCoverage }) return } // Why: a protocol-live daemon with a stale cwd or node-pty helper @@ -106,12 +131,18 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis } if (message.id === 'health-1') { - settle(message.ok === true ? 'healthy' : 'pty-spawn-unhealthy') + settle({ + verdict: message.ok === true ? 'healthy' : 'pty-spawn-unhealthy', + coverage: readPtySpawnHealthCoverage(message.payload, fallbackCoverage) + }) return } } } - const timer = setTimeout(() => settle('unreachable'), HEALTH_CHECK_TIMEOUT_MS) + const timer = setTimeout( + () => settle({ verdict: 'unreachable', coverage: fallbackCoverage }), + HEALTH_CHECK_TIMEOUT_MS + ) sock = connect({ path: socketPath }) sock.on('error', onError) @@ -122,6 +153,13 @@ export function checkDaemonHealth(socketPath: string, tokenPath: string): Promis }) } +export async function checkDaemonHealth( + socketPath: string, + tokenPath: string +): Promise { + return (await checkDaemonHealthWithCoverage(socketPath, tokenPath)).verdict +} + export async function healthCheckDaemon(socketPath: string, tokenPath: string): Promise { return (await checkDaemonHealth(socketPath, tokenPath)) === 'healthy' } diff --git a/src/main/daemon/daemon-idle-shutdown.test.ts b/src/main/daemon/daemon-idle-shutdown.test.ts index 1500718e98f..43a82df72bb 100644 --- a/src/main/daemon/daemon-idle-shutdown.test.ts +++ b/src/main/daemon/daemon-idle-shutdown.test.ts @@ -442,6 +442,35 @@ describe('current daemon lifecycle retirement', () => { adopted.dispose() }) + it('atomically retires an idle daemon and permanently fences adapter spawns', async () => { + await startServer() + const adapter = new DaemonPtyAdapter({ socketPath, tokenPath }) + + await expect(adapter.requestIdleRetirement()).resolves.toEqual({ state: 'retiring' }) + await expect( + adapter.spawn({ sessionId: 'late-after-decommission', cols: 80, rows: 24 }) + ).rejects.toThrow('Terminal daemon is decommissioning') + await waitFor(() => onIdleShutdown.mock.calls.length === 1) + adapter.dispose() + }) + + it('reopens adapter admission when the daemon refuses retirement for a live session', async () => { + await startServer() + const adapter = new DaemonPtyAdapter({ socketPath, tokenPath }) + await adapter.spawn({ sessionId: 'already-live', cols: 80, rows: 24 }) + + await expect(adapter.requestIdleRetirement()).resolves.toEqual({ + state: 'busy', + liveSessions: 1, + admissionReopened: true + }) + await expect( + adapter.spawn({ sessionId: 'allowed-after-refusal', cols: 80, rows: 24 }) + ).resolves.toMatchObject({ id: 'allowed-after-refusal' }) + expect(onIdleShutdown).not.toHaveBeenCalled() + adapter.dispose() + }) + it('does not let repeated authenticated control probes extend the startup deadline', async () => { await startServer() const healthControl = connect(socketPath) diff --git a/src/main/daemon/daemon-init.ts b/src/main/daemon/daemon-init.ts index 918547d8b79..6f0bd4eefc5 100644 --- a/src/main/daemon/daemon-init.ts +++ b/src/main/daemon/daemon-init.ts @@ -8,6 +8,9 @@ export { getDaemonEndpointFacts, getDaemonProvider, listLiveDaemonPtyIds, + listLiveDaemonSessions, + requestIdleDaemonRetirement, + releaseDaemonRetirementFence, readDaemonPidRecord, replaceDaemonProvider, shutdownDaemon, diff --git a/src/main/daemon/daemon-launched-child-identity.test.ts b/src/main/daemon/daemon-launched-child-identity.test.ts index 9e6a623e5ce..c34a53c2e14 100644 --- a/src/main/daemon/daemon-launched-child-identity.test.ts +++ b/src/main/daemon/daemon-launched-child-identity.test.ts @@ -103,6 +103,22 @@ describe('launchDaemonChild identity', () => { }) }) +describe('launchDaemonChild startup budget', () => { + it('waits for readiness past the default 10 s when the caller allows it', async () => { + vi.useFakeTimers() + const child = fakeDaemonChild(4242) + spawnDaemonChildProcessMock.mockReturnValue(child) + try { + const launch = launchDaemonChild({ ...LAUNCH_OPTIONS, startupTimeoutMs: 30_000 }) + await vi.advanceTimersByTimeAsync(20_000) + child.emit('message', { type: 'ready', pid: 4242, startedAtMs: 1_000_000 }) + await expect(launch).resolves.toMatchObject({ identity: { pid: 4242 } }) + } finally { + vi.useRealTimers() + } + }) +}) + describe('launchDaemonChild durable-scope fallback', () => { it('retries once without cgroup isolation when the scoped attempt fails', async () => { isDurableDaemonScopeSupportedMock.mockReturnValue(true) diff --git a/src/main/daemon/daemon-launched-child-spawn.ts b/src/main/daemon/daemon-launched-child-spawn.ts index 887241e2daa..1d069b70aa5 100644 --- a/src/main/daemon/daemon-launched-child-spawn.ts +++ b/src/main/daemon/daemon-launched-child-spawn.ts @@ -14,6 +14,8 @@ export type DaemonChildSpawnOptions = { pidPath: string launchNonce: string macosLoginSessionWatch: boolean + /** Defaults to 10 s; a cold first exec on Windows can need longer. */ + startupTimeoutMs?: number } function buildDaemonScriptArgs(options: DaemonChildSpawnOptions): string[] { diff --git a/src/main/daemon/daemon-launched-child.ts b/src/main/daemon/daemon-launched-child.ts index f5082cdc230..9ec02b88c02 100644 --- a/src/main/daemon/daemon-launched-child.ts +++ b/src/main/daemon/daemon-launched-child.ts @@ -12,6 +12,7 @@ import { unlinkOwnedDaemonPidFile } from './daemon-spawner' const DAEMON_CHILD_TERMINATION_GRACE_MS = 5_000 const DAEMON_CHILD_FORCE_EXIT_WAIT_MS = 1_000 const STARTUP_STDERR_MAX_BYTES = 8192 +const DEFAULT_DAEMON_STARTUP_TIMEOUT_MS = 10_000 export class DaemonEndpointUnavailableError extends Error { constructor( @@ -187,7 +188,7 @@ async function launchDaemonChildAttempt( timer = setTimeout(() => { void fail(new Error('Daemon startup timed out')) - }, 10000) + }, options.startupTimeoutMs ?? DEFAULT_DAEMON_STARTUP_TIMEOUT_MS) child.on('message', onReadyMessage) child.on('error', onStartupError) diff --git a/src/main/daemon/daemon-out-of-process-launcher.ts b/src/main/daemon/daemon-out-of-process-launcher.ts index 80a77fe5f88..1aa811c1d5d 100644 --- a/src/main/daemon/daemon-out-of-process-launcher.ts +++ b/src/main/daemon/daemon-out-of-process-launcher.ts @@ -52,9 +52,11 @@ function createPreservedDaemonHandle( return handle } +export type DaemonLaunchPolicy = { macosLoginSessionWatch?: boolean; startupTimeoutMs?: number } + export function createOutOfProcessLauncher( runtimeDir: string, - macosLoginSessionWatch = false + { macosLoginSessionWatch = false, startupTimeoutMs }: DaemonLaunchPolicy = {} ): DaemonLauncher { return async (socketPath, tokenPath, suppliedPidPath, suppliedLaunchNonce) => { const entryPath = getDaemonEntryPath() @@ -132,7 +134,8 @@ export function createOutOfProcessLauncher( tokenPath, pidPath, launchNonce, - macosLoginSessionWatch + macosLoginSessionWatch, + startupTimeoutMs }) } catch (error) { if (!(error instanceof DaemonEndpointUnavailableError) || error.reason !== 'occupied') { diff --git a/src/main/daemon/daemon-process-inspection.test.ts b/src/main/daemon/daemon-process-inspection.test.ts index 41ea4e660b9..3d0c8309fd7 100644 --- a/src/main/daemon/daemon-process-inspection.test.ts +++ b/src/main/daemon/daemon-process-inspection.test.ts @@ -33,41 +33,40 @@ describe('daemon process inspection', () => { expect(runCommand).not.toHaveBeenCalled() }) - it('asks PowerShell to report a failed CIM query instead of an absent process', async () => { - const runCommand = vi.fn( - async (_file: string, _args: string[], _timeoutMs: number) => - '{"status":"present","cmd":"daemon","start":1}' - ) + it('reads the Windows process from the process table, not a PowerShell spawn', async () => { + const runCommand = vi.fn() + const readProcessTable = vi.fn(async () => [ + { pid: 42, ppid: 1, name: 'node.exe', command: 'daemon', creationTimeMs: 7 } + ]) - await queryWindowsProcess(42, { runCommand }) - - const script = runCommand.mock.calls[0]?.[1].at(-1) ?? '' - expect(script).toContain("$ErrorActionPreference = 'Stop'") - expect(script).toMatch(/catch \{[^}]*query_failed/) + await expect(queryWindowsProcess(42, { runCommand, readProcessTable })).resolves.toEqual({ + status: 'present', + commandLine: 'daemon', + startedAtMs: 7 + }) + expect(runCommand).not.toHaveBeenCalled() }) - it('keeps a failed CIM query indeterminate instead of proving the process gone', async () => { - const runCommand = vi.fn(async () => '{"status":"query_failed"}') + it('keeps an unreadable process table indeterminate instead of proving the process gone', async () => { + const readProcessTable = vi.fn(async () => { + throw new Error('windows process table is unreadable') + }) - await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({ + await expect(queryWindowsProcess(42, { readProcessTable })).resolves.toEqual({ status: 'unavailable' }) }) - it('never reads a probe result without a success marker as proof of absence', async () => { - const runCommand = vi.fn(async () => '{"exists":false}') + it('reports absence only from a table that was read and lacks the PID', async () => { + const readProcessTable = vi.fn(async () => [ + { pid: 7, ppid: 1, name: 'other.exe', command: '' } + ]) - await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({ - status: 'unavailable' + await expect(queryWindowsProcess(42, { readProcessTable })).resolves.toEqual({ + status: 'missing' }) }) - it('reports absence only from a CIM query that ran and found nothing', async () => { - const runCommand = vi.fn(async () => '{"status":"missing"}') - - await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({ status: 'missing' }) - }) - it('reads the macOS start time through an async spawn', async () => { const runCommand = vi.fn(async () => 'Sat Jan 1 00:00:00 2028\n') @@ -158,14 +157,14 @@ describe('daemon process inspection', () => { }) it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.NaN])( - 'rejects unsafe Windows pid %s before command interpolation', + 'rejects unsafe Windows pid %s before reading the process table', async (pid) => { - const runCommand = vi.fn() + const readProcessTable = vi.fn(async () => []) - await expect(queryWindowsProcess(pid, { runCommand })).resolves.toEqual({ + await expect(queryWindowsProcess(pid, { readProcessTable })).resolves.toEqual({ status: 'unavailable' }) - expect(runCommand).not.toHaveBeenCalled() + expect(readProcessTable).not.toHaveBeenCalled() } ) }) diff --git a/src/main/daemon/daemon-process-inspection.ts b/src/main/daemon/daemon-process-inspection.ts index 4f40090bfee..9f52625e3a6 100644 --- a/src/main/daemon/daemon-process-inspection.ts +++ b/src/main/daemon/daemon-process-inspection.ts @@ -8,6 +8,11 @@ import type { ProcessSignalEvidence, WindowsProcessEvidence } from './daemon-incarnation-evidence-types' +import { + readWindowsProcessCreationTime, + readWindowsProcessTableFresh, + type WindowsProcessRow +} from '../windows/windows-process-table' const execFileAsync = promisify(execFile) @@ -16,6 +21,7 @@ type InspectionCommandRunner = (file: string, args: string[], timeoutMs: number) export type DaemonProcessInspectionDependencies = { readTextFile?: (path: string) => Promise runCommand?: InspectionCommandRunner + readProcessTable?: () => Promise } export function inspectProcessSignal(pid: number): ProcessSignalEvidence { @@ -33,6 +39,12 @@ export function inspectProcessSignal(pid: number): ProcessSignalEvidence { } } +/** EPERM counts as alive: it proves some process holds the PID. */ +export function isProcessAlive(pid: number): boolean { + const signal = inspectProcessSignal(pid) + return signal === 'occupied' || signal === 'permission_denied' +} + export function inspectProcessLiveness(pid: number): ProcessLivenessVerdict { const signal = inspectProcessSignal(pid) switch (signal) { @@ -94,9 +106,7 @@ export async function readProcessCommandLine( } } -// Why: Get-CimInstance errors (Winmgmt down, corrupt WMI repository, access denied) are -// non-terminating and exit 0 with an empty $p, which is indistinguishable from "no such -// process" — so the script reports query failure explicitly instead of asserting absence. +// Only a table that was read and lacks the PID proves absence; the reader rejects a truncated one. export async function queryWindowsProcess( pid: number, dependencies: DaemonProcessInspectionDependencies = {} @@ -104,45 +114,21 @@ export async function queryWindowsProcess( if (!Number.isSafeInteger(pid) || pid <= 0) { return { status: 'unavailable' } } - const runCommand = dependencies.runCommand ?? runInspectionCommand + let rows: WindowsProcessRow[] try { - const stdout = await runCommand( - 'powershell.exe', - [ - '-NoProfile', - '-NonInteractive', - '-Command', - `$ErrorActionPreference = 'Stop'; ` + - `try { $p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}" } ` + - `catch { @{ status = 'query_failed' } | ConvertTo-Json -Compress; exit 0 }; ` + - `if (!$p) { @{ status = 'missing' } | ConvertTo-Json -Compress; exit 0 }; ` + - `$start = $null; if ($p.CreationDate) { ` + - `$start = [long]([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds() }; ` + - `@{ status = 'present'; cmd = $p.CommandLine; start = $start } | ConvertTo-Json -Compress` - ], - 3_000 - ) - const parsed = JSON.parse(stdout.trim()) as { - status?: unknown - cmd?: unknown - start?: unknown - } - // Only a query that ran and found nothing proves absence; anything else stays indeterminate. - if (parsed.status === 'missing') { - return { status: 'missing' } - } - if (parsed.status !== 'present') { - return { status: 'unavailable' } - } - return { - status: 'present', - commandLine: typeof parsed.cmd === 'string' && parsed.cmd ? parsed.cmd : null, - startedAtMs: - typeof parsed.start === 'number' && Number.isFinite(parsed.start) ? parsed.start : null - } + rows = await (dependencies.readProcessTable ?? readWindowsProcessTableFresh)() } catch { return { status: 'unavailable' } } + const row = rows.find((candidate) => candidate.pid === pid) + if (!row) { + return { status: 'missing' } + } + return { + status: 'present', + commandLine: row.command || null, + startedAtMs: row.creationTimeMs ?? readWindowsProcessCreationTime(pid) + } } // Why: the sync procfs helper in daemon-process-start-time spawns getconf per call; CLK_TCK is fixed for @@ -219,8 +205,6 @@ async function runInspectionCommand( args: string[], timeoutMs: number ): Promise { - // powershell.exe is console-subsystem: without this it flashes a conhost and - // steals foreground on every inspection (#10488). const { stdout } = await execFileAsync(file, args, { encoding: 'utf8', timeout: timeoutMs, @@ -229,6 +213,6 @@ async function runInspectionCommand( return stdout } -function hasErrorCode(error: unknown, code: string): boolean { +export function hasErrorCode(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code } diff --git a/src/main/daemon/daemon-provider-census.test.ts b/src/main/daemon/daemon-provider-census.test.ts new file mode 100644 index 00000000000..30aade980fd --- /dev/null +++ b/src/main/daemon/daemon-provider-census.test.ts @@ -0,0 +1,54 @@ +import { afterEach, expect, it, vi } from 'vitest' + +vi.mock('../ipc/pty', () => ({ setLocalPtyProvider: vi.fn() })) + +import { DaemonPtyRouter } from './daemon-pty-router' +import { createAdapter } from './daemon-pty-router-test-fixture' +import { + disconnectDaemon, + listLiveDaemonSessions, + listLiveDaemonSessionsWithProtocol, + replaceDaemonProvider, + requestIdleDaemonRetirement +} from './daemon-provider-state' +import { PROTOCOL_VERSION } from './types' + +afterEach(async () => { + await disconnectDaemon() +}) + +it('reads a census without an installed daemon as unverifiable, never as empty', async () => { + await expect(listLiveDaemonSessions()).resolves.toBeNull() + await expect(requestIdleDaemonRetirement()).resolves.toEqual({ state: 'unverifiable' }) +}) + +it('reads a census with an unanswered generation as unverifiable', async () => { + const current = createAdapter('current', ['live-1'], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, PROTOCOL_VERSION) + vi.mocked(legacy.listSessions).mockRejectedValue(new Error('daemon unreachable')) + replaceDaemonProvider(new DaemonPtyRouter({ current, legacy: [legacy] })) + + await expect(listLiveDaemonSessions()).resolves.toBeNull() +}) + +it('labels each live session with the protocol of the generation that owns it', async () => { + const current = createAdapter('current', ['live-1'], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', ['live-2'], undefined, PROTOCOL_VERSION - 1) + replaceDaemonProvider(new DaemonPtyRouter({ current, legacy: [legacy] })) + + await expect(listLiveDaemonSessionsWithProtocol()).resolves.toEqual([ + { sessionId: 'live-1', isAlive: true, protocolVersion: PROTOCOL_VERSION }, + { sessionId: 'live-2', isAlive: true, protocolVersion: PROTOCOL_VERSION - 1 } + ]) +}) + +it('lists every generation when each one answers', async () => { + const current = createAdapter('current', ['live-1'], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', ['live-2'], undefined, PROTOCOL_VERSION) + replaceDaemonProvider(new DaemonPtyRouter({ current, legacy: [legacy] })) + + await expect(listLiveDaemonSessions()).resolves.toEqual([ + { sessionId: 'live-1', isAlive: true }, + { sessionId: 'live-2', isAlive: true } + ]) +}) diff --git a/src/main/daemon/daemon-provider-init.ts b/src/main/daemon/daemon-provider-init.ts index c1d73d95a27..9d9a1b5c3e1 100644 --- a/src/main/daemon/daemon-provider-init.ts +++ b/src/main/daemon/daemon-provider-init.ts @@ -19,7 +19,8 @@ import { } from './daemon-launch-paths' import { attributeNextDaemonReplacement, - createOutOfProcessLauncher + createOutOfProcessLauncher, + type DaemonLaunchPolicy } from './daemon-out-of-process-launcher' import type { DaemonProvider } from './daemon-provider-routing' import { installDaemonProvider } from './daemon-provider-state' @@ -46,7 +47,7 @@ function logDaemonMilestone(event: string, details: Record = {} export async function initDaemonPtyProvider( signal?: AbortSignal, - options: { macosLoginSessionWatch?: boolean } = {} + options: DaemonLaunchPolicy = {} ): Promise { logDaemonMilestone('daemon-init-start') // Why: e2e coverage for the startup PTY gate (#5232) needs a daemon init that deterministically outlasts the first-window timeout. @@ -58,7 +59,7 @@ export async function initDaemonPtyProvider( const newSpawner = new DaemonSpawner({ runtimeDir, - launcher: createOutOfProcessLauncher(runtimeDir, options.macosLoginSessionWatch ?? false) + launcher: createOutOfProcessLauncher(runtimeDir, options) }) // Why: assign the module-level spawner/adapter only after both succeed, so a failed ensureRunning() leaves no stale spawner. diff --git a/src/main/daemon/daemon-provider-state.ts b/src/main/daemon/daemon-provider-state.ts index 412dda3af83..30eff6691f9 100644 --- a/src/main/daemon/daemon-provider-state.ts +++ b/src/main/daemon/daemon-provider-state.ts @@ -11,12 +11,16 @@ import { getMacDaemonTccAttributionHealth, type MacDaemonTccAttributionHealth } from './daemon-tcc-attribution' -import { PROTOCOL_VERSION } from './types' +import { PROTOCOL_VERSION, type DaemonSessionInfo, type SessionInfo } from './types' +import type { DaemonIdleRetirementResult } from './daemon-pty-runtime-state' let spawner: DaemonSpawner | null = null let adapter: DaemonProvider | null = null -export function installDaemonProvider(newSpawner: DaemonSpawner, newAdapter: DaemonProvider): void { +export function installDaemonProvider( + newSpawner: DaemonSpawner | null, + newAdapter: DaemonProvider +): void { spawner = newSpawner replaceDaemonProvider(newAdapter) } @@ -95,26 +99,6 @@ export async function getCurrentDaemonMacTccAttributionHealth(): Promise { - if (!adapter) { - return null - } - const adapters = - adapter instanceof DaemonPtyRouter || adapter instanceof DegradedDaemonPtyProvider - ? adapter.getAllAdapters() - : [adapter] - const inventories = await Promise.allSettled( - adapters.map((daemonAdapter) => daemonAdapter.listProcesses()) - ) - if (inventories.some((inventory) => inventory.status === 'rejected')) { - return null - } - return inventories.flatMap((inventory) => - inventory.status === 'fulfilled' ? inventory.value.map((process) => process.id) : [] - ) -} - // Why: keep the module-level adapter and ipc/pty.ts's localProvider in sync so app-quit can't dispose a stale reference. export function replaceDaemonProvider(newAdapter: DaemonProvider): void { adapter = newAdapter @@ -135,3 +119,82 @@ export async function shutdownDaemon(): Promise { await spawner?.shutdown() spawner = null } + +/** Returns null unless every daemon generation supplied an authoritative inventory. */ +export async function listLiveDaemonPtyIds(): Promise { + if (!adapter) { + return null + } + const adapters = + adapter instanceof DaemonPtyRouter || adapter instanceof DegradedDaemonPtyProvider + ? adapter.getAllAdapters() + : [adapter] + const inventories = await Promise.allSettled( + adapters.map((daemonAdapter) => daemonAdapter.listProcesses()) + ) + if (inventories.some((inventory) => inventory.status === 'rejected')) { + return null + } + return inventories.flatMap((inventory) => + inventory.status === 'fulfilled' ? inventory.value.map((process) => process.id) : [] + ) +} + +/** Returns null unless every daemon generation supplied an authoritative session inventory. */ +export async function listLiveDaemonSessions(): Promise { + const sessions = await listLiveDaemonSessionsWithProtocol() + return sessions?.map(({ protocolVersion: _protocolVersion, ...session }) => session) ?? null +} + +/** Like listLiveDaemonSessions, with the protocol of the daemon generation owning each session. */ +export async function listLiveDaemonSessionsWithProtocol(): Promise { + if (!adapter) { + return null + } + const adapters = + adapter instanceof DaemonPtyRouter || adapter instanceof DegradedDaemonPtyProvider + ? adapter.getAllAdapters() + : [adapter] + const inventories = await Promise.allSettled( + adapters.map(async (daemonAdapter) => + (await daemonAdapter.listSessions()).map((session) => ({ + ...session, + protocolVersion: daemonAdapter.protocolVersion + })) + ) + ) + if (inventories.some((inventory) => inventory.status === 'rejected')) { + return null + } + return inventories.flatMap((inventory) => + inventory.status === 'fulfilled' ? inventory.value : [] + ) +} + +/** Terminals the degraded provider ran in-process; none outside degraded mode. */ +export async function countInProcessFallbackTerminals(): Promise { + return adapter instanceof DegradedDaemonPtyProvider + ? (await adapter.fallback.listProcesses()).length + : 0 +} + +/** Atomically fence new daemon terminals and retire only an idle, single-generation daemon. */ +export async function requestIdleDaemonRetirement(): Promise { + if (!adapter) { + return { state: 'unverifiable' } + } + if (adapter instanceof DegradedDaemonPtyProvider) { + return { state: 'unverifiable' } + } + if (adapter instanceof DaemonPtyRouter) { + return adapter.requestIdleRetirement() + } + return adapter.requestIdleRetirement() +} + +/** Reopens terminal admission when an idle-retirement attempt did not retire the daemon. */ +export function releaseDaemonRetirementFence(): void { + if (adapter && !(adapter instanceof DegradedDaemonPtyProvider)) { + adapter.releaseIdleRetirementFence() + } +} diff --git a/src/main/daemon/daemon-pty-event-subscriptions.ts b/src/main/daemon/daemon-pty-event-subscriptions.ts index 6d95e2de1c7..068c91f0592 100644 --- a/src/main/daemon/daemon-pty-event-subscriptions.ts +++ b/src/main/daemon/daemon-pty-event-subscriptions.ts @@ -3,7 +3,12 @@ import { removeDaemonListener } from './daemon-listener-registry' import { emitPtyListeners } from './daemon-pty-listener-emission' import type { PtyIncarnationId } from '../../shared/pty-incarnation' import { DaemonPtySessionInventory } from './daemon-pty-session-inventory' -import { CLEAN_DISCONNECT_PROTOCOL_VERSION } from './types' +import { + CLEAN_DISCONNECT_PROTOCOL_VERSION, + type ListSessionsResult, + type ShutdownIfIdleResult +} from './types' +import type { DaemonIdleRetirementResult } from './daemon-pty-runtime-state' import type { PtyBackgroundStreamEvent } from '../providers/types' export abstract class DaemonPtyEventSubscriptions extends DaemonPtySessionInventory { @@ -85,6 +90,86 @@ export abstract class DaemonPtyEventSubscriptions extends DaemonPtySessionInvent this.recordAuthenticatedIdentity() } + async requestIdleRetirement(): Promise { + if (this.protocolVersion < CLEAN_DISCONNECT_PROTOCOL_VERSION) { + return { state: 'unsupported' } + } + if (this.idleRetirementState === 'retiring') { + return { state: 'retiring' } + } + if (this.idleRetirementPromise) { + return this.idleRetirementPromise + } + if ( + this.disconnectOnlyPromise || + (this.respawnAdoptionClosed && this.idleRetirementState === 'open') + ) { + return { state: 'unverifiable' } + } + this.idleRetirementAdmissionClosed = true + this.respawnAdoptionClosed = true + this.idleRetirementState = 'checking' + const request = this.finishIdleRetirementRequest().finally(() => { + if (this.idleRetirementPromise === request) { + this.idleRetirementPromise = null + } + }) + this.idleRetirementPromise = request + return request + } + + private async finishIdleRetirementRequest(): Promise { + try { + await this.client.ensureConnected() + } catch { + // Nothing was asked of the daemon, so nothing can be retiring. + this.reopenAfterRefusedIdleRetirement() + return { state: 'unverifiable' } + } + try { + const result = await this.client.request('shutdownIfIdle', undefined) + if (result.retiring) { + this.idleRetirementState = 'retiring' + return { state: 'retiring' } + } + let liveSessions: number | null = null + try { + const inventory = await this.client.request('listSessions', undefined) + liveSessions = inventory.sessions.filter((session) => session.isAlive).length + } catch { + liveSessions = null + } + this.reopenAfterRefusedIdleRetirement() + return { + state: 'busy', + liveSessions, + admissionReopened: true + } + } catch { + // The daemon may have accepted before contact was lost; keep admission and respawn fenced. + this.idleRetirementState = 'unverifiable' + return { state: 'unverifiable' } + } + } + + /** Reopens admission an idle-retirement attempt fenced without retiring the daemon. */ + releaseIdleRetirementFence(): void { + // An unverifiable attempt may have been accepted, so it stays fenced like a retiring one. + if ( + this.idleRetirementState !== 'retiring' && + this.idleRetirementState !== 'unverifiable' && + !this.idleRetirementPromise + ) { + this.reopenAfterRefusedIdleRetirement() + } + } + + private reopenAfterRefusedIdleRetirement(): void { + this.idleRetirementState = 'open' + this.idleRetirementAdmissionClosed = false + this.respawnAdoptionClosed = false + } + // Why: unlike dispose(), leave history files unclean (no endedAt) so the next launch treats them as crash-recoverable, // but still write a final checkpoint so a daemon crash while Orca is closed has recovery data. async disconnectOnly(): Promise { diff --git a/src/main/daemon/daemon-pty-router-test-fixture.ts b/src/main/daemon/daemon-pty-router-test-fixture.ts new file mode 100644 index 00000000000..23d2c5f48b4 --- /dev/null +++ b/src/main/daemon/daemon-pty-router-test-fixture.ts @@ -0,0 +1,168 @@ +import { vi } from 'vitest' +import { settledWriteStub } from '../providers/settled-pty-write-stub' +import type { DaemonPtyAdapter } from './daemon-pty-adapter' +import type { PtyBackgroundStreamEvent, PtySpawnOptions, PtySpawnResult } from '../providers/types' +import { + AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, + AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, + GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION +} from './types' +import { SNAPSHOT_SERIALIZER_FIDELITY_DAEMON_PROTOCOL_VERSION } from './daemon-protocol-version' + +type AdapterMock = DaemonPtyAdapter & { + emitData: (id: string, data: string, sequenceChars?: number) => void + emitBackground: (event: PtyBackgroundStreamEvent) => void + emitExit: (id: string, code: number, incarnationId?: string) => void + emitIdentityChange: () => void + triggerWriteUnavailable: (id: string) => void +} + +export function createAdapter( + label: string, + sessions: string[] = [], + reconcileResult?: { alive: string[]; killed: string[] }, + protocolVersion = GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION +): AdapterMock { + const writes: { id: string; data: string }[] = [] + const dataListeners: ((payload: { id: string; data: string; sequenceChars?: number }) => void)[] = + [] + const backgroundListeners: ((payload: PtyBackgroundStreamEvent) => void)[] = [] + const writeUnavailableListeners: ((payload: { id: string }) => void)[] = [] + const exitListeners: ((payload: { id: string; code: number; incarnationId?: string }) => void)[] = + [] + const identityChangeListeners: (() => void)[] = [] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the router calls only the adapter members this mock defines. + return { + protocolVersion, + supportsGitCredentialGuardHost: () => + protocolVersion >= GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION, + supportsAgentSessionClaims: () => + protocolVersion >= AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, + supportsAgentSessionCreateOperations: () => + protocolVersion >= AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, + providesAgentSessionOwnerListings: () => + protocolVersion >= AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, + canProvideAuthoritativeBufferSnapshot: () => + protocolVersion >= SNAPSHOT_SERIALIZER_FIDELITY_DAEMON_PROTOCOL_VERSION, + spawn: vi.fn(async (opts: PtySpawnOptions): Promise => { + const id = opts.sessionId ?? `${label}-new` + sessions.push(id) + return { id } + }), + listProcesses: vi.fn(async () => + sessions.map((id) => ({ + id, + cwd: '', + title: label + })) + ), + listSessions: vi.fn(async () => sessions.map((sessionId) => ({ sessionId, isAlive: true }))), + requestIdleRetirement: vi.fn(async () => ({ state: 'retiring' as const })), + releaseIdleRetirementFence: vi.fn(), + hasPty: vi.fn((id: string) => sessions.includes(id)), + probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)), + write: vi.fn((id: string, data: string) => { + writes.push({ id, data }) + }), + writeWithSettlement: vi.fn(settledWriteStub()), + resize: vi.fn(), + setPtyBackgrounded: vi.fn(), + getBufferSnapshot: vi.fn(async () => null), + shutdown: vi.fn(async (id: string) => { + const idx = sessions.indexOf(id) + if (idx !== -1) { + sessions.splice(idx, 1) + } + }), + attach: vi.fn(async () => {}), + sendSignal: vi.fn(async () => {}), + getCwd: vi.fn(async () => ''), + getInitialCwd: vi.fn(async () => ''), + clearBuffer: vi.fn(async () => {}), + acknowledgeDataEvent: vi.fn(), + hasChildProcesses: vi.fn(async () => false), + getForegroundProcess: vi.fn(async () => null), + inspectProcess: vi.fn(async () => ({ foregroundProcess: null, hasChildProcesses: false })), + confirmForegroundProcess: vi.fn(async () => `${label}-confirmed`), + serialize: vi.fn(async () => '{}'), + revive: vi.fn(async () => {}), + getDefaultShell: vi.fn(async () => '/bin/zsh'), + getProfiles: vi.fn(async () => []), + onData: vi.fn( + (callback: (payload: { id: string; data: string; sequenceChars?: number }) => void) => { + dataListeners.push(callback) + return () => { + const idx = dataListeners.indexOf(callback) + if (idx !== -1) { + dataListeners.splice(idx, 1) + } + } + } + ), + onBackgroundStreamEvent: vi.fn((callback: (payload: PtyBackgroundStreamEvent) => void) => { + backgroundListeners.push(callback) + return () => { + const idx = backgroundListeners.indexOf(callback) + if (idx !== -1) { + backgroundListeners.splice(idx, 1) + } + } + }), + onWriteUnavailable: vi.fn((callback: (payload: { id: string }) => void) => { + writeUnavailableListeners.push(callback) + return () => { + const idx = writeUnavailableListeners.indexOf(callback) + if (idx !== -1) { + writeUnavailableListeners.splice(idx, 1) + } + } + }), + onExit: vi.fn( + (callback: (payload: { id: string; code: number; incarnationId?: string }) => void) => { + exitListeners.push(callback) + return () => { + const idx = exitListeners.indexOf(callback) + if (idx !== -1) { + exitListeners.splice(idx, 1) + } + } + } + ), + onDaemonIdentityChanged: vi.fn((callback: () => void) => { + identityChangeListeners.push(callback) + return () => { + const idx = identityChangeListeners.indexOf(callback) + if (idx !== -1) { + identityChangeListeners.splice(idx, 1) + } + } + }), + ackColdRestore: vi.fn(), + clearTombstone: vi.fn(), + reconcileOnStartup: vi.fn(async () => reconcileResult ?? { alive: sessions, killed: [] }), + dispose: vi.fn(), + disconnectOnly: vi.fn(async () => {}), + emitData: (id: string, data: string, sequenceChars?: number) => { + for (const listener of dataListeners) { + listener({ id, data, ...(sequenceChars === undefined ? {} : { sequenceChars }) }) + } + }, + emitBackground: (event: PtyBackgroundStreamEvent) => { + for (const listener of backgroundListeners) { + listener(event) + } + }, + emitExit: (id: string, code: number, incarnationId?: string) => { + for (const listener of exitListeners) { + listener({ id, code, ...(incarnationId ? { incarnationId } : {}) }) + } + }, + emitIdentityChange: () => identityChangeListeners.forEach((listener) => listener()), + triggerWriteUnavailable: (id: string) => { + for (const listener of writeUnavailableListeners) { + listener({ id }) + } + }, + _writes: writes + } as unknown as AdapterMock +} diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index 61db990b21c..7e35d416b12 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -1,29 +1,20 @@ +import { createAdapter } from './daemon-pty-router-test-fixture' import { describe, expect, it, vi } from 'vitest' import { DaemonPtyRouter } from './daemon-pty-router' +import { stubWriteSettlement } from '../providers/settled-pty-write-stub' import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors' import type { DaemonPtyAdapter } from './daemon-pty-adapter' -import { settledWriteStub, stubWriteSettlement } from '../providers/settled-pty-write-stub' -import type { PtyBackgroundStreamEvent, PtySpawnOptions, PtySpawnResult } from '../providers/types' +import type { PtySpawnResult } from '../providers/types' import { AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, - AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, - GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION + AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION } from './types' import { HISTORY_SEED_TRANSFER_PROTOCOL_VERSION, PROTOCOL_VERSION, - SNAPSHOT_SERIALIZER_FIDELITY_DAEMON_PROTOCOL_VERSION, STABLE_PANE_ATTACH_ONLY_DAEMON_PROTOCOL_VERSION } from './daemon-protocol-version' -type AdapterMock = DaemonPtyAdapter & { - emitData: (id: string, data: string, sequenceChars?: number) => void - emitBackground: (event: PtyBackgroundStreamEvent) => void - emitExit: (id: string, code: number, incarnationId?: string) => void - emitIdentityChange: () => void - triggerWriteUnavailable: (id: string) => void -} - const LARGE_RECONCILE_SESSION_COUNT = 150_000 function buildSessionIds(prefix: string, count: number): string[] { @@ -34,152 +25,6 @@ function buildSessionIds(prefix: string, count: number): string[] { return ids } -function createAdapter( - label: string, - sessions: string[] = [], - reconcileResult?: { alive: string[]; killed: string[] }, - protocolVersion = GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION -): AdapterMock { - const writes: { id: string; data: string }[] = [] - const dataListeners: ((payload: { id: string; data: string; sequenceChars?: number }) => void)[] = - [] - const backgroundListeners: ((payload: PtyBackgroundStreamEvent) => void)[] = [] - const writeUnavailableListeners: ((payload: { id: string }) => void)[] = [] - const exitListeners: ((payload: { id: string; code: number; incarnationId?: string }) => void)[] = - [] - const identityChangeListeners: (() => void)[] = [] - return { - protocolVersion, - supportsGitCredentialGuardHost: () => - protocolVersion >= GIT_CREDENTIAL_GUARD_HOST_PROTOCOL_VERSION, - supportsAgentSessionClaims: () => - protocolVersion >= AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, - supportsAgentSessionCreateOperations: () => - protocolVersion >= AGENT_SESSION_CREATE_OPERATION_DAEMON_PROTOCOL_VERSION, - providesAgentSessionOwnerListings: () => - protocolVersion >= AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION, - canProvideAuthoritativeBufferSnapshot: () => - protocolVersion >= SNAPSHOT_SERIALIZER_FIDELITY_DAEMON_PROTOCOL_VERSION, - spawn: vi.fn(async (opts: PtySpawnOptions): Promise => { - const id = opts.sessionId ?? `${label}-new` - sessions.push(id) - return { id } - }), - listProcesses: vi.fn(async () => - sessions.map((id) => ({ - id, - cwd: '', - title: label - })) - ), - hasPty: vi.fn((id: string) => sessions.includes(id)), - probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)), - write: vi.fn((id: string, data: string) => { - writes.push({ id, data }) - }), - writeWithSettlement: vi.fn(settledWriteStub()), - resize: vi.fn(), - setPtyBackgrounded: vi.fn(), - getBufferSnapshot: vi.fn(async () => null), - shutdown: vi.fn(async (id: string) => { - const idx = sessions.indexOf(id) - if (idx !== -1) { - sessions.splice(idx, 1) - } - }), - attach: vi.fn(async () => {}), - sendSignal: vi.fn(async () => {}), - getCwd: vi.fn(async () => ''), - getInitialCwd: vi.fn(async () => ''), - clearBuffer: vi.fn(async () => {}), - acknowledgeDataEvent: vi.fn(), - hasChildProcesses: vi.fn(async () => false), - getForegroundProcess: vi.fn(async () => null), - inspectProcess: vi.fn(async () => ({ foregroundProcess: null, hasChildProcesses: false })), - confirmForegroundProcess: vi.fn(async () => `${label}-confirmed`), - serialize: vi.fn(async () => '{}'), - revive: vi.fn(async () => {}), - getDefaultShell: vi.fn(async () => '/bin/zsh'), - getProfiles: vi.fn(async () => []), - onData: vi.fn( - (callback: (payload: { id: string; data: string; sequenceChars?: number }) => void) => { - dataListeners.push(callback) - return () => { - const idx = dataListeners.indexOf(callback) - if (idx !== -1) { - dataListeners.splice(idx, 1) - } - } - } - ), - onBackgroundStreamEvent: vi.fn((callback: (payload: PtyBackgroundStreamEvent) => void) => { - backgroundListeners.push(callback) - return () => { - const idx = backgroundListeners.indexOf(callback) - if (idx !== -1) { - backgroundListeners.splice(idx, 1) - } - } - }), - onWriteUnavailable: vi.fn((callback: (payload: { id: string }) => void) => { - writeUnavailableListeners.push(callback) - return () => { - const idx = writeUnavailableListeners.indexOf(callback) - if (idx !== -1) { - writeUnavailableListeners.splice(idx, 1) - } - } - }), - onExit: vi.fn( - (callback: (payload: { id: string; code: number; incarnationId?: string }) => void) => { - exitListeners.push(callback) - return () => { - const idx = exitListeners.indexOf(callback) - if (idx !== -1) { - exitListeners.splice(idx, 1) - } - } - } - ), - onDaemonIdentityChanged: vi.fn((callback: () => void) => { - identityChangeListeners.push(callback) - return () => { - const idx = identityChangeListeners.indexOf(callback) - if (idx !== -1) { - identityChangeListeners.splice(idx, 1) - } - } - }), - ackColdRestore: vi.fn(), - clearTombstone: vi.fn(), - reconcileOnStartup: vi.fn(async () => reconcileResult ?? { alive: sessions, killed: [] }), - dispose: vi.fn(), - disconnectOnly: vi.fn(async () => {}), - emitData: (id: string, data: string, sequenceChars?: number) => { - for (const listener of dataListeners) { - listener({ id, data, ...(sequenceChars === undefined ? {} : { sequenceChars }) }) - } - }, - emitBackground: (event: PtyBackgroundStreamEvent) => { - for (const listener of backgroundListeners) { - listener(event) - } - }, - emitExit: (id: string, code: number, incarnationId?: string) => { - for (const listener of exitListeners) { - listener({ id, code, ...(incarnationId ? { incarnationId } : {}) }) - } - }, - emitIdentityChange: () => identityChangeListeners.forEach((listener) => listener()), - triggerWriteUnavailable: (id: string) => { - for (const listener of writeUnavailableListeners) { - listener({ id }) - } - }, - _writes: writes - } as unknown as AdapterMock -} - it('forwards dead-endpoint write-unavailable signals from every routed adapter', () => { // Why revert-sensitive: main subscribes on the ROUTED provider, so if the router // does not forward this the STA-2373 fan-out never reaches the renderer and only @@ -247,6 +92,83 @@ it('forwards the owning legacy daemon sequence from attach', async () => { }) describe('DaemonPtyRouter', () => { + describe('idle retirement', () => { + it('retires every empty daemon generation and fences subsequent spawns', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, PROTOCOL_VERSION) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.requestIdleRetirement()).resolves.toEqual({ state: 'retiring' }) + expect(current.requestIdleRetirement).toHaveBeenCalledOnce() + expect(legacy.requestIdleRetirement).toHaveBeenCalledOnce() + await expect(router.spawn({ sessionId: 'late', cols: 80, rows: 24 })).rejects.toThrow( + 'Terminal daemon is decommissioning' + ) + }) + + it('reports live inventory before retiring any generation and reopens admission', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', ['legacy-live'], undefined, PROTOCOL_VERSION) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.requestIdleRetirement()).resolves.toEqual({ + state: 'busy', + liveSessions: 1, + admissionReopened: true + }) + expect(current.requestIdleRetirement).not.toHaveBeenCalled() + expect(legacy.requestIdleRetirement).not.toHaveBeenCalled() + await expect( + router.spawn({ sessionId: 'after-refusal', cols: 80, rows: 24 }) + ).resolves.toEqual({ + id: 'after-refusal' + }) + }) + + it('does not partially retire when a generation predates clean idle shutdown', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, 23) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.requestIdleRetirement()).resolves.toEqual({ state: 'unsupported' }) + expect(current.requestIdleRetirement).not.toHaveBeenCalled() + expect(legacy.requestIdleRetirement).not.toHaveBeenCalled() + }) + + it('keeps admission fenced after a partial multi-generation retirement', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, PROTOCOL_VERSION) + vi.mocked(legacy.requestIdleRetirement).mockResolvedValueOnce({ + state: 'busy', + liveSessions: 0 + }) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.requestIdleRetirement()).resolves.toEqual({ state: 'unverifiable' }) + await expect(router.spawn({ sessionId: 'unsafe', cols: 80, rows: 24 })).rejects.toThrow( + 'Terminal daemon is decommissioning' + ) + }) + + it('does not certify reopened admission when another generation retired beside live sessions', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, PROTOCOL_VERSION) + vi.mocked(legacy.requestIdleRetirement).mockResolvedValueOnce({ + state: 'busy', + liveSessions: 1, + admissionReopened: true + }) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + await expect(router.requestIdleRetirement()).resolves.toEqual({ + state: 'busy', + liveSessions: 1 + }) + await expect( + router.spawn({ sessionId: 'unsafe-partial', cols: 80, rows: 24 }) + ).rejects.toThrow('Terminal daemon is decommissioning') + }) + }) + it('reports separate conservative resume and fresh-create boundaries', () => { const current = createAdapter( 'current', diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 0534c9d2869..8b5b527cb31 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -1,3 +1,4 @@ +import { reconcileDaemonRouterSessions } from './daemon-router-session-reconciliation' import type { DaemonPtyAdapter } from './daemon-pty-adapter' import { DaemonPtyAdapterSubscriptionFanout } from './daemon-pty-adapter-subscription-fanout' import type { @@ -12,6 +13,8 @@ import type { PtyProcessInspection } from '../providers/pty-process-inspection' import { shouldHandoffDaemonHistory } from './daemon-history-handoff' import type { DaemonPtyRouterDataEvent, DaemonPtyRouterExitEvent } from './daemon-pty-router-events' import { DaemonSessionOwnerResolver } from './daemon-session-owner-resolution' +import type { DaemonIdleRetirementResult } from './daemon-pty-runtime-state' +import { DaemonRouterRetirement } from './daemon-router-retirement' import type { WriteSettlement } from '../../shared/pty-write-settlement' import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc-color-reply' @@ -21,6 +24,7 @@ export class DaemonPtyRouter implements IPtyProvider { private sessionAdapters = new Map() private readonly ownerResolver: DaemonSessionOwnerResolver private readonly subscriptions: DaemonPtyAdapterSubscriptionFanout + private readonly retirement = new DaemonRouterRetirement(() => this.allAdapters()) constructor(opts: { current: DaemonPtyAdapter; legacy: DaemonPtyAdapter[] }) { this.current = opts.current @@ -40,17 +44,34 @@ export class DaemonPtyRouter implements IPtyProvider { } async spawn(opts: PtySpawnOptions): Promise { - if (opts.attachOnly && opts.sessionId) { - return await this.ownerResolver.spawnAttachOnly({ ...opts, sessionId: opts.sessionId }) + if (this.retirement.admissionClosed) { + throw new Error('Terminal daemon is decommissioning') } - const adapter = opts.sessionId ? this.sessionAdapters.get(opts.sessionId) : undefined - const target = adapter ?? this.current - const result = await target.spawn(opts) - // Why: the adapter filters intentional recovery exits and canonical-ID races before publishing proof. - if (!result.exitedBeforeSpawnReply) { - this.ownerResolver.recordRoute(result.id, target, result.incarnationId) + // Why counted: an idle-retirement census must not race a spawn it cannot yet see. + this.retirement.spawnInFlight++ + try { + if (opts.attachOnly && opts.sessionId) { + return await this.ownerResolver.spawnAttachOnly({ ...opts, sessionId: opts.sessionId }) + } + const adapter = opts.sessionId ? this.sessionAdapters.get(opts.sessionId) : undefined + const target = adapter ?? this.current + const result = await target.spawn(opts) + // Why: the adapter filters intentional recovery exits and canonical-ID races before publishing proof. + if (!result.exitedBeforeSpawnReply) { + this.ownerResolver.recordRoute(result.id, target, result.incarnationId) + } + return result + } finally { + this.retirement.spawnInFlight-- } - return result + } + + requestIdleRetirement(): Promise { + return this.retirement.requestIdleRetirement() + } + + releaseIdleRetirementFence(): void { + this.retirement.releaseFence() } supportsGitCredentialGuardHost(sessionId?: string): boolean { @@ -257,38 +278,10 @@ export class DaemonPtyRouter implements IPtyProvider { this.adapterFor(sessionId).clearTombstone(sessionId) } - async reconcileOnStartup(validWorktreeIds: Set): Promise<{ - alive: string[] - killed: string[] - }> { - const alive: string[] = [] - const killed: string[] = [] - const aliveProviders = new Map>() - for (const adapter of this.allAdapters()) { - const result = await adapter.reconcileOnStartup(validWorktreeIds) - // Why: daemon startup can reconcile many restored sessions; spreading - // those arrays into push can exceed JavaScript's argument limit. - for (const id of result.alive) { - alive.push(id) - } - for (const id of result.killed) { - killed.push(id) - } - for (const id of result.alive) { - const providers = aliveProviders.get(id) ?? new Set() - providers.add(adapter) - aliveProviders.set(id, providers) - } - } - for (const id of new Set([...alive, ...killed])) { - const providers = aliveProviders.get(id) - if (providers?.size === 1) { - this.ownerResolver.recordRoute(id, providers.values().next().value!) - } else { - this.ownerResolver.forgetRoute(id) - } - } - return { alive, killed } + async reconcileOnStartup( + validWorktreeIds: Set + ): Promise<{ alive: string[]; killed: string[] }> { + return reconcileDaemonRouterSessions(this.allAdapters(), this.ownerResolver, validWorktreeIds) } dispose(): void { diff --git a/src/main/daemon/daemon-pty-runtime-state.ts b/src/main/daemon/daemon-pty-runtime-state.ts index 38480af1489..481f8a91670 100644 --- a/src/main/daemon/daemon-pty-runtime-state.ts +++ b/src/main/daemon/daemon-pty-runtime-state.ts @@ -71,6 +71,12 @@ export type DaemonIdentityChangeEvent = { current: DaemonEndpointIdentity } +export type DaemonIdleRetirementResult = + | { state: 'retiring' } + | { state: 'busy'; liveSessions: number | null; admissionReopened?: true } + | { state: 'unsupported' } + | { state: 'unverifiable' } + export abstract class DaemonPtyRuntimeState { readonly protocolVersion: number protected socketPath: string @@ -92,6 +98,9 @@ export abstract class DaemonPtyRuntimeState { protected packagedAppVersion: string | null protected pendingRespawnAdoptionRelease: (() => void) | null = null protected respawnAdoptionClosed = false + protected idleRetirementAdmissionClosed = false + protected idleRetirementState: 'open' | 'checking' | 'retiring' | 'unverifiable' = 'open' + protected idleRetirementPromise: Promise | null = null protected respawnPromise: Promise | null = null protected staleBundleReplacementPromise: Promise | null = null protected writeRecoveryPromise: Promise | null = null diff --git a/src/main/daemon/daemon-pty-session-spawn.ts b/src/main/daemon/daemon-pty-session-spawn.ts index 388919dd2e3..b66fd9e5afb 100644 --- a/src/main/daemon/daemon-pty-session-spawn.ts +++ b/src/main/daemon/daemon-pty-session-spawn.ts @@ -25,7 +25,15 @@ import { injectHistoryEnv, injectWslFishHistoryEnv, logHistoryInjection } from ' import { addWslEnvKeys } from '../wsl-env' export abstract class DaemonPtySessionSpawn extends DaemonPtySpawnResult { + // Checked again in doSpawn: retirement can close admission while spawn awaits. + private assertSpawnAdmission(): void { + if (this.idleRetirementAdmissionClosed) { + throw new Error('Terminal daemon is decommissioning') + } + } + async spawn(opts: PtySpawnOptions): Promise { + this.assertSpawnAdmission() const spawnOpts = this.withHistoryIsolation(opts) const sessionId = spawnOpts.sessionId ?? mintPtySessionId(spawnOpts.worktreeId) const operation: PendingDaemonSpawnOperation = { @@ -105,6 +113,7 @@ export abstract class DaemonPtySessionSpawn extends DaemonPtySpawnResult { operation: PendingDaemonSpawnOperation, historyRecovery: HistoryRecoveryContext ): Promise { + this.assertSpawnAdmission() if ( opts.agentSessionEnsure && this.protocolVersion < AGENT_SESSION_CLAIM_DAEMON_PROTOCOL_VERSION diff --git a/src/main/daemon/daemon-request-router.ts b/src/main/daemon/daemon-request-router.ts index d15514ea2ae..1184517c6dc 100644 --- a/src/main/daemon/daemon-request-router.ts +++ b/src/main/daemon/daemon-request-router.ts @@ -10,6 +10,7 @@ import type { DaemonSessionBackgroundRouting } from './daemon-session-background import { recordDaemonStreamBacklogEvent } from './daemon-stream-backlog-probe' import type { DaemonStreamDataBatcher } from './daemon-stream-data-batcher' import type { DaemonTerminalAdmission } from './daemon-terminal-admission' +import { ptySpawnHealthPlatformCoverage } from './daemon-health-identity' import type { TerminalHistorySeedTransferRegistry } from './terminal-history-seed-transfer-registry' import type { TerminalHost } from './terminal-host' import { SessionNotFoundError, type DaemonRequest } from './types' @@ -156,7 +157,7 @@ export class DaemonRequestRouter { return { health: await readCurrentProcessMacSystemResolverHealth() } case 'ptySpawnHealth': await this.options.ptySpawnHealthCheck() - return { healthy: true } + return { healthy: true, coverage: ptySpawnHealthPlatformCoverage() } case 'shutdown': return this.shutdown(clientId, request.id, request.payload.killSessions) } diff --git a/src/main/daemon/daemon-router-retirement.test.ts b/src/main/daemon/daemon-router-retirement.test.ts new file mode 100644 index 00000000000..66e6e60172e --- /dev/null +++ b/src/main/daemon/daemon-router-retirement.test.ts @@ -0,0 +1,64 @@ +import { expect, it, vi } from 'vitest' +import { DaemonRouterRetirement } from './daemon-router-retirement' +import { createAdapter } from './daemon-pty-router-test-fixture' +import { PROTOCOL_VERSION } from './types' + +it.each(['inventory', 'protocol', 'spawn', 'live'] as const)( + 'does not reopen admission on a %s retry after partial retirement', + async (failure) => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + const legacy = createAdapter('legacy', [], undefined, PROTOCOL_VERSION) + let adapters = [current, legacy] + const retirement = new DaemonRouterRetirement(() => adapters) + vi.mocked(legacy.requestIdleRetirement).mockResolvedValueOnce({ + state: 'busy', + liveSessions: 0 + }) + await expect(retirement.requestIdleRetirement()).resolves.toEqual({ state: 'unverifiable' }) + expect(retirement.admissionClosed).toBe(true) + if (failure === 'inventory') { + vi.mocked(current.listSessions).mockRejectedValueOnce(new Error('lost connection')) + } else if (failure === 'protocol') { + adapters = [createAdapter('old', [], undefined, 23)] + } else if (failure === 'spawn') { + retirement.spawnInFlight = 1 + } else { + adapters = [createAdapter('live', ['existing'], undefined, PROTOCOL_VERSION)] + } + expect(await retirement.requestIdleRetirement()).not.toHaveProperty('admissionReopened') + expect(retirement.admissionClosed).toBe(true) + } +) + +it('does not reopen when every native result is busy without reopening proof', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + vi.mocked(current.requestIdleRetirement).mockResolvedValue({ state: 'busy', liveSessions: 0 }) + const retirement = new DaemonRouterRetirement(() => [current]) + await expect(retirement.requestIdleRetirement()).resolves.toEqual({ state: 'unverifiable' }) + expect(retirement.admissionClosed).toBe(true) +}) + +it('keeps the fence through a lost native reply and failed retry inventory', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + vi.mocked(current.requestIdleRetirement).mockRejectedValueOnce(new Error('lost stop reply')) + const retirement = new DaemonRouterRetirement(() => [current]) + await expect(retirement.requestIdleRetirement()).rejects.toThrow('lost stop reply') + vi.mocked(current.listSessions).mockRejectedValueOnce(new Error('lost connection')) + await expect(retirement.requestIdleRetirement()).resolves.toEqual({ state: 'unverifiable' }) + expect(retirement.admissionClosed).toBe(true) +}) + +it('releases a fence left by an incomplete retirement, but never one that retired', async () => { + const current = createAdapter('current', [], undefined, PROTOCOL_VERSION) + vi.mocked(current.requestIdleRetirement).mockResolvedValueOnce({ state: 'busy', liveSessions: 0 }) + const retirement = new DaemonRouterRetirement(() => [current]) + await expect(retirement.requestIdleRetirement()).resolves.toEqual({ state: 'unverifiable' }) + expect(retirement.admissionClosed).toBe(true) + retirement.releaseFence() + expect(retirement.admissionClosed).toBe(false) + expect(current.releaseIdleRetirementFence).toHaveBeenCalledOnce() + + await expect(retirement.requestIdleRetirement()).resolves.toEqual({ state: 'retiring' }) + retirement.releaseFence() + expect(retirement.admissionClosed).toBe(true) +}) diff --git a/src/main/daemon/daemon-router-retirement.ts b/src/main/daemon/daemon-router-retirement.ts new file mode 100644 index 00000000000..5b8c46c3556 --- /dev/null +++ b/src/main/daemon/daemon-router-retirement.ts @@ -0,0 +1,82 @@ +import type { DaemonPtyAdapter } from './daemon-pty-adapter' +import type { DaemonIdleRetirementResult } from './daemon-pty-runtime-state' +import { CLEAN_DISCONNECT_PROTOCOL_VERSION } from './types' + +export class DaemonRouterRetirement { + admissionClosed = false + spawnInFlight = 0 + private retirementAttempted = false + private retired = false + private idleRetirementPromise: Promise | null = null + + constructor(private readonly allAdapters: () => DaemonPtyAdapter[]) {} + + /** Reopens a fence left by an attempt that did not retire every generation. */ + releaseFence(): void { + if (this.idleRetirementPromise || this.retired) { + return + } + this.admissionClosed = false + for (const adapter of this.allAdapters()) { + adapter.releaseIdleRetirementFence() + } + } + + async requestIdleRetirement(): Promise { + if (this.idleRetirementPromise) { + return this.idleRetirementPromise + } + this.admissionClosed = true + const request = this.finishIdleRetirementRequest().finally(() => { + if (this.idleRetirementPromise === request) { + this.idleRetirementPromise = null + } + }) + this.idleRetirementPromise = request + return request + } + + private async finishIdleRetirementRequest(): Promise { + const adapters = this.allAdapters() + if (this.spawnInFlight > 0) { + this.admissionClosed = this.retirementAttempted + return { state: 'busy', liveSessions: null } + } + if (adapters.some((adapter) => adapter.protocolVersion < CLEAN_DISCONNECT_PROTOCOL_VERSION)) { + this.admissionClosed = this.retirementAttempted + return { state: 'unsupported' } + } + const inventories = await Promise.allSettled(adapters.map((adapter) => adapter.listSessions())) + if (inventories.some((inventory) => inventory.status === 'rejected')) { + this.admissionClosed = this.retirementAttempted + return { state: 'unverifiable' } + } + // Each adapter's inventory lists live sessions only. + const liveSessions = inventories.reduce( + (count, inventory) => count + (inventory.status === 'fulfilled' ? inventory.value.length : 0), + 0 + ) + if (liveSessions > 0) { + this.admissionClosed = this.retirementAttempted + return { + state: 'busy', + liveSessions, + ...(!this.retirementAttempted ? { admissionReopened: true as const } : {}) + } + } + this.retirementAttempted = true + const results = await Promise.all(adapters.map((adapter) => adapter.requestIdleRetirement())) + if (results.every((result) => result.state === 'retiring')) { + this.retired = true + return { state: 'retiring' } + } + const refusedLiveSessions = results.reduce( + (count, result) => count + (result.state === 'busy' ? (result.liveSessions ?? 0) : 0), + 0 + ) + if (refusedLiveSessions > 0) { + return { state: 'busy', liveSessions: refusedLiveSessions } + } + return { state: 'unverifiable' } + } +} diff --git a/src/main/daemon/daemon-router-session-reconciliation.ts b/src/main/daemon/daemon-router-session-reconciliation.ts new file mode 100644 index 00000000000..d58487da106 --- /dev/null +++ b/src/main/daemon/daemon-router-session-reconciliation.ts @@ -0,0 +1,37 @@ +import type { DaemonPtyAdapter } from './daemon-pty-adapter' +import type { DaemonSessionOwnerResolver } from './daemon-session-owner-resolution' + +export async function reconcileDaemonRouterSessions( + adapters: readonly DaemonPtyAdapter[], + ownerResolver: DaemonSessionOwnerResolver, + validWorktreeIds: Set +): Promise<{ alive: string[]; killed: string[] }> { + const alive: string[] = [] + const killed: string[] = [] + const aliveProviders = new Map>() + for (const adapter of adapters) { + const result = await adapter.reconcileOnStartup(validWorktreeIds) + // Why: daemon startup can reconcile many restored sessions; spreading + // those arrays into push can exceed JavaScript's argument limit. + for (const id of result.alive) { + alive.push(id) + } + for (const id of result.killed) { + killed.push(id) + } + for (const id of result.alive) { + const providers = aliveProviders.get(id) ?? new Set() + providers.add(adapter) + aliveProviders.set(id, providers) + } + } + for (const id of new Set([...alive, ...killed])) { + const providers = aliveProviders.get(id) + if (providers?.size === 1) { + ownerResolver.recordRoute(id, providers.values().next().value!) + } else { + ownerResolver.forgetRoute(id) + } + } + return { alive, killed } +} diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 363e9125b26..3daeebca49a 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -685,3 +685,17 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.listProcesses).toHaveBeenCalledTimes(3) }) }) + +it('lists the in-process terminals a census must count, apart from the daemon inventory', async () => { + const current = createDaemonAdapter('current') + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + const fresh = await provider.spawn({ cols: 80, rows: 24 }) + + expect(fallback.spawn).toHaveBeenCalledOnce() + expect(await provider.fallback.listProcesses()).toEqual([ + expect.objectContaining({ id: fresh.id }) + ]) + expect(provider.getAllAdapters()).toEqual([current]) +}) diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 0a8868a8d80..95155c2e083 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -26,7 +26,8 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { private current: DaemonPtyAdapter private legacy: DaemonPtyAdapter[] - private fallback: IPtyProvider + /** Runs terminals in this process, so they die with it. */ + readonly fallback: IPtyProvider private sessionProviders = new Map() private freshSpawns: DegradedDaemonFreshSpawnRouter private ownerRecovery: DegradedDaemonOwnerRecovery diff --git a/src/main/daemon/pty-subprocess/spawn-preflight.ts b/src/main/daemon/pty-subprocess/spawn-preflight.ts index 412adbc567f..259583a0f78 100644 --- a/src/main/daemon/pty-subprocess/spawn-preflight.ts +++ b/src/main/daemon/pty-subprocess/spawn-preflight.ts @@ -12,7 +12,7 @@ import { resolveSafePtyDefaultCwd } from '../../providers/pty-default-cwd' import { TerminalAttachCanceledError } from '../daemon-errors' import { DaemonProtocolError } from '../types' -const PTY_SPAWN_HEALTH_TIMEOUT_MS = 4_000 +export const PTY_SPAWN_HEALTH_TIMEOUT_MS = 4_000 async function loadNodePty(): Promise { return import('node-pty') @@ -148,6 +148,13 @@ export async function preflightPtySpawn(args: { } } +export class PtySpawnHealthTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`PTY spawn health check timed out after ${timeoutMs}ms`) + this.name = 'PtySpawnHealthTimeoutError' + } +} + export function formatPtySpawnError(err: unknown, shellPath: string, spawnCwd: string): Error { const message = err instanceof Error ? err.message : String(err) const formatted = new DaemonProtocolError( @@ -159,7 +166,9 @@ export function formatPtySpawnError(err: unknown, shellPath: string, spawnCwd: s return formatted } -export async function runPtySpawnHealthProbe(): Promise { +export async function runPtySpawnHealthProbe( + timeoutMs = PTY_SPAWN_HEALTH_TIMEOUT_MS +): Promise { const cwd = isExistingDirectory(process.env.ORCA_USER_DATA_PATH) ? process.env.ORCA_USER_DATA_PATH : resolveSafePtyDefaultCwd() @@ -214,10 +223,8 @@ export async function runPtySpawnHealthProbe(): Promise { } } const timer = setTimeout(() => { - finish(new Error(`PTY spawn health check timed out after ${PTY_SPAWN_HEALTH_TIMEOUT_MS}ms`), { - kill: true - }) - }, PTY_SPAWN_HEALTH_TIMEOUT_MS) + finish(new PtySpawnHealthTimeoutError(timeoutMs), { kill: true }) + }, timeoutMs) exitDisposable = proc.onExit(({ exitCode }) => { if (exitCode === 0) { finish() diff --git a/src/main/durable-file-write.ts b/src/main/durable-file-write.ts index 05e849327b7..e573b4ecd7f 100644 --- a/src/main/durable-file-write.ts +++ b/src/main/durable-file-write.ts @@ -29,7 +29,8 @@ async function syncDirectory(directory: string): Promise { } } -function syncDirectorySync(directory: string): void { +/** Sync variant of the best-effort directory fsync, for callers that publish by rename or link. */ +export function syncDirectoryDurablySync(directory: string): void { let fd: number | null = null try { fd = openSync(directory, 'r') @@ -50,7 +51,7 @@ function syncDirectorySync(directory: string): void { /** Rename an already-fsynced file and make the containing directory durable. */ export function renameDurableSync(tmpPath: string, finalPath: string): void { renameFileWithWindowsRetry(tmpPath, finalPath) - syncDirectorySync(dirname(finalPath)) + syncDirectoryDurablySync(dirname(finalPath)) } /** Publish an already-fsynced file without replacing a concurrently created destination. */ @@ -58,7 +59,7 @@ export function publishFileDurableSync(tmpPath: string, finalPath: string): bool if (!publishFileWithoutOverwrite(tmpPath, finalPath)) { return false } - syncDirectorySync(dirname(finalPath)) + syncDirectoryDurablySync(dirname(finalPath)) rmSync(tmpPath) return true } @@ -209,12 +210,14 @@ export async function removeStaleDurableWriteTempFiles( export function writeFileDurableSync( tmpPath: string, finalPath: string, - payload: string | Uint8Array + payload: string | Uint8Array, + /** Creation mode for a new file, e.g. 0o600 for state other users must not read. */ + mode?: number ): void { let renamed = false try { // A Uint8Array payload is written verbatim; a string still defaults to UTF-8. - writeFileSync(tmpPath, payload) + writeFileSync(tmpPath, payload, mode === undefined ? undefined : { mode }) const fd = openSync(tmpPath, 'r+') try { fsyncSync(fd) diff --git a/src/main/ipc/filesystem-import-ssh-remote-existence.ts b/src/main/ipc/filesystem-import-ssh-remote-existence.ts new file mode 100644 index 00000000000..e853f291f80 --- /dev/null +++ b/src/main/ipc/filesystem-import-ssh-remote-existence.ts @@ -0,0 +1,29 @@ +import type { IFilesystemProvider } from '../providers/types' + +/** Whether a remote path exists; only a definite "missing" answer reads as false. */ +export async function remotePathExists( + provider: IFilesystemProvider, + remotePath: string +): Promise { + try { + await provider.stat(remotePath) + return true + } catch (error) { + if (isRemoteMissingError(error)) { + return false + } + throw error + } +} + +function isRemoteMissingError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false + } + return ( + ('code' in error && error.code === 'ENOENT') || + /\b(ENOENT|ENOTDIR)\b|no such file or directory|cannot find (?:the )?(?:file|path)|(?:file|path) not found/i.test( + error.message + ) + ) +} diff --git a/src/main/ipc/filesystem-import-ssh.ts b/src/main/ipc/filesystem-import-ssh.ts index a0cb300ba21..ad8025d82fd 100644 --- a/src/main/ipc/filesystem-import-ssh.ts +++ b/src/main/ipc/filesystem-import-ssh.ts @@ -7,6 +7,7 @@ import type { FileUploadSession, IFilesystemProvider } from '../providers/types' import type { ImportItemResult } from '../../shared/filesystem-import-result-types' import { assertSafeRemotePathSegment, type RemotePathFlavor } from '../ssh/ssh-remote-platform' import { isWindowsAbsolutePathLike } from '../../shared/cross-platform-path' +import { remotePathExists } from './filesystem-import-ssh-remote-existence' import { captureLocalUploadRoot, preScanSshImportDirectory, @@ -266,31 +267,3 @@ async function ensureDropStagingDir( assertCurrent?.() await provider.createDir(destDir) } - -async function remotePathExists( - provider: IFilesystemProvider, - remotePath: string -): Promise { - try { - await provider.stat(remotePath) - return true - } catch (error) { - if (isRemoteMissingError(error)) { - return false - } - throw error - } -} - -function isRemoteMissingError(error: unknown): boolean { - if (!(error instanceof Error)) { - return false - } - const code = (error as NodeJS.ErrnoException).code - return ( - code === 'ENOENT' || - /\b(ENOENT|ENOTDIR)\b|no such file or directory|cannot find (?:the )?(?:file|path)|(?:file|path) not found/i.test( - error.message - ) - ) -} diff --git a/src/main/ipc/filesystem/filesystem-write-handlers.ts b/src/main/ipc/filesystem/filesystem-write-handlers.ts index ff614b8d35a..de02a93e7ac 100644 --- a/src/main/ipc/filesystem/filesystem-write-handlers.ts +++ b/src/main/ipc/filesystem/filesystem-write-handlers.ts @@ -35,8 +35,10 @@ export function registerFilesystemWriteHandlers(context: FilesystemHandlerContex args.expectedExecutionHostId ) if (args.connectionId) { - const provider = requireSshFilesystemProvider(args.connectionId) - return provider.writeFile(args.filePath, args.content) + return requireSshFilesystemProvider(args.connectionId).writeFile( + args.filePath, + args.content + ) } const filePath = await resolveLocalWriteRequestPath(args.filePath, args.access, store) try { @@ -71,8 +73,10 @@ export function registerFilesystemWriteHandlers(context: FilesystemHandlerContex args.expectedExecutionHostId ) if (args.connectionId) { - const provider = requireSshFilesystemProvider(args.connectionId) - return provider.deletePath(args.targetPath, args.recursive) + return requireSshFilesystemProvider(args.connectionId).deletePath( + args.targetPath, + args.recursive + ) } // Why: preserve the symlink so we delete the link, not its target (realpath would trash the real file, possibly outside all roots). const targetPath = await resolveDesktopAuthorizedPath(args.targetPath, store, { diff --git a/src/main/ipc/managed-orcad-actions.ts b/src/main/ipc/managed-orcad-actions.ts new file mode 100644 index 00000000000..7c610b23d5a --- /dev/null +++ b/src/main/ipc/managed-orcad-actions.ts @@ -0,0 +1,77 @@ +/** The managed-server actions behind both the Managed servers settings and runtime RPC. */ +import { + cancelManagedOrcadStop, + getManagedOrcadRuntimeStatus, + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + stopManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} from '../ssh/orcad-runtime-lifecycle' +import type { ManagedServerActions } from '../runtime/managed-server-actions-registry' +import type { ExecutionHostId } from '../../shared/execution-host' +import { retireRemovedRuntimeEnvironment } from './runtime-environment-removal-cleanup' + +export type ManagedOrcadActionOptions = { + getUserDataPath: () => string + getActiveEnvironmentId: () => string | null | undefined + invalidateTransport: (environmentId: string) => Promise | void + /** Drops the SSH host's stale managed-server state once its server is unlinked. */ + clearHostServerStatus: (sshTargetId: string) => void + /** Drops the unlinked server's workspace session partition. */ + forgetHostSession: (hostId: ExecutionHostId) => void +} + +export function createManagedOrcadActions( + options: ManagedOrcadActionOptions +): ManagedServerActions { + const userDataPath = options.getUserDataPath + return { + status: (selector) => getManagedOrcadRuntimeStatus(userDataPath(), selector), + update: async (selector, force) => { + const result = await updateManagedOrcadEnvironment(userDataPath(), { selector, force }) + // Why: a restarted orcad drops the old connection; reconnect on the new one. + if (result.outcome === 'updated') { + await options.invalidateTransport(result.environment.id) + } + return result + }, + rollback: async (selector) => { + const result = await rollbackManagedOrcadEnvironment(userDataPath(), { selector }) + if (result.outcome === 'rolled-back') { + await options.invalidateTransport(result.environment.id) + } + return result + }, + recover: async (selector, acceptChangedState) => { + const result = await recoverManagedOrcadEnvironment(userDataPath(), { + selector, + acceptChangedState: acceptChangedState === true + }) + if (result.outcome === 'recovered' && result.activeVersion) { + await options.invalidateTransport(result.environment.id) + } + return result + }, + stop: async (selector) => { + const result = await stopManagedOrcadEnvironment( + userDataPath(), + { selector }, + { + isActiveEnvironment: (environmentId) => + options.getActiveEnvironmentId() === environmentId, + retireLocalState: (environmentId) => + retireRemovedRuntimeEnvironment( + environmentId, + options.invalidateTransport, + options.forgetHostSession + ) + } + ) + if (result.outcome === 'unlinked') { + options.clearHostServerStatus(result.sshTargetId) + } + return result + }, + cancelStop: (selector) => cancelManagedOrcadStop(userDataPath(), { selector }) + } +} diff --git a/src/main/ipc/orcad-delta-move-handlers.test.ts b/src/main/ipc/orcad-delta-move-handlers.test.ts new file mode 100644 index 00000000000..144b040db55 --- /dev/null +++ b/src/main/ipc/orcad-delta-move-handlers.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + move: vi.fn(), + keep: vi.fn(async () => {}), + publish: vi.fn(), + target: { + id: 'ssh-1', + label: 'Box', + host: 'box', + port: 22, + username: 'me', + orcadFence: { environmentId: 'env-1', sourceChangedAt: '2026-10-04T00:00:00.000Z' } + } +})) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../../shared/runtime-environment-store', () => ({ + listEnvironments: () => [{ id: 'env-1' }] +})) +vi.mock('../ssh/orcad-managed-runtime-context', () => ({ + requireManagedOrcadInfrastructure: () => ({ + claims: {}, + targetStore: { getOrcadMigrationSource: () => ({ getSshTarget: () => mocks.target }) } + }) +})) +vi.mock('../ssh/orcad-migration-delta-move', () => ({ + runOrcadDeltaMove: mocks.move, + keepOrcadServerVersion: mocks.keep +})) +vi.mock('../ssh/orcad-runtime-conversion-wiring', () => ({ + orcadMigrationDestinationFor: () => ({}) +})) +vi.mock('../ssh/orcad-migration-relay-pty-lister', () => ({ + orcadMigrationRelayPtyLister: () => null +})) +vi.mock('../ssh/orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: vi.fn() })) +vi.mock('../ssh/ssh-target-registry', () => ({ hasRegisteredDirectSshAuthority: () => false })) +vi.mock('./ssh-session-teardown', () => ({ disconnectRegisteredSshTarget: vi.fn() })) +vi.mock('./runtime-environment-managed-tunnel', () => ({ + publishResolvedChangedHostStatus: mocks.publish +})) + +const { registerOrcadDeltaMoveHandlers } = await import('./orcad-delta-move-handlers') + +function handler(channel: string): (_event: unknown, args: unknown) => Promise { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +describe('resolving a host an older build changed', () => { + beforeEach(() => { + vi.clearAllMocks() + registerOrcadDeltaMoveHandlers(() => '/tmp/user-data') + }) + + it('refreshes the host status once the move lands, and not when it is refused', async () => { + mocks.move.mockResolvedValueOnce({ outcome: 'refused', code: 'x', reason: 'y' }) + await handler('runtimeEnvironments:moveOrcadDelta')(null, { sshTargetId: 'ssh-1' }) + expect(mocks.publish).not.toHaveBeenCalled() + mocks.move.mockResolvedValueOnce({ outcome: 'moved', migrationId: 'm' }) + await handler('runtimeEnvironments:moveOrcadDelta')(null, { sshTargetId: 'ssh-1' }) + expect(mocks.publish).toHaveBeenCalledWith(mocks.target, 'env-1') + }) + + it('refreshes the host status after keeping the server version', async () => { + await handler('runtimeEnvironments:keepOrcadServerVersion')(null, { sshTargetId: 'ssh-1' }) + expect(mocks.keep).toHaveBeenCalled() + expect(mocks.publish).toHaveBeenCalledWith(mocks.target, 'env-1') + }) +}) diff --git a/src/main/ipc/orcad-delta-move-handlers.ts b/src/main/ipc/orcad-delta-move-handlers.ts new file mode 100644 index 00000000000..8fdffcf8870 --- /dev/null +++ b/src/main/ipc/orcad-delta-move-handlers.ts @@ -0,0 +1,88 @@ +/** IPC for a host an older build changed after conversion: move its newer projects, or keep the server's. */ +import { ipcMain } from 'electron' +import type { + OrcadDeltaMovePreview, + OrcadDeltaMoveResult +} from '../../shared/orcad-managed-runtime' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { requireManagedOrcadInfrastructure } from '../ssh/orcad-managed-runtime-context' +import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { keepOrcadServerVersion, runOrcadDeltaMove } from '../ssh/orcad-migration-delta-move' +import { planOrcadDeltaMove } from '../ssh/orcad-migration-delta-plan' +import { orcadMigrationRelayPtyLister } from '../ssh/orcad-migration-relay-pty-lister' +import { censusHostRelayTerminalsFor } from '../ssh/ssh-host-relay-census-for-target' +import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wiring' +import { hasRegisteredDirectSshAuthority } from '../ssh/ssh-target-registry' +import { requiredString } from './orcad-runtime-lifecycle-handlers' +import { disconnectRegisteredSshTarget } from './ssh-session-teardown' +import { publishResolvedChangedHostStatus } from './runtime-environment-managed-tunnel' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' + +export function registerOrcadDeltaMoveHandlers(getUserDataPath: () => string): void { + ipcMain.handle( + 'runtimeEnvironments:previewOrcadDeltaMove', + (_event, args: { sshTargetId: string }): OrcadDeltaMovePreview => { + const { store, target } = requireChangedHost(args) + const plan = planOrcadDeltaMove(getUserDataPath(), store, target) + const { sshTargetId, environmentId, added, notReflected, blockers } = plan + return { sshTargetId, environmentId, added, notReflected, blockers } + } + ) + ipcMain.handle( + 'runtimeEnvironments:moveOrcadDelta', + async (_event, args: { sshTargetId: string }): Promise => { + const userDataPath = getUserDataPath() + const { store, claims, target } = requireChangedHost(args) + const environment = listEnvironments(userDataPath).find( + (entry) => entry.id === target.orcadFence?.environmentId + ) + if (!environment) { + throw new Error('The managed Orca server for this host is no longer registered.') + } + const result = await runOrcadDeltaMove({ + userDataPath, + store, + claims, + target, + environment, + destination: orcadMigrationDestinationFor(environment), + listRelayPtyIds: orcadMigrationRelayPtyLister(target.id), + censusHost: censusHostRelayTerminalsFor(target), + releaseDirectSession: async (targetId) => { + if (hasRegisteredDirectSshAuthority(targetId)) { + await disconnectRegisteredSshTarget(targetId) + } + }, + ensureTunnel: async () => { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + }, + runTargetLifecycle + }) + if (result.outcome === 'moved') { + publishResolvedChangedHostStatus(target, environment.id) + } + return result + } + ) + ipcMain.handle( + 'runtimeEnvironments:keepOrcadServerVersion', + async (_event, args: { sshTargetId: string }): Promise => { + const { store, claims, target, environmentId } = requireChangedHost(args) + await runTargetLifecycle(target.id, () => + keepOrcadServerVersion({ userDataPath: getUserDataPath(), store, claims, target }) + ) + publishResolvedChangedHostStatus(target, environmentId) + } + ) +} + +function requireChangedHost(args: { sshTargetId: string } | undefined) { + const sshTargetId = requiredString(args?.sshTargetId, 'SSH target') + const { targetStore, claims } = requireManagedOrcadInfrastructure() + const store = targetStore.getOrcadMigrationSource() + const target = store.getSshTarget(sshTargetId) + if (!target?.orcadFence?.sourceChangedAt) { + throw new Error('This SSH host has no changes from an older Orca to resolve.') + } + return { store, claims, target, environmentId: target.orcadFence.environmentId } +} diff --git a/src/main/ipc/orcad-runtime-conversion-handlers.test.ts b/src/main/ipc/orcad-runtime-conversion-handlers.test.ts new file mode 100644 index 00000000000..5f570e5c586 --- /dev/null +++ b/src/main/ipc/orcad-runtime-conversion-handlers.test.ts @@ -0,0 +1,51 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + convert: vi.fn(), + pending: vi.fn() +})) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/orcad-runtime-conversion', () => ({ + convertSshTargetToManagedOrcad: mocks.convert +})) +vi.mock('../ssh/orcad-managed-migration-status', () => ({ + listPendingManagedOrcadMigrations: mocks.pending +})) +vi.mock('../ssh/orcad-runtime-conversion-wiring', () => ({ + conversionCollaborators: () => ({ marker: 'live-collaborators' }) +})) + +const { registerOrcadRuntimeConversionHandlers } = + await import('./orcad-runtime-conversion-handlers') + +function handler(channel: string): (_event: unknown, args?: unknown) => unknown { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +describe('managed server conversion IPC', () => { + beforeEach(() => { + vi.clearAllMocks() + registerOrcadRuntimeConversionHandlers(() => '/profile') + }) + + it('converts with the live collaborators and lists pending migrations from the profile', async () => { + mocks.convert.mockResolvedValue({ outcome: 'converted' }) + await handler('runtimeEnvironments:convertSshHostToManagedOrcad')(null, { + sshTargetId: ' ssh-1 ', + name: 'Builder' + }) + expect(mocks.convert).toHaveBeenCalledWith('/profile', { + sshTargetId: 'ssh-1', + name: 'Builder', + marker: 'live-collaborators' + }) + handler('runtimeEnvironments:listPendingOrcadMigrations')(null) + expect(mocks.pending).toHaveBeenCalledWith('/profile') + }) +}) diff --git a/src/main/ipc/orcad-runtime-conversion-handlers.ts b/src/main/ipc/orcad-runtime-conversion-handlers.ts new file mode 100644 index 00000000000..fa2591f379e --- /dev/null +++ b/src/main/ipc/orcad-runtime-conversion-handlers.ts @@ -0,0 +1,30 @@ +import { ipcMain } from 'electron' +import type { + OrcadManagedConversionResult, + OrcadManagedPendingMigrationRow +} from '../../shared/orcad-managed-runtime' +import { listPendingManagedOrcadMigrations } from '../ssh/orcad-managed-migration-status' +import { convertSshTargetToManagedOrcad } from '../ssh/orcad-runtime-conversion' +import { conversionCollaborators } from '../ssh/orcad-runtime-conversion-wiring' +import { requiredString } from './orcad-runtime-lifecycle-handlers' + +export function registerOrcadRuntimeConversionHandlers(getUserDataPath: () => string): void { + ipcMain.handle( + 'runtimeEnvironments:convertSshHostToManagedOrcad', + ( + _event, + args: { sshTargetId: string; name: string } + ): Promise => { + const sshTargetId = requiredString(args?.sshTargetId, 'SSH target') + return convertSshTargetToManagedOrcad(getUserDataPath(), { + sshTargetId, + name: requiredString(args?.name, 'Server name'), + ...conversionCollaborators(sshTargetId) + }) + } + ) + ipcMain.handle( + 'runtimeEnvironments:listPendingOrcadMigrations', + (): OrcadManagedPendingMigrationRow[] => listPendingManagedOrcadMigrations(getUserDataPath()) + ) +} diff --git a/src/main/ipc/orcad-runtime-lifecycle-handlers.test.ts b/src/main/ipc/orcad-runtime-lifecycle-handlers.test.ts new file mode 100644 index 00000000000..cc15058c7f8 --- /dev/null +++ b/src/main/ipc/orcad-runtime-lifecycle-handlers.test.ts @@ -0,0 +1,66 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + deploy: vi.fn(), + status: vi.fn(), + registerProvisioning: vi.fn() +})) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/orcad-runtime-lifecycle', () => ({ + createManagedOrcadEnvironment: mocks.deploy, + getManagedOrcadRuntimeStatus: mocks.status +})) +vi.mock('./orcad-ssh-provisioning-handlers', () => ({ + registerOrcadSshProvisioningHandlers: mocks.registerProvisioning +})) + +const { registerOrcadRuntimeLifecycleHandlers } = await import('./orcad-runtime-lifecycle-handlers') + +function handler(channel: string): (_event: unknown, args: unknown) => unknown { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +describe('managed orcad lifecycle IPC', () => { + beforeEach(() => { + vi.clearAllMocks() + registerOrcadRuntimeLifecycleHandlers({ getUserDataPath: () => '/profile' }) + }) + + it('registers only deploy, status and provisioning; maintenance and stop are not exposed', () => { + expect(mocks.handle.mock.calls.map(([channel]) => channel)).toEqual([ + 'runtimeEnvironments:deployOrcad', + 'runtimeEnvironments:getOrcadStatus' + ]) + expect(mocks.registerProvisioning).toHaveBeenCalledOnce() + }) + + it('trims deploy input and treats only a literal true as force', async () => { + await handler('runtimeEnvironments:deployOrcad')(null, { + name: ' Managed ', + sshTargetId: ' ssh-1 ', + force: 'yes' + }) + expect(mocks.deploy).toHaveBeenCalledWith('/profile', { + name: 'Managed', + sshTargetId: 'ssh-1', + force: false + }) + }) + + it('rejects missing selectors before touching SSH', async () => { + await expect(handler('runtimeEnvironments:deployOrcad')(null, { name: 'x' })).rejects.toThrow( + 'SSH target is required' + ) + expect(() => handler('runtimeEnvironments:getOrcadStatus')(null, undefined)).toThrow( + 'Server is required' + ) + expect(mocks.deploy).not.toHaveBeenCalled() + expect(mocks.status).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/orcad-runtime-lifecycle-handlers.ts b/src/main/ipc/orcad-runtime-lifecycle-handlers.ts new file mode 100644 index 00000000000..4d6e23833b1 --- /dev/null +++ b/src/main/ipc/orcad-runtime-lifecycle-handlers.ts @@ -0,0 +1,37 @@ +import { ipcMain } from 'electron' +import type { OrcadManagedRuntimeStatus } from '../../shared/orcad-managed-runtime' +import { + createManagedOrcadEnvironment, + getManagedOrcadRuntimeStatus +} from '../ssh/orcad-runtime-lifecycle' +import { registerOrcadSshProvisioningHandlers } from './orcad-ssh-provisioning-handlers' + +export function registerOrcadRuntimeLifecycleHandlers(options: { + getUserDataPath: () => string +}): void { + registerOrcadSshProvisioningHandlers(options.getUserDataPath) + ipcMain.handle( + 'runtimeEnvironments:deployOrcad', + async (_event, args: { name: string; sshTargetId: string; force?: boolean }) => + createManagedOrcadEnvironment(options.getUserDataPath(), { + name: requiredString(args?.name, 'Server name'), + sshTargetId: requiredString(args?.sshTargetId, 'SSH target'), + force: args?.force === true + }) + ) + ipcMain.handle( + 'runtimeEnvironments:getOrcadStatus', + (_event, args: { selector: string }): Promise => + getManagedOrcadRuntimeStatus( + options.getUserDataPath(), + requiredString(args?.selector, 'Server') + ) + ) +} + +export function requiredString(value: unknown, label: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`${label} is required.`) + } + return value.trim() +} diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts new file mode 100644 index 00000000000..340517c0ded --- /dev/null +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.test.ts @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + update: vi.fn(), + rollback: vi.fn(), + recover: vi.fn(), + stop: vi.fn(), + cancel: vi.fn(), + retire: vi.fn() +})) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/orcad-runtime-lifecycle', () => ({ + updateManagedOrcadEnvironment: mocks.update, + rollbackManagedOrcadEnvironment: mocks.rollback, + recoverManagedOrcadEnvironment: mocks.recover, + stopManagedOrcadEnvironment: mocks.stop, + cancelManagedOrcadStop: mocks.cancel +})) +vi.mock('./runtime-environment-removal-cleanup', () => ({ + retireRemovedRuntimeEnvironment: mocks.retire +})) + +const { registerOrcadRuntimeMaintenanceHandlers } = + await import('./orcad-runtime-maintenance-handlers') + +const invalidateTransport = vi.fn() +const clearHostServerStatus = vi.fn() +const forgetHostSession = vi.fn() + +function handler(channel: string): (_event: unknown, args: unknown) => Promise { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +describe('managed orcad maintenance IPC', () => { + beforeEach(() => { + vi.clearAllMocks() + registerOrcadRuntimeMaintenanceHandlers({ + getUserDataPath: () => '/profile', + getActiveEnvironmentId: () => 'active-environment', + invalidateTransport, + clearHostServerStatus, + forgetHostSession + }) + }) + + it('reconnects after an update restarts orcad, but not after a deferral', async () => { + mocks.update.mockResolvedValueOnce({ outcome: 'deferred', code: 'busy' }) + await handler('runtimeEnvironments:updateOrcad')(null, { selector: 'Managed', force: 'yes' }) + expect(mocks.update).toHaveBeenCalledWith('/profile', { selector: 'Managed', force: false }) + expect(invalidateTransport).not.toHaveBeenCalled() + mocks.update.mockResolvedValueOnce({ outcome: 'updated', environment: { id: 'e-1' } }) + await handler('runtimeEnvironments:updateOrcad')(null, { selector: 'Managed', force: true }) + expect(invalidateTransport).toHaveBeenCalledWith('e-1') + }) + + it('reconnects after rollback and after recovery restores a serving slot', async () => { + mocks.rollback.mockResolvedValueOnce({ outcome: 'rolled-back', environment: { id: 'e-1' } }) + await handler('runtimeEnvironments:rollbackOrcad')(null, { selector: 'Managed' }) + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + activeVersion: null, + environment: { id: 'e-1' } + }) + await handler('runtimeEnvironments:recoverOrcad')(null, { selector: 'Managed' }) + expect(invalidateTransport).toHaveBeenCalledTimes(1) + }) + + it('stops with the Active Server guard and the shared removal cleanup', async () => { + mocks.stop.mockResolvedValueOnce({ outcome: 'unlinked', sshTargetId: 'ssh-1' }) + await handler('runtimeEnvironments:stopOrcad')(null, { selector: ' Managed ' }) + const [, args, policy] = mocks.stop.mock.calls[0] ?? [] + expect(args).toEqual({ selector: 'Managed' }) + expect(policy.isActiveEnvironment('active-environment')).toBe(true) + expect(policy.isActiveEnvironment('e-1')).toBe(false) + policy.retireLocalState('e-1') + expect(mocks.retire).toHaveBeenCalledWith('e-1', invalidateTransport, forgetHostSession) + // The SSH host stops naming the unlinked server without waiting for a reconnect. + expect(clearHostServerStatus).toHaveBeenCalledWith('ssh-1') + }) + + it('keeps the SSH host’s managed state when the stop is refused', async () => { + mocks.stop.mockResolvedValueOnce({ + outcome: 'refused', + verdict: 'live', + code: 'c', + reason: 'r' + }) + await handler('runtimeEnvironments:stopOrcad')(null, { selector: 'Managed' }) + expect(clearHostServerStatus).not.toHaveBeenCalled() + }) + + it('rejects a missing selector before touching SSH', async () => { + await expect(handler('runtimeEnvironments:cancelOrcadStop')(null, {})).rejects.toThrow( + 'Server is required' + ) + expect(mocks.cancel).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/orcad-runtime-maintenance-handlers.ts b/src/main/ipc/orcad-runtime-maintenance-handlers.ts new file mode 100644 index 00000000000..7c3ff0aa14f --- /dev/null +++ b/src/main/ipc/orcad-runtime-maintenance-handlers.ts @@ -0,0 +1,31 @@ +import { ipcMain } from 'electron' +import { registerManagedServerActions } from '../runtime/managed-server-actions-registry' +import { createManagedOrcadActions, type ManagedOrcadActionOptions } from './managed-orcad-actions' +import { requiredString } from './orcad-runtime-lifecycle-handlers' + +export function registerOrcadRuntimeMaintenanceHandlers(options: ManagedOrcadActionOptions): void { + const actions = createManagedOrcadActions(options) + // Why: the CLI reaches these same actions over runtime RPC (managedServer.*). + registerManagedServerActions(actions) + ipcMain.handle( + 'runtimeEnvironments:updateOrcad', + async (_event, args: { selector: string; force?: boolean }) => + actions.update(requiredString(args?.selector, 'Server'), args?.force === true) + ) + ipcMain.handle('runtimeEnvironments:rollbackOrcad', async (_event, args: { selector: string }) => + actions.rollback(requiredString(args?.selector, 'Server')) + ) + ipcMain.handle( + 'runtimeEnvironments:recoverOrcad', + async (_event, args: { selector: string; acceptChangedState?: boolean }) => + actions.recover(requiredString(args?.selector, 'Server'), args?.acceptChangedState === true) + ) + ipcMain.handle('runtimeEnvironments:stopOrcad', async (_event, args: { selector: string }) => + actions.stop(requiredString(args?.selector, 'Server')) + ) + ipcMain.handle( + 'runtimeEnvironments:cancelOrcadStop', + async (_event, args: { selector: string }) => + actions.cancelStop(requiredString(args?.selector, 'Server')) + ) +} diff --git a/src/main/ipc/orcad-ssh-provisioning-handlers.test.ts b/src/main/ipc/orcad-ssh-provisioning-handlers.test.ts new file mode 100644 index 00000000000..6d5fa72da34 --- /dev/null +++ b/src/main/ipc/orcad-ssh-provisioning-handlers.test.ts @@ -0,0 +1,40 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + handle: vi.fn(), + create: vi.fn(), + resume: vi.fn(), + list: vi.fn() +})) +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/orcad-ssh-provisioning', () => ({ + createOrcadSshHost: mocks.create, + resumeOrcadSshHost: mocks.resume, + listPendingOrcadSshProvisioning: mocks.list +})) +import { registerOrcadSshProvisioningHandlers } from './orcad-ssh-provisioning-handlers' + +describe('managed SSH provisioning IPC', () => { + beforeEach(() => vi.clearAllMocks()) + + it('registers typed create, resume and pending discovery without changing legacy SSH channels', async () => { + registerOrcadSshProvisioningHandlers(() => '/active-profile') + const handlers = new Map(mocks.handle.mock.calls.map(([name, handler]) => [name, handler])) + expect([...handlers.keys()]).toEqual([ + 'runtimeEnvironments:createOrcadSshHost', + 'runtimeEnvironments:resumeOrcadSshHost', + 'runtimeEnvironments:listPendingOrcadSshProvisioning' + ]) + const request = { requestId: 'request-1', name: 'host', target: { host: 'builder' } } + const pending = { result: { outcome: 'pending', reason: 'unverifiable' } } + mocks.create.mockResolvedValue(pending) + expect(await handlers.get('runtimeEnvironments:createOrcadSshHost')!(null, request)).toBe( + pending + ) + expect(mocks.create).toHaveBeenCalledWith('/active-profile', request) + await handlers.get('runtimeEnvironments:resumeOrcadSshHost')!(null, { requestId: 'request-1' }) + expect(mocks.resume).toHaveBeenCalledWith('/active-profile', 'request-1') + handlers.get('runtimeEnvironments:listPendingOrcadSshProvisioning')!() + expect(mocks.list).toHaveBeenCalledWith('/active-profile') + }) +}) diff --git a/src/main/ipc/orcad-ssh-provisioning-handlers.ts b/src/main/ipc/orcad-ssh-provisioning-handlers.ts new file mode 100644 index 00000000000..31cdaeed6c2 --- /dev/null +++ b/src/main/ipc/orcad-ssh-provisioning-handlers.ts @@ -0,0 +1,20 @@ +import { ipcMain } from 'electron' +import type { OrcadSshProvisioningRequest } from '../../shared/orcad-ssh-provisioning' +import { + createOrcadSshHost, + listPendingOrcadSshProvisioning, + resumeOrcadSshHost +} from '../ssh/orcad-ssh-provisioning' + +export function registerOrcadSshProvisioningHandlers(getUserDataPath: () => string): void { + ipcMain.handle( + 'runtimeEnvironments:createOrcadSshHost', + (_event, args: OrcadSshProvisioningRequest) => createOrcadSshHost(getUserDataPath(), args) + ) + ipcMain.handle('runtimeEnvironments:resumeOrcadSshHost', (_event, args: { requestId: string }) => + resumeOrcadSshHost(getUserDataPath(), args?.requestId) + ) + ipcMain.handle('runtimeEnvironments:listPendingOrcadSshProvisioning', () => + listPendingOrcadSshProvisioning(getUserDataPath()) + ) +} diff --git a/src/main/ipc/parcel-watcher-child-slot.ts b/src/main/ipc/parcel-watcher-child-slot.ts new file mode 100644 index 00000000000..c26e4ba32e6 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-slot.ts @@ -0,0 +1,20 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory' + +/** The supervisor's one watcher child: the live one, the one being terminated, and its canary. */ +export class WatcherChildSlot { + child: ChildProcessHandle | null = null + terminating: ChildProcessHandle | null = null + canaryDir: string | null = null + + /** The canary belongs to the child it was launched with, so it goes when that child does. */ + removeCanary(): void { + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + } + + /** Marks `proc` as the child being terminated; a later launch waits until it is settled. */ + beginTermination(proc: ChildProcessHandle): void { + this.terminating = proc + this.removeCanary() + } +} diff --git a/src/main/ipc/parcel-watcher-child-termination.ts b/src/main/ipc/parcel-watcher-child-termination.ts index fc498410438..e7067dc32d6 100644 --- a/src/main/ipc/parcel-watcher-child-termination.ts +++ b/src/main/ipc/parcel-watcher-child-termination.ts @@ -10,6 +10,13 @@ export const WATCHER_PROCESS_HARD_KILL_DELAY_MS = 5_000 export const WATCHER_PROCESS_EXIT_DEADLINE_MS = RUNTIME_FILE_WATCH_EXIT_DEADLINE_MS const physicalExitPromises = new WeakMap>() +const signalledChildren = new WeakSet() + +/** Sends the graceful signal once; a later awaited termination only escalates and waits. */ +export function signalWatcherChild(child: ChildProcess): void { + signalledChildren.add(child) + child.kill() +} export function registerWatcherChildPhysicalExit(child: ChildProcess): () => void { let resolveExit: () => void = () => undefined @@ -86,6 +93,10 @@ export function terminateWatcherChild(child: ChildProcess): Promise { hardKillTimer.unref?.() const exitDeadlineTimer = setTimeout(() => finish(false), WATCHER_PROCESS_EXIT_DEADLINE_MS) exitDeadlineTimer.unref?.() + if (signalledChildren.has(child)) { + return + } + signalledChildren.add(child) try { child.kill() } catch { @@ -94,11 +105,10 @@ export function terminateWatcherChild(child: ChildProcess): Promise { }) } -export function createWatcherChildTerminationFailure(child: ChildProcess): WatcherProcessFailure { - const physicalExit = - child.exitCode !== null || child.signalCode !== null - ? Promise.resolve() - : (physicalExitPromises.get(child) ?? +export function watcherChildPhysicalExit(child: ChildProcess): Promise { + return child.exitCode !== null || child.signalCode !== null + ? Promise.resolve() + : (physicalExitPromises.get(child) ?? new Promise((resolve) => { const finish = (): void => { child.removeListener('exit', finish) @@ -108,11 +118,14 @@ export function createWatcherChildTerminationFailure(child: ChildProcess): Watch child.once('exit', finish) child.once('close', finish) })) +} + +export function createWatcherChildTerminationFailure(child: ChildProcess): WatcherProcessFailure { return new WatcherProcessFailure( 'file watcher process did not exit after termination deadline', 'supervisor', 'process_unavailable', - physicalExit + watcherChildPhysicalExit(child) ) } @@ -127,10 +140,12 @@ export async function terminateIdleWatcherChild( pendingUnsubscribes: Map, onFinished: (exited: boolean) => void ): Promise { + // Windows directory handles require physical exit, not merely an accepted signal. try { await requireWatcherChildTermination(child) onFinished(true) } catch (error) { + // Idle children retain capacity but cannot double-watch; the owner may remain reusable. onFinished(false) resolvePendingWatcherUnsubscribes( pendingUnsubscribes, diff --git a/src/main/ipc/parcel-watcher-entry-path.ts b/src/main/ipc/parcel-watcher-entry-path.ts index 229dcdc65e2..4633739fe15 100644 --- a/src/main/ipc/parcel-watcher-entry-path.ts +++ b/src/main/ipc/parcel-watcher-entry-path.ts @@ -4,6 +4,14 @@ import { join } from 'node:path' type ElectronAppPath = { getAppPath(): string; isPackaged(): boolean } +export function watcherProcessEntryExists(entryPath: string): boolean { + if (existsSync(entryPath)) { + return true + } + console.error(`[parcel-watcher-process] entry not found at ${entryPath}; refusing fail-open`) + return false +} + // Why the port and not require('electron'): this module is reachable from plain-Node // fork entries, where the literal text require("electron") fails the build guard even // inside a try/catch. hasAppEnvironment() gives the same "no app root here" answer. diff --git a/src/main/ipc/parcel-watcher-owned-children.test.ts b/src/main/ipc/parcel-watcher-owned-children.test.ts new file mode 100644 index 00000000000..cbb369cf3e2 --- /dev/null +++ b/src/main/ipc/parcel-watcher-owned-children.test.ts @@ -0,0 +1,120 @@ +import { EventEmitter } from 'node:events' +import type { ChildProcess } from 'node:child_process' +import { afterEach, expect, it, vi } from 'vitest' +import { WatcherOwnedChildren } from './parcel-watcher-owned-children' +import { + registerWatcherChildPhysicalExit, + signalWatcherChild, + WATCHER_PROCESS_EXIT_DEADLINE_MS, + WATCHER_PROCESS_HARD_KILL_DELAY_MS +} from './parcel-watcher-child-termination' + +afterEach(() => vi.useRealTimers()) + +function child() { + const exitState: { exitCode: number | null; signalCode: NodeJS.Signals | null } = { + exitCode: null, + signalCode: null + } + const events = Object.assign(new EventEmitter(), exitState, { kill: vi.fn(() => true) }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: termination reads only the exit fields, kill and events stubbed here. + const process = events as unknown as ChildProcess + const physicalExit = registerWatcherChildPhysicalExit(process) + events.on('exit', physicalExit) + events.on('close', physicalExit) + events.on('error', () => {}) + return { process, events, close: () => events.emit('close') } +} + +it('joins disposal and does not confuse disconnect/error with physical exit', async () => { + const owner = new WatcherOwnedChildren() + const c = child() + owner.track(c.process) + const logical = vi.fn() + const disposed = vi.fn() + const first = owner.disposeAndWait(logical) + expect(owner.disposeAndWait(logical)).toBe(first) + const result = first.then(disposed) + c.events.emit('disconnect') + c.events.emit('error', new Error('spawn or IPC failure')) + await Promise.resolve() + expect(disposed).not.toHaveBeenCalled() + expect(logical).toHaveBeenCalledOnce() + c.close() + await result + expect(disposed).toHaveBeenCalledOnce() +}) + +it('includes children already signaled by synchronous or retired-owner disposal', async () => { + const owner = new WatcherOwnedChildren() + const old = child() + const replacement = child() + owner.track(old.process) + old.process.kill() + owner.track(replacement.process) + const disposed = vi.fn() + const result = owner.disposeAndWait(() => {}).then(disposed) + replacement.close() + await Promise.resolve() + expect(disposed).not.toHaveBeenCalled() + old.close() + await result +}) + +it('retains a child after termination deadline failure and supports explicit retry', async () => { + vi.useFakeTimers() + const owner = new WatcherOwnedChildren() + const c = child() + owner.track(c.process) + const result = owner.disposeAndWait(() => {}).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_EXIT_DEADLINE_MS) + expect(await result).toMatchObject({ message: 'watcher_owned_children_shutdown_incomplete' }) + const completed = vi.fn() + const retry = owner.disposeAndWait(() => {}).then(completed) + await Promise.resolve() + expect(completed).not.toHaveBeenCalled() + c.close() + await retry +}) + +it('waits for other children even when logical disposal and one kill fail', async () => { + const owner = new WatcherOwnedChildren() + const failed = child() + const pending = child() + owner.track(failed.process) + owner.track(pending.process) + failed.events.kill.mockImplementationOnce(() => { + throw new Error('kill failed') + }) + const logicalFailure = new Error('logical cleanup failed') + const finished = vi.fn() + const result = owner + .disposeAndWait(() => { + throw logicalFailure + }) + .catch((error: unknown) => { + finished() + return error + }) + await Promise.resolve() + expect(finished).not.toHaveBeenCalled() + pending.close() + expect(await result).toMatchObject({ errors: [logicalFailure, expect.any(Error)] }) + const retry = owner.disposeAndWait(() => {}) + failed.close() + await retry +}) + +it('skips a second graceful signal after the supervisor sent one but still escalates', async () => { + vi.useFakeTimers() + const owner = new WatcherOwnedChildren() + const c = child() + owner.track(c.process) + const disposal = owner.disposeAndWait(() => signalWatcherChild(c.process)) + expect(c.events.kill).toHaveBeenCalledTimes(1) + expect(c.events.kill).toHaveBeenCalledWith() + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_HARD_KILL_DELAY_MS) + expect(c.events.kill).toHaveBeenLastCalledWith('SIGKILL') + c.events.emit('exit', null, 'SIGKILL') + await expect(disposal).resolves.toBeUndefined() +}) diff --git a/src/main/ipc/parcel-watcher-owned-children.ts b/src/main/ipc/parcel-watcher-owned-children.ts new file mode 100644 index 00000000000..7ac044ac906 --- /dev/null +++ b/src/main/ipc/parcel-watcher-owned-children.ts @@ -0,0 +1,47 @@ +import type { ChildProcessHandle } from '../../shared/child-process/process-spec' +import { + watcherChildPhysicalExit, + requireWatcherChildTermination +} from './parcel-watcher-child-termination' + +export class WatcherOwnedChildren { + private readonly children = new Set() + private disposal: Promise | null = null + + track(child: ChildProcessHandle): ChildProcessHandle { + this.children.add(child) + // Reuse launch-owned physical-exit evidence, including close without exit after spawn failure. + void watcherChildPhysicalExit(child).then(() => { + this.children.delete(child) + }) + return child + } + + disposeAndWait(disposeLogicalOwner: () => void): Promise { + if (this.disposal) { + return this.disposal + } + const failures: unknown[] = [] + try { + disposeLogicalOwner() + } catch (error) { + failures.push(error) + } + const cleanup = Promise.allSettled([...this.children].map(requireWatcherChildTermination)) + .then((results) => { + for (const result of results) { + if (result.status === 'rejected') { + failures.push(result.reason) + } + } + if (failures.length > 0) { + throw new AggregateError(failures, 'watcher_owned_children_shutdown_incomplete') + } + }) + .finally(() => { + this.disposal = null + }) + this.disposal = cleanup + return cleanup + } +} diff --git a/src/main/ipc/parcel-watcher-process-dispose.test.ts b/src/main/ipc/parcel-watcher-process-dispose.test.ts new file mode 100644 index 00000000000..fd022c16622 --- /dev/null +++ b/src/main/ipc/parcel-watcher-process-dispose.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from 'vitest' + +const { supervisorDispose, poolDispose } = vi.hoisted(() => ({ + supervisorDispose: vi.fn<() => Promise>(), + poolDispose: vi.fn<() => Promise>() +})) + +vi.mock('./parcel-watcher-process-supervisor', () => ({ + WatcherProcessSupervisor: class { + disposeAndWait = supervisorDispose + } +})) +vi.mock('./runtime-watcher-process-pool', () => ({ + RuntimeWatcherProcessPool: class { + disposeAndWait = poolDispose + } +})) + +import { disposeWatcherProcessAndWait } from './parcel-watcher-process' + +describe('disposeWatcherProcessAndWait', () => { + it('waits for both watcher hosts, and reports a failed one only after the other settles', async () => { + let finishPool!: () => void + supervisorDispose.mockRejectedValueOnce(new Error('child still running')) + poolDispose.mockReturnValueOnce( + new Promise((resolve) => { + finishPool = resolve + }) + ) + const settled = vi.fn() + const disposal = disposeWatcherProcessAndWait().catch((error: unknown) => { + settled() + throw error + }) + await Promise.resolve() + expect(settled).not.toHaveBeenCalled() + finishPool() + await expect(disposal).rejects.toMatchObject({ message: 'watcher_process_shutdown_incomplete' }) + }) + + it('resolves once every owned child has exited', async () => { + supervisorDispose.mockResolvedValueOnce() + poolDispose.mockResolvedValueOnce() + await expect(disposeWatcherProcessAndWait()).resolves.toBeUndefined() + expect(supervisorDispose).toHaveBeenCalled() + expect(poolDispose).toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/parcel-watcher-process-entry.test.ts b/src/main/ipc/parcel-watcher-process-entry.test.ts index 24cfc5981e2..ccc76293af7 100644 --- a/src/main/ipc/parcel-watcher-process-entry.test.ts +++ b/src/main/ipc/parcel-watcher-process-entry.test.ts @@ -86,8 +86,13 @@ describe('parcel watcher process canary', () => { await vi.advanceTimersByTimeAsync(0) expect(sendMock).toHaveBeenCalledWith( - expect.objectContaining({ op: 'subscribe-failed', id: 7 }) + expect.objectContaining({ op: 'subscribe-failed', id: 7 }), + expect.any(Function) ) + const failedSend = sendMock.mock.calls.find(([message]) => message.op === 'subscribe-failed') + expect(() => + failedSend![1](Object.assign(new Error('host disconnected'), { code: 'EPIPE' })) + ).not.toThrow() expect(watchMock).not.toHaveBeenCalledWith('/repo/.git', expect.anything(), expect.anything()) }) @@ -313,10 +318,13 @@ describe('parcel watcher process canary', () => { finishActiveCrawl?.({ unsubscribe: vi.fn().mockResolvedValue(undefined) }) await vi.advanceTimersByTimeAsync(0) - expect(sendMock).toHaveBeenCalledWith({ op: 'unsubscribed', id: 2 }) + expect(sendMock).toHaveBeenCalledWith({ op: 'unsubscribed', id: 2 }, expect.any(Function)) expect(subscribeMock).toHaveBeenCalledTimes(2) - expect(sendMock).not.toHaveBeenCalledWith({ op: 'subscribe-started', id: 2 }) - expect(sendMock).not.toHaveBeenCalledWith({ op: 'subscribed', id: 2 }) + expect(sendMock).not.toHaveBeenCalledWith( + { op: 'subscribe-started', id: 2 }, + expect.any(Function) + ) + expect(sendMock).not.toHaveBeenCalledWith({ op: 'subscribed', id: 2 }, expect.any(Function)) }) it('unsubscribes a late cancel after the crawl already finished', async () => { @@ -332,13 +340,16 @@ describe('parcel watcher process canary', () => { process.emit('message', { op: 'subscribe', id: 1, dir: '/finished', opts: {} }) await vi.advanceTimersByTimeAsync(0) - expect(sendMock).toHaveBeenCalledWith({ op: 'subscribed', id: 1 }) + expect(sendMock).toHaveBeenCalledWith({ op: 'subscribed', id: 1 }, expect.any(Function)) process.emit('message', { op: 'cancel-subscribe', id: 1 }) await vi.advanceTimersByTimeAsync(0) expect(unsubscribe).toHaveBeenCalledTimes(1) - expect(sendMock).toHaveBeenCalledWith({ op: 'unsubscribed', id: 1 }) - expect(sendMock).not.toHaveBeenCalledWith({ op: 'cancel-requires-restart', id: 1 }) + expect(sendMock).toHaveBeenCalledWith({ op: 'unsubscribed', id: 1 }, expect.any(Function)) + expect(sendMock).not.toHaveBeenCalledWith( + { op: 'cancel-requires-restart', id: 1 }, + expect.any(Function) + ) }) it('reports native unsubscribe rejection without acknowledging handle release', async () => { @@ -356,12 +367,15 @@ describe('parcel watcher process canary', () => { process.emit('message', { op: 'unsubscribe', id: 1 }) await vi.advanceTimersByTimeAsync(0) - expect(sendMock).toHaveBeenCalledWith({ - op: 'unsubscribe-failed', - id: 1, - message: 'native handle still active' - }) - expect(sendMock).not.toHaveBeenCalledWith({ op: 'unsubscribed', id: 1 }) + expect(sendMock).toHaveBeenCalledWith( + { + op: 'unsubscribe-failed', + id: 1, + message: 'native handle still active' + }, + expect.any(Function) + ) + expect(sendMock).not.toHaveBeenCalledWith({ op: 'unsubscribed', id: 1 }, expect.any(Function)) }) it('asks the host to restart when an active crawl is cancelled', async () => { @@ -380,10 +394,13 @@ describe('parcel watcher process canary', () => { await vi.advanceTimersByTimeAsync(0) process.emit('message', { op: 'cancel-subscribe', id: 1 }) - expect(sendMock).toHaveBeenCalledWith({ op: 'cancel-requires-restart', id: 1 }) + expect(sendMock).toHaveBeenCalledWith( + { op: 'cancel-requires-restart', id: 1 }, + expect.any(Function) + ) finishCrawl?.({ unsubscribe: vi.fn().mockResolvedValue(undefined) }) await vi.advanceTimersByTimeAsync(0) - expect(sendMock).not.toHaveBeenCalledWith({ op: 'subscribed', id: 1 }) + expect(sendMock).not.toHaveBeenCalledWith({ op: 'subscribed', id: 1 }, expect.any(Function)) }) it('still restarts after consecutive missed events once every subscription is live', async () => { @@ -481,11 +498,14 @@ describe('parcel watcher process canary', () => { callback?.(null, [{ type: 'update', path: '/repo/after-overflow.txt' }]) await vi.advanceTimersByTimeAsync(0) - expect(sendMock).toHaveBeenCalledWith({ - op: 'watch-error', - id: 1, - message: 'Events were dropped by the FSEvents client. File system must be re-scanned.' - }) + expect(sendMock).toHaveBeenCalledWith( + { + op: 'watch-error', + id: 1, + message: 'Events were dropped by the FSEvents client. File system must be re-scanned.' + }, + expect.any(Function) + ) expect(sendMock).toHaveBeenCalledWith( { op: 'events', diff --git a/src/main/ipc/parcel-watcher-process-entry.ts b/src/main/ipc/parcel-watcher-process-entry.ts index ce024aa36a4..9eb929fe2d6 100644 --- a/src/main/ipc/parcel-watcher-process-entry.ts +++ b/src/main/ipc/parcel-watcher-process-entry.ts @@ -118,7 +118,7 @@ async function startCanary(getStableActivityRevision: () => number | null): Prom function main(): void { const send = (message: WatcherToHostMessage): void => { try { - process.send?.(message) + process.send?.(message, () => undefined) } catch { // Host is gone; the disconnect handler below exits this process. } diff --git a/src/main/ipc/parcel-watcher-process-supervisor.ts b/src/main/ipc/parcel-watcher-process-supervisor.ts index 86f3fecdec8..a495096070a 100644 --- a/src/main/ipc/parcel-watcher-process-supervisor.ts +++ b/src/main/ipc/parcel-watcher-process-supervisor.ts @@ -1,9 +1,8 @@ import type { ChildProcess } from 'node:child_process' -import { existsSync } from 'node:fs' import { restartCancelledWatcherChild } from './parcel-watcher-cancellation-restart' import { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' -import { getWatcherProcessEntryPath } from './parcel-watcher-entry-path' -import { removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory' +import { getWatcherProcessEntryPath, watcherProcessEntryExists } from './parcel-watcher-entry-path' +import { WatcherChildSlot } from './parcel-watcher-child-slot' import * as termination from './parcel-watcher-child-termination' import { launchWatcherChild } from './parcel-watcher-child-launch' import { sendToWatcherChild } from './parcel-watcher-child-messaging' @@ -39,19 +38,19 @@ import { } from './parcel-watcher-supervisor-subscribe' import { disposeWatcherSupervisor } from './parcel-watcher-supervisor-disposal' import { handleWatcherSupervisorMessage } from './parcel-watcher-supervisor-message' +import { WatcherOwnedChildren } from './parcel-watcher-owned-children' export class WatcherProcessSupervisor { - private child: ChildProcess | null = null private nextSubscriptionId = 1 private readonly crashFuse = new WatcherProcessCrashFuse() private shutdown = { requested: false, disposalRevision: 0 } - private canaryDir: string | null = null - private terminatingChild: ChildProcess | null = null + private readonly slot = new WatcherChildSlot() private readonly terminationQueue = new termination.WatcherTerminationQueue() private readonly records = new Map() private readonly pendingUnsubscribes = new Map() private readonly cancelledSubscribes = new WatcherCancellationTracker() private readonly capacityWait = new WatcherSupervisorCapacityWait() + private ownedChildren = new WatcherOwnedChildren() constructor(private readonly options: WatcherProcessSupervisorOptions = {}) {} @@ -78,7 +77,7 @@ export class WatcherProcessSupervisor { records: this.records, pendingUnsubscribes: this.pendingUnsubscribes, ensureWatcherProcess: (entryPath) => this.ensureWatcherProcess(entryPath), - getChild: () => this.child, + getChild: () => this.slot.child, getTerminationPromise: () => this.terminationQueue.getCurrent(), killWatcherChildIfIdle: () => this.killWatcherChildIfIdle(), terminateUnavailableChild: (child) => this.terminateUnavailableChild(child), @@ -95,63 +94,65 @@ export class WatcherProcessSupervisor { this.shutdown.requested = true this.shutdown.disposalRevision++ this.capacityWait.dispose() - const proc = this.child - this.child = null - this.canaryDir = disposeWatcherSupervisor( + const proc = this.slot.child + this.slot.child = null + this.slot.canaryDir = disposeWatcherSupervisor( proc, this.records, this.pendingUnsubscribes, this.cancelledSubscribes, - this.canaryDir + this.slot.canaryDir ) } resetForTest(): void { this.dispose() + this.ownedChildren = new WatcherOwnedChildren() this.shutdown.requested = false - this.terminatingChild = null + this.slot.terminating = null this.terminationQueue.resetForTest() this.crashFuse.reset() resetWatcherChildRegistryForTest() } + disposeAndWait = (): Promise => this.ownedChildren.disposeAndWait(() => this.dispose()) + private ensureWatcherProcess( entryPath = this.options.entryPath ?? getWatcherProcessEntryPath() ): ChildProcess | null { - if (this.shutdown.requested || this.terminatingChild) { + if (this.shutdown.requested || this.slot.terminating) { return null } - if (this.child?.connected) { - return this.child + if (this.slot.child?.connected) { + return this.slot.child } if (this.crashFuse.isOpen()) { return null } - if (!existsSync(entryPath)) { - console.error(`[parcel-watcher-process] entry not found at ${entryPath}; refusing fail-open`) + if (!watcherProcessEntryExists(entryPath)) { return null } const launched = launchWatcherChild( entryPath, - this.canaryDir, + this.slot.canaryDir, (child, message) => { - if (this.child === child) { + if (this.slot.child === child) { this.handleChildMessage(message) } }, (child, code, signal) => this.handleChildGone(child, code, signal) ) if (!launched) { - this.canaryDir = null + this.slot.canaryDir = null return null } - this.canaryDir = launched.canaryDir - this.child = launched.child + this.slot.canaryDir = launched.canaryDir + this.slot.child = this.ownedChildren.track(launched.child) return launched.child } private handleChildMessage(message: WatcherToHostMessage): void { - const child = this.child + const child = this.slot.child const disposalRevision = this.shutdown.disposalRevision handleWatcherSupervisorMessage(message, { records: this.records, @@ -184,14 +185,14 @@ export class WatcherProcessSupervisor { code?: number | null, signal?: NodeJS.Signals | null ): void { - if (this.child !== proc) { + if (this.slot.child !== proc) { return } if (code === undefined) { this.terminateUnavailableChild(proc) return } - this.child = null + this.slot.child = null this.cancelledSubscribes.completeForChild(proc) resolvePendingWatcherUnsubscribes(this.pendingUnsubscribes) recoverWatcherRecordsAfterChildGone( @@ -200,9 +201,7 @@ export class WatcherProcessSupervisor { this.shutdown.requested, () => this.ensureWatcherProcess(), sendWatcherSubscribe, - () => { - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - }, + () => this.slot.removeCanary(), code, signal ) @@ -213,13 +212,12 @@ export class WatcherProcessSupervisor { if (currentTermination) { return currentTermination } - const proc = requestedChild ?? this.terminatingChild + const proc = requestedChild ?? this.slot.terminating if (!proc) { return Promise.resolve() } - this.child = null - this.terminatingChild = proc - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + this.slot.child = null + this.slot.beginTermination(proc) return this.terminationQueue.track( terminateDisconnectedWatcherChild( proc, @@ -228,7 +226,7 @@ export class WatcherProcessSupervisor { this.cancelledSubscribes, this.crashFuse, (exited) => { - this.terminatingChild = null + this.slot.terminating = null if (!exited) { this.shutdown.requested = true } @@ -236,9 +234,7 @@ export class WatcherProcessSupervisor { }, () => this.ensureWatcherProcess(), sendWatcherSubscribe, - () => { - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - } + () => this.slot.removeCanary() ) ) } @@ -248,21 +244,15 @@ export class WatcherProcessSupervisor { if (terminationPromise) { return terminationPromise } - const proc = this.child + const proc = this.slot.child if (!proc || this.records.size > 0) { return Promise.resolve() } - this.child = null - this.terminatingChild = proc - // Why: destructive Windows cleanup must await exit to release directory handles. - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + this.slot.child = null + this.slot.beginTermination(proc) return this.terminationQueue.track( termination.terminateIdleWatcherChild(proc, this.pendingUnsubscribes, () => { - // Why: an idle child owns zero records, so a missed exit deadline has no - // double-watch hazard; the child keeps its capacity reservation until - // physical exit, and poisoning this supervisor would permanently end - // local watching — the shared singleton has no retire-and-replace path. - this.terminatingChild = null + this.slot.terminating = null }) ) } @@ -275,7 +265,7 @@ export class WatcherProcessSupervisor { record, error, records: this.records, - child: this.child, + child: this.slot.child, cancelledSubscribes: this.cancelledSubscribes, onChildUnavailable: (child) => this.terminateUnavailableChild(child), restartChild: (child) => this.restartAfterCancelledSubscribe(child), @@ -288,11 +278,10 @@ export class WatcherProcessSupervisor { if (activeTermination || !proc || !this.cancelledSubscribes.beginRestart(proc)) { return activeTermination ?? Promise.resolve() } - if (this.child === proc) { - this.child = null + if (this.slot.child === proc) { + this.slot.child = null } - this.terminatingChild = proc - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + this.slot.beginTermination(proc) return this.terminationQueue.track( restartCancelledWatcherChild( proc, @@ -300,7 +289,7 @@ export class WatcherProcessSupervisor { this.pendingUnsubscribes, this.cancelledSubscribes, (exited) => { - this.terminatingChild = null + this.slot.terminating = null if (!exited) { this.shutdown.requested = true } @@ -308,9 +297,7 @@ export class WatcherProcessSupervisor { }, () => this.ensureWatcherProcess(), sendWatcherSubscribe, - () => { - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - } + () => this.slot.removeCanary() ) ) } diff --git a/src/main/ipc/parcel-watcher-process.ts b/src/main/ipc/parcel-watcher-process.ts index 223fed564f8..74e2c3ffa69 100644 --- a/src/main/ipc/parcel-watcher-process.ts +++ b/src/main/ipc/parcel-watcher-process.ts @@ -62,6 +62,20 @@ export function disposeWatcherProcess(): void { } } +/** Dispose both watcher hosts and wait for every child they own to exit. */ +export async function disposeWatcherProcessAndWait(): Promise { + const results = await Promise.allSettled([ + sharedWatcherProcessSupervisor.disposeAndWait(), + runtimeWatcherProcessPool.disposeAndWait() + ]) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length > 0) { + throw new AggregateError(failures, 'watcher_process_shutdown_incomplete') + } +} + export function resetWatcherProcessForTest(): void { sharedWatcherProcessSupervisor.resetForTest() } diff --git a/src/main/ipc/parcel-watcher-supervisor-disposal.test.ts b/src/main/ipc/parcel-watcher-supervisor-disposal.test.ts new file mode 100644 index 00000000000..d3aaa15770c --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-disposal.test.ts @@ -0,0 +1,67 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { acknowledgeWatcherSubscribe, FakeWatcherChild } from './parcel-watcher-process-test-child' +import { resetWatcherChildRegistryForTest } from './parcel-watcher-child-registry' + +const { forkMock } = vi.hoisted(() => ({ forkMock: vi.fn() })) +vi.mock('node:child_process', () => ({ fork: forkMock })) +vi.mock('node:fs', () => ({ + existsSync: vi.fn(() => true), + mkdtempSync: vi.fn(() => '/tmp/orca-watcher-disposal-test'), + rmSync: vi.fn() +})) +import { WatcherProcessSupervisor } from './parcel-watcher-process-supervisor' + +const children: FakeWatcherChild[] = [] +beforeEach(() => { + resetWatcherChildRegistryForTest() + forkMock.mockImplementation(() => { + const child = new FakeWatcherChild() + children.push(child) + return child + }) +}) +afterEach(() => { + for (const child of children.splice(0)) { + child.emit('close') + } + vi.clearAllMocks() + resetWatcherChildRegistryForTest() +}) + +async function subscribe(supervisor: WatcherProcessSupervisor): Promise { + const pending = supervisor.subscribe('/repo', vi.fn(), {}) + const child = children.at(-1)! + acknowledgeWatcherSubscribe(child) + await pending + return child +} + +it('awaited supervisor disposal retains a child after prior synchronous disposal', async () => { + const supervisor = new WatcherProcessSupervisor({ useInProcessVitestFallback: false }) + const child = await subscribe(supervisor) + supervisor.dispose() + const finished = vi.fn() + const shutdown = supervisor.disposeAndWait().then(finished) + await new Promise((resolve) => setImmediate(resolve)) + expect(finished).not.toHaveBeenCalled() + await expect(supervisor.subscribe('/another', vi.fn(), {})).rejects.toThrow() + expect(forkMock).toHaveBeenCalledOnce() + child.emit('close') + await shutdown + expect(finished).toHaveBeenCalledOnce() +}) + +it('test reset isolates old physical-exit callbacks from the new lifetime owner', async () => { + const supervisor = new WatcherProcessSupervisor({ useInProcessVitestFallback: false }) + const old = await subscribe(supervisor) + const oldShutdown = supervisor.disposeAndWait() + supervisor.resetForTest() + const current = await subscribe(supervisor) + const finished = vi.fn() + const shutdown = supervisor.disposeAndWait().then(finished) + old.emit('close') + await oldShutdown + expect(finished).not.toHaveBeenCalled() + current.emit('close') + await shutdown +}) diff --git a/src/main/ipc/parcel-watcher-supervisor-disposal.ts b/src/main/ipc/parcel-watcher-supervisor-disposal.ts index 42dc918e05b..07bc73ef3ee 100644 --- a/src/main/ipc/parcel-watcher-supervisor-disposal.ts +++ b/src/main/ipc/parcel-watcher-supervisor-disposal.ts @@ -9,6 +9,7 @@ import { resetPendingSubscribeAttempt, takePendingSubscribe } from './parcel-watcher-pending-subscribe' +import { signalWatcherChild } from './parcel-watcher-child-termination' import { watcherHostFailure } from './parcel-watcher-process-failure' import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' @@ -27,6 +28,8 @@ export function disposeWatcherSupervisor( resolvePendingWatcherUnsubscribes(pendingUnsubscribes) cancelledSubscribes.completeAll() records.clear() - child?.kill() + if (child) { + signalWatcherChild(child) + } return removeWatcherCanaryDirectory(canaryDir) } diff --git a/src/main/ipc/pty-ssh-stop-verdict.test.ts b/src/main/ipc/pty-ssh-stop-verdict.test.ts index 8001028ea1d..de2efc22396 100644 --- a/src/main/ipc/pty-ssh-stop-verdict.test.ts +++ b/src/main/ipc/pty-ssh-stop-verdict.test.ts @@ -184,6 +184,88 @@ describe('stopping a PTY whose SSH provider is unregistered', () => { } }) + it('waits for an observed SSH exit to reach the runtime record before confirming', async () => { + const connectionId = 'ssh-observed-exit' + const ptyId = 'ssh-observed-exit-pty' + const exitListeners = new Set<(payload: { id: string }) => void>() + const provider = { + onExit: vi.fn((listener: (payload: { id: string }) => void) => { + exitListeners.add(listener) + return () => exitListeners.delete(listener) + }), + shutdown: vi.fn(async () => { + for (const listener of exitListeners) { + listener({ id: ptyId }) + } + }), + listProcesses: vi.fn(async () => []) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + let recordExit: (recorded: boolean) => void = () => {} + const waitForPtyExitRecord = vi.fn( + (_ptyId: string, _timeoutMs: number) => + new Promise((resolve) => (recordExit = resolve)) + ) + Object.assign(runtime, { waitForPtyExitRecord }) + + let settled = false + const stop = controller.stopAndWait(ptyId).then((stopped) => { + settled = true + return stopped + }) + await vi.waitFor(() => expect(waitForPtyExitRecord).toHaveBeenCalled()) + const [waitedPtyId, timeoutMs] = waitForPtyExitRecord.mock.calls[0] ?? [] + expect(waitedPtyId).toBe(ptyId) + expect(timeoutMs).toBeGreaterThan(0) + expect(timeoutMs).toBeLessThanOrEqual(10_000) + expect(settled).toBe(false) + recordExit(true) + await expect(stop).resolves.toBe(true) + // The intake delivers the exit itself; the stop must not fabricate a second one. + expect(runtime.onPtyExit).not.toHaveBeenCalled() + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + + it('caps the observed-exit wait at the caller deadline', async () => { + const connectionId = 'ssh-observed-exit-deadline' + const ptyId = 'ssh-observed-exit-deadline-pty' + const exitListeners = new Set<(payload: { id: string }) => void>() + const provider = { + onExit: vi.fn((listener: (payload: { id: string }) => void) => { + exitListeners.add(listener) + return () => exitListeners.delete(listener) + }), + shutdown: vi.fn(async () => { + for (const listener of exitListeners) { + listener({ id: ptyId }) + } + }), + listProcesses: vi.fn(async () => []) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + const waitForPtyExitRecord = vi.fn(async (_ptyId: string, _timeoutMs: number) => false) + Object.assign(runtime, { waitForPtyExitRecord }) + + await expect(controller.stopAndWait(ptyId, { deadlineMs: Date.now() + 2_000 })).resolves.toBe( + true + ) + const timeoutMs = waitForPtyExitRecord.mock.calls[0]?.[1] + expect(timeoutMs).toBeLessThanOrEqual(2_000) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + it('reports lost contact when a registered provider drops during the stop', async () => { const connectionId = 'ssh-mid-stop-drop' const ptyId = 'ssh-mid-stop-pty' diff --git a/src/main/ipc/pty/runtime/kill.ts b/src/main/ipc/pty/runtime/kill.ts index edbd12b0b18..5d8079ef62b 100644 --- a/src/main/ipc/pty/runtime/kill.ts +++ b/src/main/ipc/pty/runtime/kill.ts @@ -7,6 +7,9 @@ import { isPtyAlreadyGoneError, delay, verifyPtyStopped } from '../provider/live import { recordUndeliveredSshPtyKill } from './undelivered-ssh-kill' import type { PtyRuntimeControllerDeps } from './controller-deps' +// Bounds the wait for an observed exit to reach the runtime record when the caller set no deadline. +const OBSERVED_EXIT_RECORD_WAIT_MS = 10_000 + export function killPtyFromRuntimeController( deps: PtyRuntimeControllerDeps, ptyId: string @@ -306,6 +309,11 @@ export async function stopAndWaitPtyFromRuntimeController( code: 0, ...(incarnationId ? { incarnationId } : {}) }) + } else { + // Why: an SSH exit frame reaches the runtime only after its output intake drains, so callers + // reading the verdict right after this stop would otherwise see a still-connected PTY. + const waitUntil = Math.min(deadlineMs ?? Infinity, Date.now() + OBSERVED_EXIT_RECORD_WAIT_MS) + await runtime?.waitForPtyExitRecord?.(ptyId, waitUntil - Date.now()) } return true } diff --git a/src/main/ipc/renderer-shutdown-checkpoint.test.ts b/src/main/ipc/renderer-shutdown-checkpoint.test.ts index abbb77ab087..43491ba31af 100644 --- a/src/main/ipc/renderer-shutdown-checkpoint.test.ts +++ b/src/main/ipc/renderer-shutdown-checkpoint.test.ts @@ -82,6 +82,26 @@ describe('registerRendererShutdownCheckpointHandler', () => { expect(event.returnValue).toEqual({ ok: true }) }) + it('never stages the source partition of a fenced host', () => { + const store = { + stageWorkspaceSessionBeforeUnload: vi.fn(), + getSshTarget: vi.fn(() => ({ orcadFence: { environmentId: 'env-1' } })), + updateUI: vi.fn(), + flushPendingOrThrowAsync: vi.fn(() => Promise.resolve()) + } + registerRendererShutdownCheckpointHandler(store as never) + + const event: { returnValue?: unknown } = {} + syncHandlers.get('app:stage-before-unload-sync')?.(event, { + sessions: [{ state: {} }, { state: {}, hostId: 'ssh:target-1' }], + ui: {} + }) + + expect(store.stageWorkspaceSessionBeforeUnload).toHaveBeenCalledTimes(1) + expect(store.stageWorkspaceSessionBeforeUnload).toHaveBeenCalledWith({}, undefined) + expect(event.returnValue).toEqual({ ok: true }) + }) + it('reports a staging failure so the renderer can retry', () => { const store = { stageWorkspaceSessionBeforeUnload: vi.fn(), diff --git a/src/main/ipc/renderer-shutdown-checkpoint.ts b/src/main/ipc/renderer-shutdown-checkpoint.ts index 1e5df10f1a2..82fcbee0d4e 100644 --- a/src/main/ipc/renderer-shutdown-checkpoint.ts +++ b/src/main/ipc/renderer-shutdown-checkpoint.ts @@ -3,6 +3,7 @@ import type { ExecutionHostId } from '../../shared/execution-host' import type { PersistedUIState } from '../../shared/persisted-ui-state-types' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import type { Store } from '../persistence' +import { isFrozenOrcadSourceSessionPartition } from '../ssh/orcad-retained-source' type StageBeforeUnloadSyncArgs = { sessions: { state: WorkspaceSessionState; hostId?: ExecutionHostId }[] @@ -49,7 +50,9 @@ export function registerRendererShutdownCheckpointHandler(store: Store): void { let ok = true try { for (const { state, hostId } of args.sessions) { - store.stageWorkspaceSessionBeforeUnload(state, hostId) + if (!isFrozenOrcadSourceSessionPartition(store, hostId)) { + store.stageWorkspaceSessionBeforeUnload(state, hostId) + } } store.updateUI(args.ui) } catch (error) { diff --git a/src/main/ipc/repos-remote-test-harness.ts b/src/main/ipc/repos-remote-test-harness.ts index 075b911ad50..3e7cfe783c2 100644 --- a/src/main/ipc/repos-remote-test-harness.ts +++ b/src/main/ipc/repos-remote-test-harness.ts @@ -25,7 +25,8 @@ export type ReposIpcMocks = { | 'updateProjectGroup' | 'deleteProjectGroup' | 'moveProjectToGroup' - | 'getSshTarget', + | 'getSshTarget' + | 'getSshTargets', ReposIpcSpy > & { updateRepo: Mock<(repoId: string, updates: Record) => unknown> } mockGitProvider: Record< @@ -62,7 +63,8 @@ export function createReposIpcMocks(): ReposIpcMocks { updateProjectGroup: vi.fn(), deleteProjectGroup: vi.fn(), moveProjectToGroup: vi.fn(), - getSshTarget: vi.fn() + getSshTarget: vi.fn(), + getSshTargets: vi.fn().mockReturnValue([]) }, mockGitProvider: { isGitRepo: vi.fn().mockReturnValue(true), diff --git a/src/main/ipc/repos/folder-workspace-handlers.ts b/src/main/ipc/repos/folder-workspace-handlers.ts index 2c2968a3c9a..634c6aae010 100644 --- a/src/main/ipc/repos/folder-workspace-handlers.ts +++ b/src/main/ipc/repos/folder-workspace-handlers.ts @@ -18,13 +18,14 @@ import { FolderWorkspaceUpdateArgs, parseProjectGroupIpcArgs } from './repo-ipc-arg-schemas' +import { visibleFolderWorkspaces } from '../../ssh/orcad-retained-source' export function registerFolderWorkspaceHandlers( mainWindow: BrowserWindow, store: Store, runtime: OrcaRuntimeService ): void { - ipcMain.handle('folderWorkspaces:list', (): FolderWorkspace[] => store.getFolderWorkspaces()) + ipcMain.handle('folderWorkspaces:list', (): FolderWorkspace[] => visibleFolderWorkspaces(store)) ipcMain.handle('folderWorkspaces:getPathStatus', async (_event, rawArgs: unknown) => { const args = parseProjectGroupIpcArgs( diff --git a/src/main/ipc/repos/host-repo-catalog-snapshot.ts b/src/main/ipc/repos/host-repo-catalog-snapshot.ts index e6a55441c28..3e16cf6245b 100644 --- a/src/main/ipc/repos/host-repo-catalog-snapshot.ts +++ b/src/main/ipc/repos/host-repo-catalog-snapshot.ts @@ -12,6 +12,7 @@ import { import { isAdmissibleDirectSshAuthority } from '../../../shared/ssh-retained-payload-admission' import { isCurrentSshProviderAuthority } from '../../ssh/ssh-provider-authority' import { getSshGitProvider } from '../../providers/ssh-git-dispatch' +import { visibleRepos } from '../../ssh/orcad-retained-source' function hasValidCatalogSshAuthority( args: ListReposForExecutionHostArgs @@ -62,7 +63,7 @@ export async function listReposForExecutionHost( if ('expectedAuthority' in args) { return rejected('rejected') } - const repos = getConsistentRepoCatalogForHost(store.getRepos(), parsedHost) + const repos = getConsistentRepoCatalogForHost(visibleRepos(store), parsedHost) if (!repos) { return rejected('rejected') } @@ -86,7 +87,7 @@ export async function listReposForExecutionHost( if (!provider) { return rejected('unavailable') } - const matchingRepos = getConsistentRepoCatalogForHost(store.getRepos(), parsedHost) + const matchingRepos = getConsistentRepoCatalogForHost(visibleRepos(store), parsedHost) if (!matchingRepos) { return rejected('rejected') } diff --git a/src/main/ipc/repos/project-group-handlers.ts b/src/main/ipc/repos/project-group-handlers.ts index 257e14c935a..25f9c4506a4 100644 --- a/src/main/ipc/repos/project-group-handlers.ts +++ b/src/main/ipc/repos/project-group-handlers.ts @@ -14,9 +14,10 @@ import { parseProjectGroupIpcArgs } from './repo-ipc-arg-schemas' import { activeNestedRepoScans, runNestedRepoScanForIpc } from './nested-repo-scan-ipc' +import { visibleProjectGroups } from '../../ssh/orcad-retained-source' export function registerProjectGroupHandlers(mainWindow: BrowserWindow, store: Store): void { - ipcMain.handle('projectGroups:list', () => store.getProjectGroups()) + ipcMain.handle('projectGroups:list', () => visibleProjectGroups(store)) ipcMain.handle('projectGroups:create', (_event, rawArgs: unknown): ProjectGroup => { const args = parseProjectGroupIpcArgs( diff --git a/src/main/ipc/repos/repo-catalog-handlers.ts b/src/main/ipc/repos/repo-catalog-handlers.ts index 8f3e91f5d29..825daef9850 100644 --- a/src/main/ipc/repos/repo-catalog-handlers.ts +++ b/src/main/ipc/repos/repo-catalog-handlers.ts @@ -13,6 +13,7 @@ import { invalidateAuthorizedRootsCache } from '../registered-worktree-roots-cac import { notifyReposChanged } from './repos-changed-notification' import { ProjectUpdateIpcArgs, parseProjectGroupIpcArgs } from './repo-ipc-arg-schemas' import { listReposForExecutionHost } from './host-repo-catalog-snapshot' +import { visibleRepos } from '../../ssh/orcad-retained-source' export function registerRepoCatalogHandlers(mainWindow: BrowserWindow, store: Store): void { // Why one shared reference: enrichment dedupes coalesced callers by callback identity, so a fresh @@ -23,7 +24,7 @@ export function registerRepoCatalogHandlers(mainWindow: BrowserWindow, store: St enrichMissingRepoGitRemoteIdentities(store, { onChanged: broadcastReposChanged }) // Why: username resolution spawns git/gh, so keep it off this sync handler (issue #7225); it re-lists when values land. enrichRepoGitUsernames(store, { onChanged: broadcastReposChanged }) - return store.getRepos() + return visibleRepos(store) }) ipcMain.handle( diff --git a/src/main/ipc/runtime-environment-capability-evidence.ts b/src/main/ipc/runtime-environment-capability-evidence.ts index 197ec71f4f8..7dd47712fb2 100644 --- a/src/main/ipc/runtime-environment-capability-evidence.ts +++ b/src/main/ipc/runtime-environment-capability-evidence.ts @@ -64,6 +64,10 @@ export function advanceRuntimeEnvironmentCapabilityIncarnation(environmentId: st state.accepted = null } +export function getRuntimeEnvironmentCapabilityIncarnation(environmentId: string): number { + return stateFor(environmentId).epoch +} + export function applyRuntimeEnvironmentCapabilityVerdict(args: { evidence: RuntimeEnvironmentCapabilityEvidence verdict: RuntimeEnvironmentCapabilityVerdict diff --git a/src/main/ipc/runtime-environment-connectivity-handlers.ts b/src/main/ipc/runtime-environment-connectivity-handlers.ts index 237228b0eab..013cdef9ea3 100644 --- a/src/main/ipc/runtime-environment-connectivity-handlers.ts +++ b/src/main/ipc/runtime-environment-connectivity-handlers.ts @@ -14,11 +14,9 @@ import { RuntimeRpcCallQueueOverloadError } from '../../shared/runtime-rpc-call- import type { RuntimeRpcFailure, RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' import type { RuntimeStatus } from '../../shared/runtime-types' import type { Store } from '../persistence' +import { retireRemovedRuntimeEnvironment } from './runtime-environment-removal-cleanup' import { readSettingsWithRuntimeEnvironmentPreference } from './runtime-environment-preference' -import { clearBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-runtime' -import { retireBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-retirement' import { verifyAndAddRuntimeEnvironmentFromPairingCode } from './runtime-environment-pairing-verification' -import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' import { closeRemoteRuntimeRequestConnection, getRuntimeEnvironmentStatusOwner, @@ -35,6 +33,7 @@ import { callRuntimeEnvironment, getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' +import { publicRuntimeEnvironmentWithHostKey } from './runtime-environment-host-key' function manuallyDisconnectedResponse( environment: ReturnType @@ -69,7 +68,7 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ ipcMain.handle('runtimeEnvironments:list', () => { const environments = listEnvironments(getUserDataPath()) readSettingsWithRuntimeEnvironmentPreference(store, getUserDataPath()) - return environments.map(redactRuntimeEnvironment) + return environments.map(publicRuntimeEnvironmentWithHostKey) }) ipcMain.handle( 'runtimeEnvironments:addFromPairingCode', @@ -109,22 +108,10 @@ export function registerRuntimeEnvironmentConnectivityHandlers({ throw new Error('Choose another Active Server in Advanced before removing this server.') } const removed = removeEnvironment(getUserDataPath(), args.selector) - clearRuntimeEnvironmentCapabilityEvidence(removed.id) - clearRuntimeEnvironmentManualDisconnect(removed.id) - const retiring = Promise.resolve(invalidateTransport(removed.id)) + void retireRemovedRuntimeEnvironment(removed.id, invalidateTransport, (hostId) => + store.removeWorkspaceSessionHost(hostId) + ) closeLegacySelectorTransport(args.selector, removed.id) - // Why: removal is an explicit lifecycle decision, so its client-hosted browser storage goes - // too -- but only once the client host releases its partitions, or every one refuses as live. - void retireBrowserRoutePartitionStorageForEnvironment({ - environmentId: removed.id, - whenClientHostClosed: retiring, - clearStorage: clearBrowserRoutePartitionStorageForEnvironment, - onError: (error) => { - console.warn('[runtime-environments] browser partition storage clear failed:', error) - } - }).catch((error) => { - console.warn('[runtime-environments] browser partition storage clear failed:', error) - }) return { removed: redactRuntimeEnvironment(removed) } } ) diff --git a/src/main/ipc/runtime-environment-handler-channels.ts b/src/main/ipc/runtime-environment-handler-channels.ts index 23b40fe5183..3ca79bc3aa5 100644 --- a/src/main/ipc/runtime-environment-handler-channels.ts +++ b/src/main/ipc/runtime-environment-handler-channels.ts @@ -12,5 +12,22 @@ export const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:getStatusSnapshots', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', - 'runtimeEnvironments:unsubscribe' + 'runtimeEnvironments:unsubscribe', + 'runtimeEnvironments:linkSshAccess', + 'runtimeEnvironments:unlinkSshAccess', + 'runtimeEnvironments:deployOrcad', + 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', + 'runtimeEnvironments:convertSshHostToManagedOrcad', + 'runtimeEnvironments:listPendingOrcadMigrations', + 'runtimeEnvironments:previewOrcadDeltaMove', + 'runtimeEnvironments:moveOrcadDelta', + 'runtimeEnvironments:keepOrcadServerVersion', + 'runtimeEnvironments:createOrcadSshHost', + 'runtimeEnvironments:resumeOrcadSshHost', + 'runtimeEnvironments:listPendingOrcadSshProvisioning' ] as const diff --git a/src/main/ipc/runtime-environment-host-key.test.ts b/src/main/ipc/runtime-environment-host-key.test.ts new file mode 100644 index 00000000000..06e391fa5e9 --- /dev/null +++ b/src/main/ipc/runtime-environment-host-key.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { publicRuntimeEnvironmentWithHostKey } from './runtime-environment-host-key' + +function environment(publicKeyB64: string): KnownRuntimeEnvironment { + return { + id: 'env-1', + name: 'Box', + createdAt: 1, + updatedAt: 1, + lastUsedAt: null, + runtimeId: null, + preferredEndpointId: 'ws', + endpoints: [ + { + id: 'ws', + kind: 'websocket', + label: 'Box', + endpoint: 'ws://127.0.0.1:46768/', + deviceToken: 'secret-token', + publicKeyB64 + } + ] + } +} + +describe('the public runtime environment record', () => { + it('carries a stable host key digest but never the key or the device token', () => { + const first = publicRuntimeEnvironmentWithHostKey(environment('key-a')) + const again = publicRuntimeEnvironmentWithHostKey(environment('key-a')) + const other = publicRuntimeEnvironmentWithHostKey(environment('key-b')) + + expect(first.hostKeyFingerprint).toMatch(/^[0-9a-f]{32}$/) + expect(again.hostKeyFingerprint).toBe(first.hostKeyFingerprint) + expect(other.hostKeyFingerprint).not.toBe(first.hostKeyFingerprint) + expect(JSON.stringify(first)).not.toMatch(/key-a|secret-token/) + }) +}) diff --git a/src/main/ipc/runtime-environment-host-key.ts b/src/main/ipc/runtime-environment-host-key.ts new file mode 100644 index 00000000000..f46bb509742 --- /dev/null +++ b/src/main/ipc/runtime-environment-host-key.ts @@ -0,0 +1,24 @@ +import { createHash } from 'node:crypto' +import { + getPreferredPairingOffer, + redactRuntimeEnvironment, + type KnownRuntimeEnvironment, + type PublicKnownRuntimeEnvironment +} from '../../shared/runtime-environments' + +/** + * The redacted record plus a digest of the host's E2EE public key. The host keeps that key in its + * own profile across updates and proves it in every pairing handshake; a reinstalled or different + * host has another. The renderer cannot see the key itself, so it compares this digest. + */ +export function publicRuntimeEnvironmentWithHostKey( + environment: KnownRuntimeEnvironment +): PublicKnownRuntimeEnvironment { + const publicKey = getPreferredPairingOffer(environment).publicKeyB64 + return { + ...redactRuntimeEnvironment(environment), + ...(publicKey + ? { hostKeyFingerprint: createHash('sha256').update(publicKey).digest('hex').slice(0, 32) } + : {}) + } +} diff --git a/src/main/ipc/runtime-environment-managed-tunnel.ts b/src/main/ipc/runtime-environment-managed-tunnel.ts new file mode 100644 index 00000000000..2654314d36d --- /dev/null +++ b/src/main/ipc/runtime-environment-managed-tunnel.ts @@ -0,0 +1,88 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { + SshManagedServerStatus, + SshManagedServerUpdateNote, + SshTarget +} from '../../shared/ssh-types' +import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' +import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' +import { setManagedOrcadStartListener } from '../ssh/orcad-managed-serving' +import { updateManagedOrcadOnRestore } from '../ssh/orcad-managed-update-on-restore' +import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context' +import { broadcastSshState } from './ssh-renderer-broadcast' + +export async function resolveManagedRuntimeEnvironment( + userDataPath: string, + selector: string +): Promise> { + const environment = resolveEnvironment(userDataPath, selector) + await ensureOrcadManagedTunnel(userDataPath, environment.id) + // Why: a server that stopped (idle, killed, host rebooted) starts before the call that needs it. + await verifyOrcadManagedServing(userDataPath, environment.id) + // Why here: an auto-restored host may never see an SSH connect, so it would never update. + void updateManagedOrcadOnRestore(environment.id, () => ({ + ...managedServerUpdateDeps(userDataPath), + target: () => { + const targetId = environment.orcadDeployment?.sshTargetId + return targetId ? (getSshTargetRegistryStore()?.getTarget(targetId) ?? null) : null + }, + publish: publishRestoreUpdate + })) + return resolveEnvironment(userDataPath, environment.id) +} + +function publishRestoreUpdate( + target: SshTarget, + environmentId: string, + phase: 'updating' | 'settled', + note?: SshManagedServerUpdateNote +): void { + publishHostServerStatus( + target, + phase === 'updating' + ? { kind: 'setting-up', phase: 'updating' } + : { kind: 'managed', environmentId, ...(note ? { update: note } : {}) } + ) +} + +/** Shows a managed server's start on the host's status line, wherever the start began. */ +export function installManagedOrcadStartStatus(): void { + setManagedOrcadStartListener({ + starting: (target) => + publishHostServerStatus(target, { kind: 'setting-up', phase: 'starting' }), + settled: (target, environmentId, serving) => + publishHostServerStatus(target, { + kind: 'managed', + environmentId, + ...(serving.state === 'unverifiable' ? { serving } : {}) + }) + }) +} + +/** A changed host the user resolved (moved or kept) is managed again; its status line says so now. */ +export function publishResolvedChangedHostStatus(target: SshTarget, environmentId: string): void { + setSshHostServerStatus(target.id, { kind: 'managed', environmentId }) + // Why a disconnected state too: the move released the relay session, which had the stale line. + broadcastSshState( + getCurrentMainWindow, + target.id, + connectionManager?.getState(target.id) ?? { + targetId: target.id, + status: 'disconnected', + error: null, + reconnectAttempt: 0 + } + ) +} + +function publishHostServerStatus(target: SshTarget, status: SshManagedServerStatus): void { + setSshHostServerStatus(target.id, status) + // Only a host with a connection state has a status line to refresh. + const state = connectionManager?.getState(target.id) + if (state) { + broadcastSshState(getCurrentMainWindow, target.id, state) + } +} diff --git a/src/main/ipc/runtime-environment-removal-cleanup.test.ts b/src/main/ipc/runtime-environment-removal-cleanup.test.ts new file mode 100644 index 00000000000..261a24f652d --- /dev/null +++ b/src/main/ipc/runtime-environment-removal-cleanup.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../browser/browser-route-partition-storage-runtime', () => ({ + clearBrowserRoutePartitionStorageForEnvironment: vi.fn() +})) +vi.mock('../browser/browser-route-partition-storage-retirement', () => ({ + retireBrowserRoutePartitionStorageForEnvironment: vi.fn(async () => undefined) +})) + +const { retireRemovedRuntimeEnvironment } = await import('./runtime-environment-removal-cleanup') + +describe('retiring a removed runtime environment', () => { + it('forgets the server’s workspace session partition under its runtime host id', async () => { + const forgetHostSession = vi.fn() + await retireRemovedRuntimeEnvironment('env 1', vi.fn(), forgetHostSession) + expect(forgetHostSession).toHaveBeenCalledWith('runtime:env%201') + }) +}) diff --git a/src/main/ipc/runtime-environment-removal-cleanup.ts b/src/main/ipc/runtime-environment-removal-cleanup.ts new file mode 100644 index 00000000000..c1961c137d9 --- /dev/null +++ b/src/main/ipc/runtime-environment-removal-cleanup.ts @@ -0,0 +1,31 @@ +import { clearBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-runtime' +import { retireBrowserRoutePartitionStorageForEnvironment } from '../browser/browser-route-partition-storage-retirement' +import { clearRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' +import { clearRuntimeEnvironmentManualDisconnect } from './runtime-environment-manual-disconnect' +import { toRuntimeExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' + +/** Retires a removed server's client-side state; resolves once its transport is invalidated. */ +export function retireRemovedRuntimeEnvironment( + environmentId: string, + invalidateTransport: (environmentId: string) => Promise | void, + forgetHostSession?: (hostId: ExecutionHostId) => void +): Promise { + clearRuntimeEnvironmentCapabilityEvidence(environmentId) + // Why: listings enumerate session partitions as known hosts, so a kept one names a dead server. + forgetHostSession?.(toRuntimeExecutionHostId(environmentId)) + clearRuntimeEnvironmentManualDisconnect(environmentId) + const retiring = Promise.resolve(invalidateTransport(environmentId)) + // Why: removal is an explicit lifecycle decision, so its client-hosted browser storage goes + // too -- but only once the client host releases its partitions, or every one refuses as live. + void retireBrowserRoutePartitionStorageForEnvironment({ + environmentId, + whenClientHostClosed: retiring, + clearStorage: clearBrowserRoutePartitionStorageForEnvironment, + onError: (error) => { + console.warn('[runtime-environments] browser partition storage clear failed:', error) + } + }).catch((error) => { + console.warn('[runtime-environments] browser partition storage clear failed:', error) + }) + return retiring +} diff --git a/src/main/ipc/runtime-environment-removal-storage.test.ts b/src/main/ipc/runtime-environment-removal-storage.test.ts index 2b055df5443..012df15f47a 100644 --- a/src/main/ipc/runtime-environment-removal-storage.test.ts +++ b/src/main/ipc/runtime-environment-removal-storage.test.ts @@ -47,7 +47,8 @@ describe('runtime environment removal storage clearing', () => { }) clearStorageMock.mockResolvedValue({ clearedPartitions: ['persist:one'], livePartitions: [] }) registerRuntimeEnvironmentConnectivityHandlers({ - store: { getSettings: () => ({}) } as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: remove reads only settings and the session-host prune. + store: { getSettings: () => ({}), removeWorkspaceSessionHost: vi.fn() } as never, getUserDataPath: () => '/tmp/orca-user-data', invalidateTransport: () => teardown }) @@ -67,7 +68,8 @@ describe('runtime environment removal storage clearing', () => { .mockResolvedValueOnce({ clearedPartitions: [], livePartitions: ['persist:one'] }) .mockResolvedValueOnce({ clearedPartitions: ['persist:one'], livePartitions: [] }) registerRuntimeEnvironmentConnectivityHandlers({ - store: { getSettings: () => ({}) } as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: remove reads only settings and the session-host prune. + store: { getSettings: () => ({}), removeWorkspaceSessionHost: vi.fn() } as never, getUserDataPath: () => '/tmp/orca-user-data', invalidateTransport: () => Promise.resolve() }) diff --git a/src/main/ipc/runtime-environment-revision-guard.ts b/src/main/ipc/runtime-environment-revision-guard.ts index af70d556356..03d6169675d 100644 --- a/src/main/ipc/runtime-environment-revision-guard.ts +++ b/src/main/ipc/runtime-environment-revision-guard.ts @@ -15,6 +15,14 @@ export function runtimeEnvironmentRevisionFailure( if (!pairingChanged && !runtimeChanged) { return null } + return runtimeEnvironmentChangedFailure(environment, method, pairingChanged) +} + +export function runtimeEnvironmentChangedFailure( + environment: Pick, + method: string, + pairingChanged = false +): RuntimeRpcResponse { return { id: method, ok: false, diff --git a/src/main/ipc/runtime-environment-session-reconcile.test.ts b/src/main/ipc/runtime-environment-session-reconcile.test.ts new file mode 100644 index 00000000000..ffd234a7f75 --- /dev/null +++ b/src/main/ipc/runtime-environment-session-reconcile.test.ts @@ -0,0 +1,47 @@ +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import type { ExecutionHostId } from '../../shared/execution-host' +import { addEnvironmentFromPairingCode } from '../../shared/runtime-environment-store' +import { getEnvironmentStorePath } from '../../shared/runtime-environment-store-file' +import { pairingCode } from './runtime-environments-ipc-test-harness' +import { reconcileOrphanedRuntimeSessions } from './runtime-environment-session-reconcile' + +function sessionStore(hostIds: ExecutionHostId[]) { + return { + getWorkspaceSessionHostIds: () => hostIds, + removeWorkspaceSessionHost: vi.fn() + } +} + +describe('reconciling orphaned runtime sessions at startup', () => { + it('drops only runtime sessions whose server is gone, never local or ssh', () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-session-reconcile-')) + const kept = addEnvironmentFromPairingCode(userDataPath, { + name: 'kept', + pairingCode: pairingCode() + }) + const store = sessionStore([ + 'local', + 'ssh:target-1', + `runtime:${kept.id}`, + 'runtime:removed-env' + ]) + expect(reconcileOrphanedRuntimeSessions(store, userDataPath)).toEqual(['runtime:removed-env']) + expect(store.removeWorkspaceSessionHost).toHaveBeenCalledTimes(1) + expect(store.removeWorkspaceSessionHost).toHaveBeenCalledWith('runtime:removed-env') + }) + + it('deletes nothing when the environment store is missing or unreadable', () => { + const missing = mkdtempSync(join(tmpdir(), 'orca-session-reconcile-missing-')) + const store = sessionStore(['runtime:removed-env']) + expect(reconcileOrphanedRuntimeSessions(store, missing)).toEqual([]) + + const corrupt = mkdtempSync(join(tmpdir(), 'orca-session-reconcile-corrupt-')) + writeFileSync(getEnvironmentStorePath(corrupt), '{not json', { mode: 0o600 }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + expect(reconcileOrphanedRuntimeSessions(store, corrupt)).toEqual([]) + expect(store.removeWorkspaceSessionHost).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/runtime-environment-session-reconcile.ts b/src/main/ipc/runtime-environment-session-reconcile.ts new file mode 100644 index 00000000000..9aeb3396519 --- /dev/null +++ b/src/main/ipc/runtime-environment-session-reconcile.ts @@ -0,0 +1,42 @@ +/** + * Startup reconcile for workspace sessions of servers that no longer exist. Unlinking a server drops + * its `runtime:` session, but a crash between the two, or a build that unlinked before that + * drop existed, leaves an orphan that listings keep naming as an unselectable host. + */ +import { existsSync } from 'node:fs' +import { parseExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { getEnvironmentStorePath } from '../../shared/runtime-environment-store-file' + +type SessionHostStore = { + getWorkspaceSessionHostIds: () => ExecutionHostId[] + removeWorkspaceSessionHost: (hostId: ExecutionHostId) => void +} + +/** Returns the dropped hosts. Local and ssh: sessions are never touched. */ +export function reconcileOrphanedRuntimeSessions( + store: SessionHostStore, + userDataPath: string +): ExecutionHostId[] { + // Why both guards: a missing file reads as "no servers", and a session must never be deleted + // on evidence that only means the environment store could not be read. + if (!existsSync(getEnvironmentStorePath(userDataPath))) { + return [] + } + let known: Set + try { + known = new Set(listEnvironments(userDataPath).map((environment) => environment.id)) + } catch (error) { + console.warn('[runtime-environments] skipped orphaned session reconcile:', error) + return [] + } + // Safe because a runtime: session is only ever written after that server is registered. + const dropped = store.getWorkspaceSessionHostIds().filter((hostId) => { + const parsed = parseExecutionHostId(hostId) + return parsed?.kind === 'runtime' && !known.has(parsed.environmentId) + }) + for (const hostId of dropped) { + store.removeWorkspaceSessionHost(hostId) + } + return dropped +} diff --git a/src/main/ipc/runtime-environment-status-owner.ts b/src/main/ipc/runtime-environment-status-owner.ts index f88741ca9a3..96cc5d1b6c0 100644 --- a/src/main/ipc/runtime-environment-status-owner.ts +++ b/src/main/ipc/runtime-environment-status-owner.ts @@ -8,6 +8,7 @@ import { } from '../../shared/runtime-environments' import { recordRuntimeEnvironmentUsage } from './runtime-environment-usage-record' import { RuntimeHostStatusOwner } from '../../shared/runtime-host-status-owner' +import type { RuntimeStatus } from '../../shared/runtime-types' import { RUNTIME_HOST_STATUS_CHANNEL, type RuntimeHostStatusResponse @@ -15,9 +16,12 @@ import { import { applyRuntimeEnvironmentCapabilityVerdict, getAcceptedRuntimeEnvironmentCapabilityOutcome, + getRuntimeEnvironmentCapabilityIncarnation, captureRuntimeEnvironmentCapabilityEvidence } from './runtime-environment-capability-evidence' import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' +import { runtimeEnvironmentChangedFailure } from './runtime-environment-revision-guard' +import { resolveManagedRuntimeEnvironment } from './runtime-environment-managed-tunnel' export function createRuntimeEnvironmentStatusOwner( userDataPath: string, @@ -34,13 +38,23 @@ export function createRuntimeEnvironmentStatusOwner( return new RuntimeHostStatusOwner({ environmentId: environment.id, pairingRevision: environment.pairingRevision ?? environment.createdAt, - request: (signal) => { + request: async (signal) => { + const incarnation = getRuntimeEnvironmentCapabilityIncarnation(environment.id) + const isCurrent = (): boolean => + getRuntimeEnvironmentCapabilityIncarnation(environment.id) === incarnation + if (environment.connectionDependency === 'ssh-tunnel') { + await resolveManagedRuntimeEnvironment(userDataPath, environment.id) + if (!isCurrent()) { + return runtimeEnvironmentChangedFailure(environment, 'status.get') + } + signal.throwIfAborted() + } evidence = captureRuntimeEnvironmentCapabilityEvidence(environment.id, pairing) - return transport.isReady() && - getAcceptedRuntimeEnvironmentCapabilityOutcome(environment.id, pairing, null)?.kind === - 'supported' + const response = await (transport.isReady() && + getAcceptedRuntimeEnvironmentCapabilityOutcome(environment.id, pairing, null)?.kind === + 'supported' ? transport.request(signal) - : sendRemoteRuntimeRequest( + : sendRemoteRuntimeRequest( pairing, 'status.get', undefined, @@ -48,7 +62,8 @@ export function createRuntimeEnvironmentStatusOwner( undefined, signal, ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES - ) + )) + return isCurrent() ? response : runtimeEnvironmentChangedFailure(environment, 'status.get') }, verified: (response, active) => { const capable = @@ -61,7 +76,8 @@ export function createRuntimeEnvironmentStatusOwner( if (accepted && active && !isRuntimeEnvironmentManuallyDisconnected(environment.id)) { recordRuntimeEnvironmentUsage(userDataPath, environment.id, { runtimeId: response._meta.runtimeId, - pairedDeviceId: response.result.pairedDeviceId + pairedDeviceId: response.result.pairedDeviceId, + pairingDeviceToken: pairing.deviceToken }) if (capable) { transport.establish() diff --git a/src/main/ipc/runtime-environment-status-probe.ts b/src/main/ipc/runtime-environment-status-probe.ts new file mode 100644 index 00000000000..54a93822a9e --- /dev/null +++ b/src/main/ipc/runtime-environment-status-probe.ts @@ -0,0 +1,42 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' +import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' +import type { RuntimeStatus } from '../../shared/runtime-types' +import { getRuntimeEnvironmentCapabilityIncarnation } from './runtime-environment-capability-evidence' +import { getRuntimeEnvironmentStatusOwner } from './runtime-environment-request-connections' +import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' +import { runtimeEnvironmentChangedFailure } from './runtime-environment-revision-guard' +import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' +import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' + +export async function getRuntimeEnvironmentStatus( + userDataPath: string, + selector: string, + timeoutMs?: number, + options?: { observeOnly?: true; signal?: AbortSignal; reconnect?: true } +): Promise> { + const environment = resolveEnvironment(userDataPath, selector) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + return { + id: 'status.get', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: 'Runtime environment is manually disconnected.' + } + } + } + const incarnation = getRuntimeEnvironmentCapabilityIncarnation(environment.id) + const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ + timeoutMs, + ...options + }) + // A retired request must not publish old status or borrow its replacement's diagnostics. + if (getRuntimeEnvironmentCapabilityIncarnation(environment.id) !== incarnation) { + return runtimeEnvironmentChangedFailure(environment, 'status.get') + } + return attachRemoteControlDiagnostics( + withTailscaleHintForResponse(response, getPreferredPairingOffer(environment).endpoint), + environment.id + ) +} diff --git a/src/main/ipc/runtime-environment-status-retirement.test.ts b/src/main/ipc/runtime-environment-status-retirement.test.ts new file mode 100644 index 00000000000..37ebf690a60 --- /dev/null +++ b/src/main/ipc/runtime-environment-status-retirement.test.ts @@ -0,0 +1,110 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { + addEnvironmentFromPairingCode, + resolveEnvironment +} from '../../shared/runtime-environment-store' +import { pairingCode } from './runtime-environments-ipc-test-harness' +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { + advanceRuntimeEnvironmentCapabilityIncarnation, + resetRuntimeEnvironmentCapabilityEvidence +} from './runtime-environment-capability-evidence' + +const mocks = vi.hoisted(() => ({ + request: vi.fn(), + ensure: vi.fn(), + reconnect: vi.fn(), + diagnostics: vi.fn() +})) +vi.mock('../../shared/remote-runtime-client', () => ({ sendRemoteRuntimeRequest: mocks.request })) +vi.mock('./runtime-environment-request-connections', async () => { + const { withRuntimeStatusOwners } = await import('./runtime-environments-ipc-test-harness') + return withRuntimeStatusOwners({ + ensureRemoteRuntimeSharedControlConnection: mocks.ensure, + reconnectRemoteRuntimeSharedControlConnection: mocks.reconnect, + getRemoteRuntimeSharedControlDiagnostics: mocks.diagnostics, + pauseRemoteRuntimeSharedControlRetry: vi.fn(), + closeRemoteRuntimeRequestConnection: vi.fn() + }) +}) +vi.mock('electron', () => ({ BrowserWindow: { getAllWindows: () => [] } })) + +import { getRuntimeEnvironmentStatus } from './runtime-environment-transport-routing' +import { resetRuntimeEnvironmentStatusOwners } from './runtime-environment-request-connections' + +let directory: string +let envId: string +beforeEach(() => { + vi.clearAllMocks() + resetRuntimeEnvironmentCapabilityEvidence() + directory = mkdtempSync(join(tmpdir(), 'orca-status-retirement-')) + envId = addEnvironmentFromPairingCode(directory, { + name: 'Host', + now: 1, + pairingCode: pairingCode(), + connectionDependency: 'ssh-tunnel' + }).id + mocks.diagnostics.mockReturnValue(null) +}) +afterEach(() => { + resetRuntimeEnvironmentStatusOwners() + rmSync(directory, { recursive: true, force: true }) +}) + +function response() { + return { + id: 'status', + ok: true as const, + result: { runtimeId: 'old-host', capabilities: [REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY] }, + _meta: { runtimeId: 'old-host' } + } +} + +it.each(['success', 'failure', 'throw'] as const)( + 'fences a retired status probe settling with %s without publishing status or diagnostics', + async (settlement) => { + const pending = Promise.withResolvers() + mocks.request.mockReturnValue(pending.promise) + const probe = getRuntimeEnvironmentStatus(directory, envId) + await vi.waitFor(() => expect(mocks.request).toHaveBeenCalledOnce()) + mocks.diagnostics.mockClear() + advanceRuntimeEnvironmentCapabilityIncarnation(envId) + if (settlement === 'throw') { + pending.reject(new Error('old endpoint offline')) + } else { + pending.resolve( + settlement === 'success' + ? response() + : { + id: 'status', + ok: false, + error: { code: 'offline', message: 'old endpoint' }, + _meta: { runtimeId: 'old-host' } + } + ) + } + await expect(probe).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_environment_changed' } + }) + expect(resolveEnvironment(directory, envId).runtimeId).toBeNull() + expect(mocks.ensure).not.toHaveBeenCalled() + expect(mocks.reconnect).not.toHaveBeenCalled() + expect(mocks.diagnostics).not.toHaveBeenCalled() + } +) + +it('does not fence another environment or a fresh probe after retirement', async () => { + mocks.request.mockImplementation(async () => { + advanceRuntimeEnvironmentCapabilityIncarnation('other') + return response() + }) + await expect(getRuntimeEnvironmentStatus(directory, envId)).resolves.toMatchObject({ ok: true }) + advanceRuntimeEnvironmentCapabilityIncarnation(envId) + await expect(getRuntimeEnvironmentStatus(directory, envId)).resolves.toMatchObject({ ok: true }) + expect(resolveEnvironment(directory, envId).runtimeId).toBe('old-host') + expect(mocks.ensure).toHaveBeenCalledTimes(2) +}) diff --git a/src/main/ipc/runtime-environment-subscription-identity.test.ts b/src/main/ipc/runtime-environment-subscription-identity.test.ts new file mode 100644 index 00000000000..8712b5f1cc3 --- /dev/null +++ b/src/main/ipc/runtime-environment-subscription-identity.test.ts @@ -0,0 +1,130 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' +import type { subscribeRuntimeEnvironment } from './runtime-environment-transport-routing' + +const mocks = vi.hoisted(() => ({ handle: vi.fn(), on: vi.fn(), subscribe: vi.fn() })) +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle, on: mocks.on } })) +vi.mock('../../shared/runtime-environment-store', () => ({ + resolveEnvironment: () => ({ id: 'env', pairingRevision: 1, createdAt: 1 }) +})) +vi.mock('./runtime-environment-transport-routing', () => ({ + subscribeRuntimeEnvironment: mocks.subscribe +})) +import { + registerRuntimeEnvironmentSubscriptionHandlers, + type PendingRuntimeSubscription, + type RetainedRemoteRuntimeSubscription +} from './runtime-environment-subscription-handlers' + +const remoteRuntimeSubscriptions = new Map() +const pendingSubscriptions = new Map() + +type Callbacks = Parameters[5] +const sender = { + id: 1, + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn(), + removeListener: vi.fn() +} +const args = { selector: 'env', method: 'terminal.multiplex', subscriptionId: 'reused-id' } +function handler(channel: string) { + return mocks.handle.mock.calls.find(([name]) => name === channel)![1] +} +const open = () => handler('runtimeEnvironments:subscribe')({ sender }, args) +const unsubscribe = () => + handler('runtimeEnvironments:unsubscribe')({ sender }, { subscriptionId: args.subscriptionId }) +const connection = (): RemoteRuntimeSubscription => ({ + requestId: 'request', + sendBinary: vi.fn(() => true), + close: vi.fn() +}) + +beforeEach(() => { + vi.clearAllMocks() + mocks.subscribe.mockReset() + registerRuntimeEnvironmentSubscriptionHandlers({ + getUserDataPath: () => '/profile', + remoteRuntimeSubscriptions, + pendingSubscriptions + }) +}) +afterEach(() => { + for (const pending of pendingSubscriptions.values()) { + pending.close() + } + pendingSubscriptions.clear() + for (const subscription of remoteRuntimeSubscriptions.values()) { + subscription.close() + } + remoteRuntimeSubscriptions.clear() +}) + +it('reserves an ID while setup is pending so a second open cannot take its ownership', async () => { + const gate = Promise.withResolvers() + mocks.subscribe.mockReturnValue(gate.promise) + const first = open() + await expect(open()).rejects.toThrow('already exists') + expect(mocks.subscribe).toHaveBeenCalledOnce() + gate.resolve(connection()) + await expect(first).resolves.toMatchObject({ subscriptionId: 'reused-id' }) +}) + +it('releases the reservation after failed setup so an explicit retry can open', async () => { + mocks.subscribe.mockRejectedValueOnce(new Error('offline')).mockResolvedValueOnce(connection()) + await expect(open()).rejects.toThrow('offline') + await expect(open()).resolves.toMatchObject({ subscriptionId: 'reused-id' }) +}) + +it('does not let callbacks from an old subscription publish to or remove its replacement', async () => { + const callbacks: Callbacks[] = [] + const currentChecks: (() => boolean)[] = [] + const connections: RemoteRuntimeSubscription[] = [] + mocks.subscribe.mockImplementation( + async (...parameters: Parameters) => { + callbacks.push(parameters[5]) + currentChecks.push(parameters[6]!) + const value = connection() + connections.push(value) + return value + } + ) + await open() + expect(unsubscribe()).toEqual({ unsubscribed: true }) + await open() + sender.send.mockClear() + expect(connections[0].close).toHaveBeenCalledOnce() + expect(currentChecks[1]()).toBe(true) + const payload = { type: 'binary' as const, bytes: new Uint8Array([1]) } + callbacks[0].onEvent(payload) + callbacks[0].onEvent({ type: 'close' }) + callbacks[0].onClose() + expect(sender.send).not.toHaveBeenCalled() + callbacks[1].onEvent(payload) + expect(sender.send).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', { + subscriptionId: 'reused-id', + ...payload + }) + expect(unsubscribe()).toEqual({ unsubscribed: true }) + expect(connections[1].close).toHaveBeenCalledOnce() +}) + +it('does not retain a subscription that closes before setup resolves', async () => { + const value = connection() + mocks.subscribe.mockImplementationOnce( + async (...parameters: Parameters) => { + parameters[5].onEvent({ type: 'close' }) + parameters[5].onClose() + return value + } + ) + // The renderer hears the close, and the id is not kept for a dead connection. + await expect(open()).resolves.toMatchObject({ subscriptionId: 'reused-id' }) + expect(sender.send).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', { + subscriptionId: 'reused-id', + type: 'close' + }) + expect(value.close).toHaveBeenCalledOnce() + mocks.subscribe.mockResolvedValueOnce(connection()) + await expect(open()).resolves.toMatchObject({ subscriptionId: 'reused-id' }) +}) diff --git a/src/main/ipc/runtime-environment-support-routing.ts b/src/main/ipc/runtime-environment-support-routing.ts index c26cb0e0e39..8904bc377ae 100644 --- a/src/main/ipc/runtime-environment-support-routing.ts +++ b/src/main/ipc/runtime-environment-support-routing.ts @@ -1,3 +1,4 @@ +import { isOrchestrationMutation } from '../../shared/orchestration-rpc-contract' import { waitForPromiseWithSignal } from '../../shared/abort-signal-reason' import type { PairingOffer } from '../../shared/pairing' import type { @@ -18,7 +19,10 @@ import { isRuntimeEnvironmentCapabilityOutcomeCurrent, type RuntimeEnvironmentCapabilityOutcome } from './runtime-environment-capability-evidence' -import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' +import { + runtimeEnvironmentChangedFailure, + runtimeEnvironmentRevisionFailure +} from './runtime-environment-revision-guard' import { supportsSharedControl } from './runtime-environment-shared-control-support' import { sendRemoteRuntimeRequestAbortable, @@ -239,21 +243,6 @@ export async function routeRuntimeEnvironmentSubscriptionBySupport { - return { - id: method, - ok: false, - error: { - code: 'runtime_environment_changed', - message: 'Runtime environment pairing changed; refresh and try again' - }, - _meta: { runtimeId: environment.runtimeId } - } -} - function subscriptionCallbacks( args: Pick< Parameters[0], @@ -285,3 +274,13 @@ function subscriptionCallbacks( } } } + +export function shouldUseSharedControlEnvelope( + method: string, + params: unknown, + envelope: RuntimeOrchestrationEnvelope | undefined +): RuntimeOrchestrationEnvelope | undefined { + return envelope && method.startsWith('orchestration.') && !isOrchestrationMutation(method, params) + ? envelope + : undefined +} diff --git a/src/main/ipc/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environment-transport-routing.ts index 59a634064eb..e81e1f64b45 100644 --- a/src/main/ipc/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environment-transport-routing.ts @@ -1,32 +1,28 @@ +import { getRuntimeEnvironmentStatus } from './runtime-environment-status-probe' import { getPreferredPairingOffer } from '../../shared/runtime-environments' import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/electron-remote-runtime-client-capabilities' import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store' +import { resolveManagedRuntimeEnvironment } from './runtime-environment-managed-tunnel' import { recordRuntimeEnvironmentUsage } from './runtime-environment-usage-record' -import { isOrchestrationMutation } from '../../shared/orchestration-rpc-contract' import type { RuntimeOrchestrationEnvelope, RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' -import type { RuntimeStatus } from '../../shared/runtime-types' import { subscribeRemoteRuntimeRequest, type RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import { withRemoteRuntimeTailscaleHint } from '../../shared/remote-runtime-tailscale-hint' import { enqueueRuntimeCall } from './runtime-environment-call-queue' -import { getRuntimeEnvironmentStatusOwner } from './runtime-environment-request-connections' import { sendRemoteRuntimeConnectionRequestAbortable, sendRemoteRuntimeRequestAbortable } from './runtime-environment-abortable-requests' -import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' - -import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-manual-disconnect' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' -import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' import { resetSharedControlSupport } from './runtime-environment-shared-control-support' import { executeSupportRoutedCall, + shouldUseSharedControlEnvelope, shouldRouteCallBySupport, shouldRouteSubscriptionBySupport, subscribeSupportRoutedRuntimeEnvironment @@ -36,32 +32,7 @@ const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000 export { resetSharedControlSupport } -export async function getRuntimeEnvironmentStatus( - userDataPath: string, - selector: string, - timeoutMs?: number, - options?: { observeOnly?: true; signal?: AbortSignal; reconnect?: true } -): Promise> { - const environment = resolveEnvironment(userDataPath, selector) - if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { - return { - id: 'status.get', - ok: false, - error: { - code: 'runtime_manually_disconnected', - message: 'Runtime environment is manually disconnected.' - } - } - } - const response = await getRuntimeEnvironmentStatusOwner(userDataPath, environment.id).refresh({ - timeoutMs, - ...options - }) - return attachRemoteControlDiagnostics( - withTailscaleHintForResponse(response, getPreferredPairingOffer(environment).endpoint), - environment.id - ) -} +export { getRuntimeEnvironmentStatus } from './runtime-environment-status-probe' export async function callRuntimeEnvironment( userDataPath: string, @@ -94,7 +65,10 @@ export async function callRuntimeEnvironment( environment.id, method, async () => { - const currentEnvironment = resolveEnvironment(userDataPath, environment.id) + const currentEnvironment = await resolveManagedRuntimeEnvironment( + userDataPath, + environment.id + ) const revisionFailure = runtimeEnvironmentRevisionFailure( currentEnvironment, expectedEnvironmentPairingRevision, @@ -188,7 +162,7 @@ export async function subscribeRuntimeEnvironment( isCurrent: () => boolean = () => true, signal?: AbortSignal ): Promise { - const environment = resolveEnvironment(userDataPath, selector) + const environment = await resolveManagedRuntimeEnvironment(userDataPath, selector) const pairing = getPreferredPairingOffer(environment) const effectiveTimeoutMs = timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS let markedUsed = false @@ -263,13 +237,3 @@ function markEnvironmentUsedFromResponse( function shouldUseCachedRequestConnection(method: string): boolean { return method === 'terminal.send' || method === 'terminal.updateViewport' } - -function shouldUseSharedControlEnvelope( - method: string, - params: unknown, - envelope: RuntimeOrchestrationEnvelope | undefined -): RuntimeOrchestrationEnvelope | undefined { - return envelope && method.startsWith('orchestration.') && !isOrchestrationMutation(method, params) - ? envelope - : undefined -} diff --git a/src/main/ipc/runtime-environment-usage-record.ts b/src/main/ipc/runtime-environment-usage-record.ts index af56ede3a30..a6bc4f6d475 100644 --- a/src/main/ipc/runtime-environment-usage-record.ts +++ b/src/main/ipc/runtime-environment-usage-record.ts @@ -13,7 +13,7 @@ import { markEnvironmentUsed } from '../../shared/runtime-environment-store' export function recordRuntimeEnvironmentUsage( userDataPath: string, selector: string, - args: { runtimeId?: string | null; pairedDeviceId?: string } = {} + args: Parameters[2] = {} ): void { try { markEnvironmentUsed(userDataPath, selector, args) diff --git a/src/main/ipc/runtime-environments-capability-cache.test.ts b/src/main/ipc/runtime-environments-capability-cache.test.ts index 32f724df981..695eb511bcb 100644 --- a/src/main/ipc/runtime-environments-capability-cache.test.ts +++ b/src/main/ipc/runtime-environments-capability-cache.test.ts @@ -82,6 +82,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { let store: { getSettings: () => { activeRuntimeEnvironmentId: string | null } updateSettings: ReturnType + removeWorkspaceSessionHost: ReturnType } beforeEach(() => { @@ -89,6 +90,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { activeRuntimeEnvironmentId = null store = { getSettings: () => ({ activeRuntimeEnvironmentId }), + removeWorkspaceSessionHost: vi.fn(), updateSettings: vi.fn((updates: { activeRuntimeEnvironmentId: string | null }) => { activeRuntimeEnvironmentId = updates.activeRuntimeEnvironmentId }) diff --git a/src/main/ipc/runtime-environments-pairing.test.ts b/src/main/ipc/runtime-environments-pairing.test.ts index 6919c68f198..d0029c202a3 100644 --- a/src/main/ipc/runtime-environments-pairing.test.ts +++ b/src/main/ipc/runtime-environments-pairing.test.ts @@ -100,6 +100,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { let store: { getSettings: () => { activeRuntimeEnvironmentId: string | null } updateSettings: ReturnType + removeWorkspaceSessionHost: ReturnType } beforeEach(() => { @@ -107,6 +108,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { activeRuntimeEnvironmentId = null store = { getSettings: () => ({ activeRuntimeEnvironmentId }), + removeWorkspaceSessionHost: vi.fn(), updateSettings: vi.fn((updates: { activeRuntimeEnvironmentId: string | null }) => { activeRuntimeEnvironmentId = updates.activeRuntimeEnvironmentId }) @@ -152,6 +154,23 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:retryConnectionsNow', 'runtimeEnvironments:getStatus', 'runtimeEnvironments:call', + 'runtimeEnvironments:linkSshAccess', + 'runtimeEnvironments:unlinkSshAccess', + 'runtimeEnvironments:createOrcadSshHost', + 'runtimeEnvironments:resumeOrcadSshHost', + 'runtimeEnvironments:listPendingOrcadSshProvisioning', + 'runtimeEnvironments:deployOrcad', + 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:convertSshHostToManagedOrcad', + 'runtimeEnvironments:listPendingOrcadMigrations', + 'runtimeEnvironments:previewOrcadDeltaMove', + 'runtimeEnvironments:moveOrcadDelta', + 'runtimeEnvironments:keepOrcadServerVersion', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe' ]) @@ -178,6 +197,23 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', 'runtimeEnvironments:unsubscribe', + 'runtimeEnvironments:linkSshAccess', + 'runtimeEnvironments:unlinkSshAccess', + 'runtimeEnvironments:deployOrcad', + 'runtimeEnvironments:getOrcadStatus', + 'runtimeEnvironments:updateOrcad', + 'runtimeEnvironments:rollbackOrcad', + 'runtimeEnvironments:recoverOrcad', + 'runtimeEnvironments:stopOrcad', + 'runtimeEnvironments:cancelOrcadStop', + 'runtimeEnvironments:convertSshHostToManagedOrcad', + 'runtimeEnvironments:listPendingOrcadMigrations', + 'runtimeEnvironments:previewOrcadDeltaMove', + 'runtimeEnvironments:moveOrcadDelta', + 'runtimeEnvironments:keepOrcadServerVersion', + 'runtimeEnvironments:createOrcadSshHost', + 'runtimeEnvironments:resumeOrcadSshHost', + 'runtimeEnvironments:listPendingOrcadSshProvisioning', 'runtimeEnvironments:retryConnectionsNow' ]) expect(removeAllListenersMock).toHaveBeenCalledWith('runtimeEnvironments:subscriptionBinary') @@ -247,6 +283,10 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) expect(activeRuntimeEnvironmentId).toBeNull() expect(closeRemoteRuntimeRequestConnectionMock).toHaveBeenCalledWith(added.environment.id) + // A removed server's session partition goes too, so listings stop naming it as a host. + expect(store.removeWorkspaceSessionHost).toHaveBeenCalledWith( + `runtime:${encodeURIComponent(added.environment.id)}` + ) expect(JSON.stringify(removed)).not.toContain('device-token') expect(await list(null, undefined)).toEqual([]) }) diff --git a/src/main/ipc/runtime-environments-subscription-teardown.test.ts b/src/main/ipc/runtime-environments-subscription-teardown.test.ts index afb1adf457a..440ed729965 100644 --- a/src/main/ipc/runtime-environments-subscription-teardown.test.ts +++ b/src/main/ipc/runtime-environments-subscription-teardown.test.ts @@ -90,6 +90,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { let store: { getSettings: () => { activeRuntimeEnvironmentId: string | null } updateSettings: ReturnType + removeWorkspaceSessionHost: ReturnType } beforeEach(() => { @@ -97,6 +98,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { activeRuntimeEnvironmentId = null store = { getSettings: () => ({ activeRuntimeEnvironmentId }), + removeWorkspaceSessionHost: vi.fn(), updateSettings: vi.fn((updates: { activeRuntimeEnvironmentId: string | null }) => { activeRuntimeEnvironmentId = updates.activeRuntimeEnvironmentId }) @@ -485,7 +487,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { expect(deliveredCloses).toEqual([]) }) - it('suppresses stale payloads from a retired transport but never re-sends its close', async () => { + it('fences late payloads and duplicate close after full retirement', async () => { registerRuntimeEnvironmentHandlers(store as never) let transportCallbacks: { onResponse: (response: Record) => void @@ -533,11 +535,16 @@ describe('registerRuntimeEnvironmentHandlers', () => { } ) - invalidateRuntimeEnvironmentTransport(added.environment.id) + await invalidateRuntimeEnvironmentTransport(added.environment.id) expect(retirePairedRuntimeBrowserClientHostEnvironmentMock).toHaveBeenCalledWith( added.environment.id, expect.objectContaining({ message: 'Runtime environment transport was invalidated' }) ) + expect(closeRemoteRuntimeRequestConnectionMock).toHaveBeenCalledWith(added.environment.id) + expect(senderSend).toHaveBeenCalledWith('runtimeEnvironments:subscriptionEvent', { + subscriptionId: 'multiplex-stale', + type: 'close' + }) senderSend.mockClear() // A late frame from the retired socket must not reach the renderer... transportCallbacks!.onResponse({ diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 30db92fd48a..9e3506eb5fb 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -19,6 +19,13 @@ import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment import { advanceRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { resetSharedControlSupport } from './runtime-environment-transport-routing' import { RUNTIME_ENVIRONMENT_HANDLER_CHANNELS } from './runtime-environment-handler-channels' +import { registerOrcadRuntimeLifecycleHandlers } from './orcad-runtime-lifecycle-handlers' +import { registerOrcadRuntimeConversionHandlers } from './orcad-runtime-conversion-handlers' +import { registerOrcadDeltaMoveHandlers } from './orcad-delta-move-handlers' +import { registerOrcadRuntimeMaintenanceHandlers } from './orcad-runtime-maintenance-handlers' +import { clearPublishedManagedServer } from './ssh-renderer-broadcast' +import { reconcileOrphanedRuntimeSessions } from './runtime-environment-session-reconcile' +import { registerRuntimeSshAccessHandlers } from './runtime-ssh-access-handlers' import { retirePairedRuntimeBrowserClientHostEnvironment } from '../browser/paired-runtime-browser-client-host-runtime' import { registerRuntimeEnvironmentBrowserClientHostHandler } from './runtime-environment-browser-client-host-handler' import { advanceRuntimeEnvironmentCapabilityIncarnation } from './runtime-environment-capability-evidence' @@ -92,6 +99,7 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { ipcMain.removeHandler(channel) } ipcMain.removeAllListeners('runtimeEnvironments:subscriptionBinary') + reconcileOrphanedRuntimeSessions(store, getUserDataPath()) registerRuntimeEnvironmentConnectivityHandlers({ store, @@ -109,6 +117,20 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { getRuntimeEnvironmentStatusOwner(getUserDataPath(), environment.id).activate() } } + registerRuntimeSshAccessHandlers({ + getUserDataPath, + invalidateTransport: invalidateRuntimeEnvironmentTransport + }) + registerOrcadRuntimeLifecycleHandlers({ getUserDataPath }) + registerOrcadRuntimeConversionHandlers(getUserDataPath) + registerOrcadDeltaMoveHandlers(getUserDataPath) + registerOrcadRuntimeMaintenanceHandlers({ + getUserDataPath, + getActiveEnvironmentId: () => store.getSettings().activeRuntimeEnvironmentId, + invalidateTransport: invalidateRuntimeEnvironmentTransport, + clearHostServerStatus: clearPublishedManagedServer, + forgetHostSession: (hostId) => store.removeWorkspaceSessionHost(hostId) + }) registerRuntimeEnvironmentSubscriptionHandlers({ getUserDataPath, remoteRuntimeSubscriptions, diff --git a/src/main/ipc/runtime-ssh-access-handlers.test.ts b/src/main/ipc/runtime-ssh-access-handlers.test.ts new file mode 100644 index 00000000000..16da2a6105e --- /dev/null +++ b/src/main/ipc/runtime-ssh-access-handlers.test.ts @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ handle: vi.fn(), link: vi.fn(), unlink: vi.fn() })) +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/runtime-ssh-access', () => ({ + linkRuntimeSshAccess: mocks.link, + unlinkRuntimeSshAccess: mocks.unlink +})) +import { registerRuntimeSshAccessHandlers } from './runtime-ssh-access-handlers' +import { EventEmitter } from 'node:events' + +describe('existing paired server SSH access IPC', () => { + const invalidateTransport = vi.fn() + const request = { + selector: 'host', + requestId: 'request-1', + sshTargetId: 'ssh-host', + remotePort: 6768 + } + beforeEach(() => { + vi.clearAllMocks() + registerRuntimeSshAccessHandlers({ + getUserDataPath: () => '/test-profile', + invalidateTransport + }) + }) + const sender = new EventEmitter() + const event = { sender } + function handler( + channel: string + ): (_event: typeof event | null, input: unknown) => Promise { + const registered = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registered) { + throw new Error('Handler missing') + } + return registered[1] + } + + it('registers access-only actions and forwards only validated arguments and main-owned invalidation', async () => { + expect(mocks.handle.mock.calls.map(([name]) => name)).toEqual([ + 'runtimeEnvironments:linkSshAccess', + 'runtimeEnvironments:unlinkSshAccess' + ]) + const result = { + id: 'host', + endpoints: [{ id: 'ssh-endpoint', endpoint: 'ws://127.0.0.1:41000' }] + } + mocks.link.mockResolvedValue(result) + expect(await handler('runtimeEnvironments:linkSshAccess')(event, request)).toBe(result) + expect(mocks.link).toHaveBeenCalledExactlyOnceWith('/test-profile', request, { + signal: expect.any(AbortSignal), + invalidateTransport + }) + expect(sender.listenerCount('destroyed')).toBe(0) + await handler('runtimeEnvironments:unlinkSshAccess')(null, { + selector: 'host', + requestId: 'unlink-1' + }) + expect(mocks.unlink).toHaveBeenCalledExactlyOnceWith( + '/test-profile', + { selector: 'host', requestId: 'unlink-1' }, + { invalidateTransport } + ) + }) + + it.each([ + { remotePort: 0 }, + { remotePort: 65536 }, + { requestId: '../request' }, + { verifiedRuntimeId: 'renderer-supplied' }, + { userDataPath: '/other-profile' }, + { owner: { type: 'orcad-runtime', environmentId: 'other' } } + ])('refuses invalid or authority-bearing link arguments: %j', async (change) => { + await expect( + handler('runtimeEnvironments:linkSshAccess')(null, { ...request, ...change }) + ).rejects.toThrow() + expect(mocks.link).not.toHaveBeenCalled() + }) + + it('cancels a pending link when its window closes', async () => { + let signal: AbortSignal | undefined + mocks.link.mockImplementation(async (_path, _args, options: { signal: AbortSignal }) => { + signal = options.signal + sender.emit('destroyed') + return null + }) + await handler('runtimeEnvironments:linkSshAccess')(event, request) + expect(signal?.aborted).toBe(true) + }) + + it('refuses renderer-supplied unlink snapshots', async () => { + await expect( + handler('runtimeEnvironments:unlinkSshAccess')(null, { + selector: 'host', + requestId: 'unlink-1', + expectedEnvironment: { id: 'other' } + }) + ).rejects.toThrow() + expect(mocks.unlink).not.toHaveBeenCalled() + }) + + it('propagates pending failures instead of reporting a successful link', async () => { + const error = new Error('SSH endpoint identity was not verified') + mocks.link.mockRejectedValue(error) + await expect(handler('runtimeEnvironments:linkSshAccess')(event, request)).rejects.toBe(error) + }) +}) diff --git a/src/main/ipc/runtime-ssh-access-handlers.ts b/src/main/ipc/runtime-ssh-access-handlers.ts new file mode 100644 index 00000000000..b3f09cfba42 --- /dev/null +++ b/src/main/ipc/runtime-ssh-access-handlers.ts @@ -0,0 +1,33 @@ +import { ipcMain } from 'electron' +import { + RuntimeSshAccessLinkRequestSchema, + RuntimeSshAccessUnlinkRequestSchema +} from '../../shared/runtime-ssh-access' +import { linkRuntimeSshAccess, unlinkRuntimeSshAccess } from '../ssh/runtime-ssh-access' + +export function registerRuntimeSshAccessHandlers(options: { + getUserDataPath: () => string + invalidateTransport: (environmentId: string) => void | Promise +}): void { + ipcMain.handle('runtimeEnvironments:linkSshAccess', async (event, input: unknown) => { + const args = RuntimeSshAccessLinkRequestSchema.parse(input) + // Why: a link to an unreachable host holds two lifecycle queues; a closed window cancels it. + const controller = new AbortController() + const cancel = (): void => controller.abort() + event.sender.once('destroyed', cancel) + try { + return await linkRuntimeSshAccess(options.getUserDataPath(), args, { + signal: controller.signal, + invalidateTransport: options.invalidateTransport + }) + } finally { + event.sender.removeListener('destroyed', cancel) + } + }) + ipcMain.handle('runtimeEnvironments:unlinkSshAccess', async (_event, input: unknown) => { + const args = RuntimeSshAccessUnlinkRequestSchema.parse(input) + return unlinkRuntimeSshAccess(options.getUserDataPath(), args, { + invalidateTransport: options.invalidateTransport + }) + }) +} diff --git a/src/main/ipc/runtime-watcher-disposal-owners.test.ts b/src/main/ipc/runtime-watcher-disposal-owners.test.ts new file mode 100644 index 00000000000..a889db510a4 --- /dev/null +++ b/src/main/ipc/runtime-watcher-disposal-owners.test.ts @@ -0,0 +1,103 @@ +import { expect, it, vi } from 'vitest' +import { RuntimeWatcherDisposalOwners } from './runtime-watcher-disposal-owners' + +function owner() { + return { + dispose: vi.fn(), + subscribe: vi.fn(), + disposeAndWait: vi.fn(async () => {}) + } +} + +it('retains retired cleanup and joins concurrent shutdown callers', async () => { + const owners = new RuntimeWatcherDisposalOwners() + const retired = owner() + const pending = Promise.withResolvers() + retired.disposeAndWait.mockReturnValue(pending.promise) + owners.retire(retired) + const shutdown = owners.disposeAndWait(() => {}) + expect(owners.disposeAndWait(() => {})).toBe(shutdown) + const finished = vi.fn() + const result = shutdown.then(finished) + await Promise.resolve() + expect(finished).not.toHaveBeenCalled() + expect(retired.disposeAndWait).toHaveBeenCalledOnce() + pending.resolve() + await result + await owners.disposeAndWait(() => {}) + expect(retired.disposeAndWait).toHaveBeenCalledOnce() +}) + +it('does not retry a newly failed attempt until a later explicit shutdown call', async () => { + const owners = new RuntimeWatcherDisposalOwners() + const failed = owner() + const sibling = owner() + const pending = Promise.withResolvers() + const failure = new Error('child still live') + failed.disposeAndWait.mockRejectedValueOnce(failure) + sibling.disposeAndWait.mockReturnValue(pending.promise) + const finished = vi.fn() + const shutdown = owners.disposeAndWait(() => { + owners.retire(failed) + owners.retire(sibling) + }) + const result = shutdown.catch((error: unknown) => { + finished() + return error + }) + await new Promise((resolve) => setImmediate(resolve)) + expect(finished).not.toHaveBeenCalled() + expect(owners.disposeAndWait(() => {})).toBe(shutdown) + expect(failed.disposeAndWait).toHaveBeenCalledOnce() + pending.resolve() + expect(await result).toMatchObject({ errors: [failure] }) + await owners.disposeAndWait(() => {}) + expect(failed.disposeAndWait).toHaveBeenCalledTimes(2) + expect(sibling.disposeAndWait).toHaveBeenCalledOnce() +}) + +it('retains synchronous failures and attempts sibling owners', async () => { + const owners = new RuntimeWatcherDisposalOwners() + const failed = owner() + const sibling = owner() + failed.disposeAndWait.mockImplementationOnce(() => { + throw new Error('sync failure') + }) + await expect( + owners.disposeAndWait(() => { + owners.retire(failed) + owners.retire(sibling) + }) + ).rejects.toThrow('watcher_pool_shutdown_incomplete') + expect(sibling.disposeAndWait).toHaveBeenCalledOnce() + await owners.disposeAndWait(() => {}) + expect(failed.disposeAndWait).toHaveBeenCalledTimes(2) +}) + +it('publishes the retained attempt before a synchronous disposal callback reenters', async () => { + const owners = new RuntimeWatcherDisposalOwners() + const child = owner() + const pending = Promise.withResolvers() + let nested: Promise | undefined + child.disposeAndWait.mockImplementation(() => { + owners.retire(child) + nested = owners.disposeAndWait(() => {}) + return pending.promise + }) + const shutdown = owners.disposeAndWait(() => owners.retire(child)) + expect(nested).toBe(shutdown) + expect(child.disposeAndWait).toHaveBeenCalledOnce() + pending.resolve() + await shutdown +}) + +it('refuses to acknowledge a supervisor without a physical-disposal API', async () => { + const owners = new RuntimeWatcherDisposalOwners() + const legacy = { dispose: vi.fn(), subscribe: vi.fn() } + await expect(owners.disposeAndWait(() => owners.retire(legacy))).rejects.toMatchObject({ + errors: [ + expect.objectContaining({ message: 'watcher_supervisor_awaited_disposal_unavailable' }) + ] + }) + expect(legacy.dispose).toHaveBeenCalledOnce() +}) diff --git a/src/main/ipc/runtime-watcher-disposal-owners.ts b/src/main/ipc/runtime-watcher-disposal-owners.ts new file mode 100644 index 00000000000..e88c741844f --- /dev/null +++ b/src/main/ipc/runtime-watcher-disposal-owners.ts @@ -0,0 +1,90 @@ +import type { RuntimeWatcherPoolSupervisor } from './runtime-watcher-pool-state' + +type Attempt = { pending: Promise | null; failure?: unknown } + +type Completion = { promise: Promise; resolve: () => void; reject: (error: unknown) => void } + +// Why not Promise.withResolvers: the relay bundles this pool and still targets Node 18 hosts. +function createCompletion(): Completion { + let resolve!: () => void + let reject!: (error: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +export class RuntimeWatcherDisposalOwners { + private readonly retained = new Map() + private shutdown: Promise | null = null + + retire(owner: RuntimeWatcherPoolSupervisor): void { + if (!this.retained.has(owner)) { + this.start(owner) + } + } + + disposeAndWait(disposePool: () => void): Promise { + if (this.shutdown) { + return this.shutdown + } + const retry = [...this.retained].filter(([, attempt]) => !attempt.pending) + const completion = createCompletion() + this.shutdown = completion.promise + const failures: unknown[] = [] + try { + disposePool() + } catch (error) { + failures.push(error) + } + for (const [owner, attempt] of retry) { + if (this.retained.get(owner) === attempt) { + this.start(owner) + } + } + const pending = [...this.retained.values()].map( + (attempt) => attempt.pending ?? Promise.reject(attempt.failure) + ) + void Promise.allSettled(pending).then((results) => { + for (const result of results) { + if (result.status === 'rejected') { + failures.push(result.reason) + } + } + this.shutdown = null + if (failures.length > 0) { + completion.reject(new AggregateError(failures, 'watcher_pool_shutdown_incomplete')) + } else { + completion.resolve() + } + }) + return completion.promise + } + + private start(owner: RuntimeWatcherPoolSupervisor): void { + const completion = createCompletion() + const attempt: Attempt = { pending: completion.promise } + this.retained.set(owner, attempt) + void completion.promise.catch(() => {}) + const failed = (error: unknown): void => { + attempt.pending = null + attempt.failure = error + completion.reject(error) + } + try { + if (!owner.disposeAndWait) { + owner.dispose() + throw new Error('watcher_supervisor_awaited_disposal_unavailable') + } + void owner.disposeAndWait().then(() => { + if (this.retained.get(owner) === attempt) { + this.retained.delete(owner) + } + completion.resolve() + }, failed) + } catch (error) { + failed(error) + } + } +} diff --git a/src/main/ipc/runtime-watcher-pool-shutdown.test.ts b/src/main/ipc/runtime-watcher-pool-shutdown.test.ts new file mode 100644 index 00000000000..1f37f3540c5 --- /dev/null +++ b/src/main/ipc/runtime-watcher-pool-shutdown.test.ts @@ -0,0 +1,79 @@ +import { expect, it, vi } from 'vitest' +import { RuntimeWatcherProcessPool } from './runtime-watcher-process-pool' +import type { WatcherProcessHooks } from './parcel-watcher-process-subscription' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' + +function supervisor() { + const pending = Promise.withResolvers() + let hooks: WatcherProcessHooks | undefined + return { + pending, + dispose: vi.fn(), + disposeAndWait: vi.fn(() => pending.promise), + subscribe: vi.fn( + async ( + _dir: string, + _callback: unknown, + _options: unknown, + options?: WatcherProcessHooks + ) => { + hooks = options + return { unsubscribe: vi.fn(async () => {}) } + } + ), + fail: () => + hooks?.onTerminalError?.( + new WatcherProcessFailure('failed', 'supervisor', 'process_unavailable') + ) + } +} + +it('awaits retired and replacement supervisors after logical retirement removed the old slot', async () => { + const old = supervisor() + const current = supervisor() + const create = vi.fn().mockReturnValueOnce(old).mockReturnValueOnce(current) + const pool = new RuntimeWatcherProcessPool({ createSupervisor: create }) + await pool.subscribe('/first', vi.fn(), {}) + old.fail() + await new Promise((resolve) => setImmediate(resolve)) + expect(old.disposeAndWait).toHaveBeenCalledOnce() + await pool.subscribe('/second', vi.fn(), {}) + const finished = vi.fn() + const shutdown = pool.disposeAndWait() + expect(pool.disposeAndWait()).toBe(shutdown) + const result = shutdown.then(finished) + current.pending.resolve() + await new Promise((resolve) => setImmediate(resolve)) + expect(finished).not.toHaveBeenCalled() + await expect(pool.subscribe('/third', vi.fn(), {})).rejects.toThrow('disposed') + expect(create).toHaveBeenCalledTimes(2) + old.pending.resolve() + await result +}) + +it('retains failure after synchronous pool disposal and retries it explicitly', async () => { + const child = supervisor() + const failure = new Error('child still running') + child.disposeAndWait.mockRejectedValueOnce(failure) + const pool = new RuntimeWatcherProcessPool({ createSupervisor: () => child }) + await pool.subscribe('/first', vi.fn(), {}) + const shutdown = pool.disposeAndWait() + await expect(shutdown).rejects.toMatchObject({ errors: [failure] }) + const retry = pool.disposeAndWait() + expect(child.disposeAndWait).toHaveBeenCalledTimes(2) + child.pending.resolve() + await retry +}) + +it('does not duplicate termination when a queued retirement races with synchronous disposal', async () => { + const child = supervisor() + const pool = new RuntimeWatcherProcessPool({ createSupervisor: () => child }) + await pool.subscribe('/first', vi.fn(), {}) + child.fail() + pool.dispose() + const shutdown = pool.disposeAndWait() + await new Promise((resolve) => setImmediate(resolve)) + expect(child.disposeAndWait).toHaveBeenCalledOnce() + child.pending.resolve() + await shutdown +}) diff --git a/src/main/ipc/runtime-watcher-pool-state.ts b/src/main/ipc/runtime-watcher-pool-state.ts index e6c016b9844..a99c9b93895 100644 --- a/src/main/ipc/runtime-watcher-pool-state.ts +++ b/src/main/ipc/runtime-watcher-pool-state.ts @@ -1,6 +1,7 @@ import type { WatcherProcessSupervisor } from './parcel-watcher-process-supervisor' -export type RuntimeWatcherPoolSupervisor = Pick +export type RuntimeWatcherPoolSupervisor = Pick & + Partial> export type RuntimeWatcherPoolSlot = { supervisor: RuntimeWatcherPoolSupervisor @@ -10,6 +11,13 @@ export type RuntimeWatcherPoolSlot = { disposed: boolean } +export function activeWatcherSlots( + slots: ReadonlySet, + isolated: boolean +): RuntimeWatcherPoolSlot[] { + return [...slots].filter((slot) => slot.isolated === isolated && !slot.retired) +} + export type RuntimeWatcherPoolAssignment = { slot: RuntimeWatcherPoolSlot leases: number diff --git a/src/main/ipc/runtime-watcher-process-pool.ts b/src/main/ipc/runtime-watcher-process-pool.ts index 4bafc38208d..66957d4e7e8 100644 --- a/src/main/ipc/runtime-watcher-process-pool.ts +++ b/src/main/ipc/runtime-watcher-process-pool.ts @@ -7,15 +7,17 @@ import type { WatcherProcessSubscription } from './parcel-watcher-process-subscription' import { RuntimeWatcherPendingAssignment } from './runtime-watcher-pending-assignment' +import { RuntimeWatcherDisposalOwners } from './runtime-watcher-disposal-owners' import { RuntimeWatcherPoolLifecycle } from './runtime-watcher-pool-lifecycle' import { RuntimeWatcherPredecessorBarriers } from './runtime-watcher-predecessor-barriers' import { RuntimeWatcherQuarantineQueue } from './runtime-watcher-quarantine-queue' import { handleRuntimeWatcherSubscriptionFailure } from './runtime-watcher-subscription-failure' -import type { - RuntimeWatcherPoolAssignment, - RuntimeWatcherPoolSlot, - RuntimeWatcherPoolSupervisor, - RuntimeWatcherProcessPoolOptions +import { + activeWatcherSlots, + type RuntimeWatcherPoolAssignment, + type RuntimeWatcherPoolSlot, + type RuntimeWatcherPoolSupervisor, + type RuntimeWatcherProcessPoolOptions } from './runtime-watcher-pool-state' export type { RuntimeWatcherProcessPoolOptions } from './runtime-watcher-pool-state' @@ -39,6 +41,7 @@ export class RuntimeWatcherProcessPool { RuntimeWatcherPendingAssignment >() private readonly lifecycle = new RuntimeWatcherPoolLifecycle() + private disposalOwners = new RuntimeWatcherDisposalOwners() private readonly predecessorBarriers = new RuntimeWatcherPredecessorBarriers() private readonly quarantineQueue: RuntimeWatcherQuarantineQueue @@ -142,9 +145,15 @@ export class RuntimeWatcherProcessPool { resetForTest(): void { this.dispose() + this.disposalOwners = new RuntimeWatcherDisposalOwners() this.lifecycle.reset() } + disposeAndWait = (): Promise => this.disposalOwners.disposeAndWait(() => this.dispose()) + + /** Accepts subscriptions again after a disposal the owner decided not to follow with exit. */ + reopen = (): void => this.lifecycle.reset() + forgetRoot(dir: string): void { // Physical subscriptions release their assignment through unsubscribe or // terminal callbacks. This clears fault history after setup gives up. @@ -221,9 +230,7 @@ export class RuntimeWatcherProcessPool { } private sharedSlot(): RuntimeWatcherPoolSlot { - const sharedSlots = Array.from(this.activeSlots).filter( - (slot) => !slot.isolated && !slot.retired - ) + const sharedSlots = activeWatcherSlots(this.activeSlots, false) if (sharedSlots.length < this.maxSharedSupervisors) { return this.createSlot(false) } @@ -233,9 +240,7 @@ export class RuntimeWatcherProcessPool { } private quarantineSlot(dir: string): RuntimeWatcherPoolSlot | Promise { - const quarantineSlots = Array.from(this.activeSlots).filter( - (slot) => slot.isolated && !slot.retired - ) + const quarantineSlots = activeWatcherSlots(this.activeSlots, true) if (quarantineSlots.length < this.maxQuarantineSupervisors) { return this.createSlot(true) } @@ -273,7 +278,11 @@ export class RuntimeWatcherProcessPool { slot.roots.clear() // Why: failAllSubscriptions is still iterating callbacks; defer disposal // so every logical root receives the supervisor failure first. + const owners = this.disposalOwners queueMicrotask(() => { + if (this.disposalOwners !== owners) { + return + } this.disposeSlot(slot) this.drainQuarantineWaiters() }) @@ -308,8 +317,7 @@ export class RuntimeWatcherProcessPool { private drainQuarantineWaiters(): void { while ( this.quarantineQueue.length > 0 && - Array.from(this.activeSlots).filter((slot) => slot.isolated && !slot.retired).length < - this.maxQuarantineSupervisors + activeWatcherSlots(this.activeSlots, true).length < this.maxQuarantineSupervisors ) { this.quarantineQueue.grantNext(this.createSlot(true)) } @@ -320,7 +328,7 @@ export class RuntimeWatcherProcessPool { return } slot.disposed = true - slot.supervisor.dispose() + this.disposalOwners.retire(slot.supervisor) this.allSlots.delete(slot) } } diff --git a/src/main/ipc/session.test.ts b/src/main/ipc/session.test.ts new file mode 100644 index 00000000000..32f4ac8d6c6 --- /dev/null +++ b/src/main/ipc/session.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from 'vitest' + +const { handlers } = vi.hoisted(() => ({ + handlers: new Map unknown>() +})) + +vi.mock('electron', () => ({ + ipcMain: { + handle: vi.fn( + ( + channel: string, + handler: (event: { returnValue?: unknown }, ...args: unknown[]) => unknown + ) => { + handlers.set(channel, handler) + } + ), + on: vi.fn( + ( + channel: string, + handler: (event: { returnValue?: unknown }, ...args: unknown[]) => unknown + ) => { + handlers.set(channel, handler) + } + ) + } +})) + +import { registerSessionHandlers } from './session' + +describe('registerSessionHandlers', () => { + it('drops renderer writes to a fenced host source partition and keeps every other one', async () => { + const store = { + getSshTarget: vi.fn((id: string) => + id === 'fenced' ? { orcadFence: { environmentId: 'env-1' } } : {} + ), + setWorkspaceSession: vi.fn(), + patchWorkspaceSession: vi.fn(), + flushPendingOrThrowAsync: vi.fn(() => Promise.resolve()) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the handlers under test touch only the store methods stubbed above and never the runtime. + registerSessionHandlers(store as never, {} as never) + + for (const hostId of ['ssh:fenced', 'ssh:open', undefined]) { + await handlers.get('session:set')?.({}, {}, hostId) + await handlers.get('session:patch')?.({}, {}, hostId) + } + expect(store.setWorkspaceSession.mock.calls.map(([, hostId]) => hostId)).toEqual([ + 'ssh:open', + undefined + ]) + expect(store.patchWorkspaceSession.mock.calls.map(([, hostId]) => hostId)).toEqual([ + 'ssh:open', + undefined + ]) + + const event: { returnValue?: unknown } = {} + handlers.get('session:set-sync')?.(event, {}, 'ssh:fenced') + await vi.waitFor(() => expect(event.returnValue).toBe(true)) + expect(store.setWorkspaceSession).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/ipc/session.ts b/src/main/ipc/session.ts index 6c26bff4670..11b118c8731 100644 --- a/src/main/ipc/session.ts +++ b/src/main/ipc/session.ts @@ -2,6 +2,7 @@ import { ipcMain } from 'electron' import type { Store } from '../persistence' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { parseTerminalSurfaceCloseTarget } from '../../shared/terminal-surface-close-target' +import { isFrozenOrcadSourceSessionPartition } from '../ssh/orcad-retained-source' import { markAgentLaunchesClosedByUser } from '../agent-launch/agent-launch-pane-attachment' import type { WorkspaceSessionPatch, @@ -9,6 +10,10 @@ import type { } from '../../shared/workspace-session-state-types' export function registerSessionHandlers(store: Store, runtime: OrcaRuntimeService): void { + // Why: renderer saves would change a fenced host's frozen source partition. + const isFenced = (hostId?: string | null): boolean => + isFrozenOrcadSourceSessionPartition(store, hostId) + // Why: hostId is an optional second arg so an older renderer that invokes // these channels without it keeps reading/writing the 'local' partition // exactly as before. Channel names stay stable. @@ -24,11 +29,15 @@ export function registerSessionHandlers(store: Store, runtime: OrcaRuntimeServic }) ipcMain.handle('session:set', (_event, args: WorkspaceSessionState, hostId?: string | null) => { - store.setWorkspaceSession(args, hostId) + if (!isFenced(hostId)) { + store.setWorkspaceSession(args, hostId) + } }) ipcMain.handle('session:patch', (_event, args: WorkspaceSessionPatch, hostId?: string | null) => { - store.patchWorkspaceSession(args, hostId) + if (!isFenced(hostId)) { + store.patchWorkspaceSession(args, hostId) + } }) // Why: a renderer save cannot shrink membership main owns, so each close commits it explicitly. @@ -63,7 +72,9 @@ export function registerSessionHandlers(store: Store, runtime: OrcaRuntimeServic ipcMain.on('session:set-sync', (event, args: WorkspaceSessionState, hostId?: string | null) => { void (async () => { try { - store.setWorkspaceSession(args, hostId) + if (!isFenced(hostId)) { + store.setWorkspaceSession(args, hostId) + } await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) } catch (error) { console.error('[persistence] Failed to flush legacy session checkpoint:', error) diff --git a/src/main/ipc/ssh-active-relay-sessions.ts b/src/main/ipc/ssh-active-relay-sessions.ts index 69b781be41d..3c12496f7d4 100644 --- a/src/main/ipc/ssh-active-relay-sessions.ts +++ b/src/main/ipc/ssh-active-relay-sessions.ts @@ -1,5 +1,8 @@ import type { SshRelaySession } from '../ssh/ssh-relay-session' -import { setSshActiveMultiplexerResolver } from '../ssh/ssh-target-registry' +import { + setDirectSshAuthorityResolver, + setSshActiveMultiplexerResolver +} from '../ssh/ssh-target-registry' import { setWorktreeRemovalSshHostHomeResolver } from '../worktree-removal-execution-host-route' // One session per SSH target owns the whole relay lifecycle (mux, providers, abort controller, state machine). @@ -22,3 +25,5 @@ export function getActiveSshHostHomeDirectory(targetId: string): string | null { } setWorktreeRemovalSshHostHomeResolver(getActiveSshHostHomeDirectory) + +setDirectSshAuthorityResolver((targetId) => activeSessions.has(targetId)) diff --git a/src/main/ipc/ssh-app-shutdown.test.ts b/src/main/ipc/ssh-app-shutdown.test.ts index b587d666a6c..5842f37fb08 100644 --- a/src/main/ipc/ssh-app-shutdown.test.ts +++ b/src/main/ipc/ssh-app-shutdown.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-connect-attempt-registry.ts b/src/main/ipc/ssh-connect-attempt-registry.ts index 5a1e3632982..5aba526d683 100644 --- a/src/main/ipc/ssh-connect-attempt-registry.ts +++ b/src/main/ipc/ssh-connect-attempt-registry.ts @@ -44,3 +44,44 @@ export const resetRelayInFlight = new Map>() // Why: ssh:testConnection connects then disconnects; suppressing broadcasts during the test avoids worktree cards flashing connected → disconnected. export const testingTargets = new Set() export const testConnectionProbes = new Set>() +const testConnectionProbesByTarget = new Map>>() + +export function hasSshTestConnectionProbes(targetId: string): boolean { + return (testConnectionProbesByTarget.get(targetId)?.size ?? 0) > 0 +} + +export function runSshTestConnectionProbe( + targetId: string, + operation: () => Promise +): Promise { + const probes = testConnectionProbesByTarget.get(targetId) ?? new Set>() + let probe!: Promise + // Publish before connection callbacks can start a reset or shutdown drain. + probe = Promise.resolve() + .then(operation) + .finally(() => { + testConnectionProbes.delete(probe) + probes.delete(probe) + if (probes.size === 0 && testConnectionProbesByTarget.get(targetId) === probes) { + testConnectionProbesByTarget.delete(targetId) + testingTargets.delete(targetId) + credentialRequestedForTarget.delete(targetId) + } + }) + probes.add(probe) + testConnectionProbesByTarget.set(targetId, probes) + testConnectionProbes.add(probe) + testingTargets.add(targetId) + return probe +} + +/** Reserve target admission before joining; failed probes still own cleanup until settled. */ +export async function awaitSshTestConnectionProbes(targetId: string): Promise { + while (true) { + const probes = testConnectionProbesByTarget.get(targetId) + if (!probes?.size) { + return + } + await Promise.allSettled(probes) + } +} diff --git a/src/main/ipc/ssh-connect-flow.ts b/src/main/ipc/ssh-connect-flow.ts index 882d3c9ff91..281ada6ec7c 100644 --- a/src/main/ipc/ssh-connect-flow.ts +++ b/src/main/ipc/ssh-connect-flow.ts @@ -9,7 +9,17 @@ import { isCurrentSshProviderAuthority, rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' +import { allowsDirectSshRelay } from '../ssh/ssh-connection-store' +import { adoptSshConnection, runAttributedToSshOwner } from '../ssh/ssh-connection-attribution' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { + decideHostServer, + recheckWhenManagedFenceClears, + publishHostServerDecisionFailure, + refineRelayTerminalDecision, + publishManagedServerConnect, + recordRelayDecision +} from './ssh-host-server-connect' import { activeSessions } from './ssh-active-relay-sessions' import { assertSshConnectsNotFenced, @@ -39,7 +49,11 @@ import { getPublicSshState, relayStateOverrides } from './ssh-renderer-broadcast' -import { abandonCancelledConnectAttempt, abandonFailedSshSession } from './ssh-session-teardown' +import { + abandonCancelledConnectAttempt, + abandonDecisionTransport, + abandonFailedSshSession +} from './ssh-session-teardown' import { awaitTargetLifecycle } from './ssh-target-lifecycle-queue' export async function connectTarget(targetId: string): Promise { @@ -164,6 +178,59 @@ async function doConnect( } } + // Why before the decision: a transport the relay connect below reuses was not this attempt's. + const priorConnection = connectionManager!.getConnection(targetId) + // A transport the decision's census, deploy or conversion opens is attributed to this attempt, + // so a cancelled attempt closes exactly that one and nothing a newer owner took over. + const owner = Symbol(targetId) + // Why after the teardown above: deploy and conversion refuse while a direct session or transport + // exists, and the authority rotated synchronously so concurrent connects still join this one. + const server = await runAttributedToSshOwner(owner, () => decideHostServer(target)).catch( + async (error: unknown) => { + if (!isCurrentConnectAttempt(targetId, authority)) { + await abandonDecisionTransport(targetId, owner, authority) + throw createCancelledConnectAttemptError() + } + // A failed setup leaves no relay to own the transport its decision dialed. + await abandonDecisionTransport(targetId, owner, authority) + publishHostServerDecisionFailure(targetId, error) + throw error + } + ) + // A shutdown that began during the decision is the actionable reason, ahead of the rotation. + assertSshConnectsNotFenced() + if (!isCurrentConnectAttempt(targetId, authority)) { + await abandonDecisionTransport(targetId, owner, authority) + throw createCancelledConnectAttemptError() + } + adoptCurrentTransport(targetId, owner) + if (server?.route === 'managed') { + if (server.fenceHeld) { + recheckWhenManagedFenceClears(target, server.environmentId) + } + return publishManagedServerConnect( + targetId, + server.environmentId, + server.update, + server.serving + ) + } + if (server) { + recordRelayDecision(target, server) + } + // Re-read: a conversion attempt may have fenced the host since the lookup above. + const relayTarget = getSshTargetRegistryStore()!.getTarget(targetId) ?? target + if (!allowsDirectSshRelay(relayTarget)) { + // A setup that failed but kept its fence: the relay decision's detail is the real cause. + const blocked = new Error( + server?.detail ?? + 'This SSH host serves a managed Orca server; it is reached through that server.' + ) + await abandonDecisionTransport(targetId, owner, authority) + publishHostServerDecisionFailure(targetId, blocked) + throw blocked + } + // Why here and not only at entry: this is the publication point, and it is the last statement // before the transport opens. Checking it in the same synchronous block as activeSessions.set // means a connect either registers before the shutdown drain snapshots, or registers never and @@ -183,14 +250,12 @@ async function doConnect( const ownsSession = (): boolean => isCurrentConnectAttempt(targetId, authority) && activeSessions.get(targetId) === session - // Why captured here and not with existingState: connect() reuses an already-connected transport, - // and only a transport this attempt opened is this attempt's to close when it loses the race. - const priorConnection = connectionManager!.getConnection(targetId) const mintedConnection = (): SshConnection | null => conn && conn !== priorConnection ? conn : null try { conn = await connectionManager!.connect(target) + adoptSshConnection(conn, owner) if (!ownsSession()) { throw createCancelledConnectAttemptError() } @@ -228,6 +293,11 @@ async function doConnect( if (!ownsSession()) { throw createCancelledConnectAttemptError() } + await refineRelayTerminalDecision(target, server, ownsSession) + // The re-check can wait seconds on the relay; a connect cancelled meanwhile must not report. + if (!ownsSession()) { + throw createCancelledConnectAttemptError() + } // Why: we manually pushed `deploying-relay`, so send `connected` straight to the renderer — routing through onStateChange would trigger reconnect logic. clearRelayStateOverride(targetId) @@ -265,3 +335,10 @@ async function doConnect( return getPublicSshState(targetId)! } + +function adoptCurrentTransport(targetId: string, owner: symbol): void { + const current = connectionManager!.getConnection(targetId) + if (current) { + adoptSshConnection(current, owner) + } +} diff --git a/src/main/ipc/ssh-connect-recheck-cancellation.test.ts b/src/main/ipc/ssh-connect-recheck-cancellation.test.ts new file mode 100644 index 00000000000..e55a00cdf89 --- /dev/null +++ b/src/main/ipc/ssh-connect-recheck-cancellation.test.ts @@ -0,0 +1,228 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = await vi.hoisted(async () => { + const { createSshIpcMocks } = await import('./ssh-ipc-module-mocks') + return createSshIpcMocks() +}) + +vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) +vi.mock('electron', () => mocks.electron) +vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) +vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) +vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) +vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) +vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) +vi.mock('../ssh/ssh-channel-multiplexer', () => mocks.sshChannelMultiplexer) +vi.mock('../providers/ssh-pty-provider', () => mocks.sshPtyProvider) +vi.mock('../providers/ssh-filesystem-provider', () => mocks.sshFilesystemProvider) +vi.mock('./pty', () => mocks.pty) +vi.mock('../providers/ssh-filesystem-dispatch', () => mocks.sshFilesystemDispatch) +vi.mock('../providers/ssh-git-provider', () => mocks.sshGitProvider) +vi.mock('../providers/ssh-git-dispatch', () => mocks.sshGitDispatch) +vi.mock('../ssh/ssh-port-forward', () => mocks.sshPortForward) +vi.mock('../ssh/ssh-port-scanner', () => mocks.sshPortScanner) + +import type { SshTarget } from '../../shared/ssh-types' +import type { SshConnection } from '../ssh/ssh-connection' +import { recordSshConnectionOpened } from '../ssh/ssh-connection-attribution' +import { + decideHostServer, + publishHostServerDecisionFailure, + refineRelayTerminalDecision +} from './ssh-host-server-connect' +import { createSshIpcHarness } from './ssh-ipc-test-harness' + +const { mockSshStore, mockConnectionManager, mockDeployAndLaunchRelay } = mocks + +describe('a connect cancelled during the relay-terminal re-check', () => { + const harness = createSshIpcHarness(mocks) + const { handlers, createRelayLaunchResult } = harness + + beforeEach(harness.reset) + + it('never reports connected for a connect cancelled during the relay-terminal re-check', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const conn = { id: 'rechecking-transport' } + let releaseRecheck = (): void => {} + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.getConnection.mockReturnValue(undefined) + mockConnectionManager.connect.mockResolvedValue(conn) + mockConnectionManager.disconnect.mockResolvedValue(undefined) + mockDeployAndLaunchRelay.mockResolvedValueOnce(createRelayLaunchResult()) + vi.mocked(refineRelayTerminalDecision).mockImplementationOnce( + () => new Promise((resolve) => (releaseRecheck = resolve)) + ) + const connect = Promise.resolve(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })) + await vi.waitFor(() => expect(refineRelayTerminalDecision).toHaveBeenCalled()) + await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' }) + releaseRecheck() + + await expect(connect).rejects.toThrow('SSH connection attempt was cancelled') + expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', conn) + }) + + it('publishes the error when a managed host fails to set up, rather than staying connecting', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const failure = new Error('listen EADDRINUSE 127.0.0.1:46768') + mockSshStore.getTarget.mockReturnValue(target) + vi.mocked(decideHostServer).mockRejectedValueOnce(failure) + await expect(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })).rejects.toBe(failure) + expect(publishHostServerDecisionFailure).toHaveBeenCalledWith('ssh-1', failure) + }) + + it('publishes the setup failure when a failed setup kept the host fenced', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy', + orcadFence: { environmentId: 'env-1' } + } + mockSshStore.getTarget.mockReturnValue(target) + vi.mocked(decideHostServer).mockResolvedValueOnce({ + route: 'relay', + reason: 'failed', + detail: 'The destination could not stage the migration.' + }) + await expect(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })).rejects.toThrow( + 'The destination could not stage the migration.' + ) + expect(publishHostServerDecisionFailure).toHaveBeenCalledWith( + 'ssh-1', + expect.objectContaining({ message: 'The destination could not stage the migration.' }) + ) + }) + + it('closes a transport the cancelled decision opened after the user disconnected', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const opened = { id: 'census-transport' } + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.getConnection.mockReturnValue(undefined) + mockConnectionManager.disconnect.mockResolvedValue(undefined) + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + // The real decision awaits its module loads before any census, as here. + await Promise.resolve() + await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' }) + // The census dials after the teardown, opening a transport no one else holds. + recordSshConnectionOpened(asTransport(opened)) + mockConnectionManager.getConnection.mockReturnValue(opened) + return null + }) + await expect( + Promise.resolve(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })) + ).rejects.toThrow('SSH connection attempt was cancelled') + expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', opened) + }) + + it('leaves a transport a completed replacement connect adopted from the stale decision', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const opened = { id: 'shared-transport' } + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.getConnection.mockReturnValue(undefined) + mockConnectionManager.disconnect.mockResolvedValue(undefined) + mockConnectionManager.connect.mockResolvedValue(opened) + mockDeployAndLaunchRelay.mockResolvedValue(createRelayLaunchResult()) + let resumeStale = (): void => {} + let staleDecided = false + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + await Promise.resolve() + staleDecided = true + recordSshConnectionOpened(asTransport(opened)) + mockConnectionManager.getConnection.mockReturnValue(opened) + await new Promise((resolve) => (resumeStale = resolve)) + return null + }) + const stale = Promise.resolve(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })) + await vi.waitFor(() => expect(staleDecided).toBe(true)) + await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' }) + // The replacement reuses the pooled transport and completes, leaving connectInFlight. + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + mockConnectionManager.disconnectConnection.mockClear() + resumeStale() + await expect(stale).rejects.toThrow('SSH connection attempt was cancelled') + expect(mockConnectionManager.disconnectConnection).not.toHaveBeenCalledWith('ssh-1', opened) + }) + + it('closes the transport a still-current decision dialed when that decision fails', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy', + orcadFence: { environmentId: 'env-1' } + } + const opened = { id: 'tunnel-transport' } + const failure = new Error('listen EADDRINUSE 127.0.0.1:46768') + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.getConnection.mockReturnValue(undefined) + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + await Promise.resolve() + recordSshConnectionOpened(asTransport(opened)) + mockConnectionManager.getConnection.mockReturnValue(opened) + throw failure + }) + await expect( + Promise.resolve(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })) + ).rejects.toBe(failure) + expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', opened) + }) + + it('closes the transport a fenced failed setup dialed before reporting it', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy', + orcadFence: { environmentId: 'env-1' } + } + const opened = { id: 'conversion-transport' } + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.getConnection.mockReturnValue(undefined) + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + await Promise.resolve() + recordSshConnectionOpened(asTransport(opened)) + mockConnectionManager.getConnection.mockReturnValue(opened) + return { route: 'relay', reason: 'failed', detail: 'Staging failed.' } + }) + await expect( + Promise.resolve(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })) + ).rejects.toThrow('Staging failed.') + expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', opened) + }) +}) + +/** The stand-in transports these tests hand the mocked pool. */ +type FakeTransport = { id: string } + +function asTransport(value: FakeTransport): SshConnection { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: attribution keys on identity only. + return value as unknown as SshConnection +} diff --git a/src/main/ipc/ssh-connection-handlers.ts b/src/main/ipc/ssh-connection-handlers.ts index 9ea533c77c3..b4244a99733 100644 --- a/src/main/ipc/ssh-connection-handlers.ts +++ b/src/main/ipc/ssh-connection-handlers.ts @@ -1,39 +1,26 @@ +import { isLiveSshPtyLease } from '../../shared/ssh-pty-lease-liveness' import { ipcMain } from 'electron' -import { - sshRemotePtyLeaseAllowsReattach, - type SshTarget, - type SshTerminateSessionsResult -} from '../../shared/ssh-types' -import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' -import { isSshPtyNotFoundError } from '../providers/ssh-pty-errors' -import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' +import type { SshTarget } from '../../shared/ssh-types' +import { toAppSshPtyId } from '../providers/ssh-pty-id' import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' import { forceStopRelayForTarget } from '../ssh/ssh-relay-reset' import { setSshTargetRegistryHandlers, getSshTargetRegistryStore } from '../ssh/ssh-target-registry' -import { - clearProviderPtyState, - deletePtyOwnership, - getPtyIdsForConnection, - getSshPtyProvider -} from './pty' +import { clearProviderPtyState, deletePtyOwnership, getPtyIdsForConnection } from './pty' import { activeSessions } from './ssh-active-relay-sessions' import { assertSshConnectsNotFenced, connectInFlight, credentialRequestedForTarget, - invalidateConnectAttempt, resetRelayInFlight, - testConnectionProbes, - testingTargets + runSshTestConnectionProbe } from './ssh-connect-attempt-registry' import { connectTarget } from './ssh-connect-flow' import { connectionManager, persistedStore } from './ssh-ipc-context' import { getPublicSshState } from './ssh-renderer-broadcast' -import { - disconnectRegisteredSshTarget, - teardownActiveSshSession, - teardownSshTargetTransport -} from './ssh-session-teardown' +import { disconnectRegisteredSshTarget, teardownActiveSshSession } from './ssh-session-teardown' +import { terminateSshTargetSessions } from './ssh-terminate-sessions' +import { assertNotManagedServerHost } from './ssh-target-crud-handlers' +import { moveSshHostToManagedServer } from './ssh-managed-server-move' import { runTargetLifecycle } from './ssh-target-lifecycle-queue' async function doResetRelay(targetId: string, target: SshTarget): Promise { @@ -80,7 +67,7 @@ async function doResetRelay(targetId: string, target: SshTarget): Promise // (docs/reference/ssh-execution-boundary.md). Nothing here can adopt a stranger either — the // replacement relay namespaces every id under a fresh mint epoch, so an old orphan lease can // only fail its next reattach. - if (lease.state !== 'terminated' && lease.state !== 'expired') { + if (isLiveSshPtyLease(lease)) { ptyIds.add(lease.ptyId) // Why: only a host-acknowledged force-stop may retire a lease. When it threw we never // observed those shells, so expiring them would record a verdict we do not hold; mirrors @@ -116,85 +103,20 @@ export function registerSshConnectionHandlers(): void { await disconnectRegisteredSshTarget(args.targetId) }) - ipcMain.handle('ssh:terminateSessions', async (_event, args: { targetId: string }) => { - invalidateConnectAttempt(args.targetId) - // Why (#12661): an offline sweep tears down local transport only. The caller must be able to tell - // "the host stopped these" from "nobody asked the host", so carry the verdict out of the lifecycle queue. - let outcome: SshTerminateSessionsResult = { terminated: 0, unverifiable: 0 } - await runTargetLifecycle(args.targetId, async () => { - const provider = getSshPtyProvider(args.targetId) - const leases = persistedStore!.getSshRemotePtyLeases(args.targetId) - const ptyIdsByRelayId = new Map() - // Why: only leases the app still believes it owns may force a reconnect; a lease whose route - // died for good is swept opportunistically instead, so a target that can no longer answer - // never blocks its own removal (issue #2626, and the renderer tolerates the refusal there). - const ownedRelayIds = new Set() - const trackPtyId = (ptyId: string, owned: boolean): void => { - const relayPtyId = toRelaySshPtyId(args.targetId, ptyId) - if (!ptyIdsByRelayId.has(relayPtyId)) { - ptyIdsByRelayId.set(relayPtyId, toAppSshPtyId(args.targetId, ptyId)) - } - if (owned) { - ownedRelayIds.add(relayPtyId) - } + ipcMain.handle( + 'ssh:terminateSessions', + (_event, args: { targetId: string; forRemoval?: boolean }) => { + // Why here: only main sees a fence that landed after the renderer loaded its targets. + if (args.forRemoval) { + assertNotManagedServerHost(args.targetId) } - for (const ptyId of getPtyIdsForConnection(args.targetId)) { - trackPtyId(ptyId, true) - } - for (const lease of leases) { - if (lease.state === 'terminated') { - continue - } - // Why the predicate and not `state !== 'expired'`: an `expired` lease carrying no - // retirement mark records only that reattach gave up, never that the remote shell died, so - // it is exactly the orphan the user's terminate must reach — and reaching it needs the - // relay, which is what the fence below demands. Only `supersededBy` / `relayIdRecycled` - // prove the route is dead for good, and those stay unowned. - trackPtyId(lease.ptyId, sshRemotePtyLeaseAllowsReattach(lease)) - } - const ptyIds = Array.from(ptyIdsByRelayId, ([relayPtyId, appPtyId]) => ({ - relayPtyId, - appPtyId - })) + return terminateSshTargetSessions(args.targetId) + } + ) - if (ownedRelayIds.size > 0 && !provider) { - throw new Error( - `${SSH_TERMINATE_RECONNECT_REQUIRED}: SSH relay is not connected; reconnect before terminating remote sessions.` - ) - } - const shutdownResults = provider - ? await Promise.allSettled( - ptyIds.map(({ appPtyId }) => - provider.shutdown(appPtyId, { immediate: true, keepHistory: false }) - ) - ) - : [] - if (!provider) { - // Nothing observed these remote shells, so their state is unknown — not "nothing to do". - outcome = { terminated: 0, unverifiable: ptyIds.length } - } - const shutdownFailures: string[] = [] - for (const [index, result] of shutdownResults.entries()) { - const { appPtyId, relayPtyId } = ptyIds[index] - if (result.status !== 'fulfilled' && !isSshPtyNotFoundError(result.reason)) { - shutdownFailures.push( - `${relayPtyId}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}` - ) - continue - } - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - persistedStore!.markSshRemotePtyLease(args.targetId, relayPtyId, 'terminated') - outcome = { ...outcome, terminated: outcome.terminated + 1 } - } - if (shutdownFailures.length > 0) { - // Why: a failed relay shutdown can leave the remote process alive in the grace window; keep the lease/session so the user can retry. - throw new Error(`Failed to terminate SSH host sessions: ${shutdownFailures.join('; ')}`) - } - await teardownSshTargetTransport(args.targetId, (session) => session.disposeAndPersist()) - }) - return outcome - }) + ipcMain.handle('ssh:moveToManagedServer', (_event, args: { targetId: string }) => + moveSshHostToManagedServer(args.targetId) + ) ipcMain.handle('ssh:resetRelay', (_event, args: { targetId: string }) => { const existingReset = resetRelayInFlight.get(args.targetId) @@ -266,18 +188,16 @@ export function registerSshConnectionHandlers(): void { } } - testingTargets.add(args.targetId) // Why a tracked promise and not just the id: a probe holds a real transport that no session owns, // so shutdown has to be able to join it before the final drain disconnects what is left. - const probe = (async () => { + const probe = runSshTestConnectionProbe(args.targetId, async () => { // Why: a probe transport opened after the shutdown drain would outlive orderly teardown. assertSshConnectsNotFenced() const conn = await connectionManager!.connect(target) const state = conn.getState() await connectionManager!.disconnect(args.targetId) return state - })() - testConnectionProbes.add(probe) + }) try { return { success: true, state: await probe } } catch (err) { @@ -285,11 +205,6 @@ export function registerSshConnectionHandlers(): void { success: false, error: err instanceof Error ? err.message : String(err) } - } finally { - testConnectionProbes.delete(probe) - testingTargets.delete(args.targetId) - // Why: clear so a test's credential prompt doesn't leave lastRequiredPassphrase=true and defer this target at startup. - credentialRequestedForTarget.delete(args.targetId) } }) } diff --git a/src/main/ipc/ssh-connection-state-callbacks.ts b/src/main/ipc/ssh-connection-state-callbacks.ts index 5daa25ed2b1..51c68b94832 100644 --- a/src/main/ipc/ssh-connection-state-callbacks.ts +++ b/src/main/ipc/ssh-connection-state-callbacks.ts @@ -2,6 +2,7 @@ import type { SshConnectionCallbacks } from '../ssh/ssh-connection' import type { SshConnectionState, SshTarget } from '../../shared/ssh-types' import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { isSshHostCensusInFlight } from '../ssh/ssh-connection-attribution' import { activeSessions } from './ssh-active-relay-sessions' import { connectInFlight, @@ -79,10 +80,10 @@ export function handleSshConnectionStateChange(targetId: string, state: SshConne return } else if ( state.status === 'connected' && - session !== undefined && + // No session yet: the server decision's census, deploy or conversion opened the transport. sessionState !== 'ready' && !completedTransportReconnect && - connectInFlight.has(targetId) + (connectInFlight.has(targetId) || isSshHostCensusInFlight(targetId)) ) { // Why: the raw SSH transport reaches 'connected' before the relay session establishes during an // explicit connect. Forwarding it makes the renderer treat the host as fully up — it remounts @@ -96,7 +97,8 @@ export function handleSshConnectionStateChange(targetId: string, state: SshConne clearRelayStateOverride(targetId) broadcastSshState(getCurrentMainWindow, targetId, { targetId, - status: 'deploying-relay', + // Before any session the connect is still deciding the host's server. + status: session ? 'deploying-relay' : 'connecting', error: state.error, reconnectAttempt: state.reconnectAttempt }) diff --git a/src/main/ipc/ssh-disconnect-cancellation.test.ts b/src/main/ipc/ssh-disconnect-cancellation.test.ts index ae18a14debe..5448580c7ed 100644 --- a/src/main/ipc/ssh-disconnect-cancellation.test.ts +++ b/src/main/ipc/ssh-disconnect-cancellation.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-handler-reregistration.test.ts b/src/main/ipc/ssh-handler-reregistration.test.ts index da790d78256..52c5ca366b2 100644 --- a/src/main/ipc/ssh-handler-reregistration.test.ts +++ b/src/main/ipc/ssh-handler-reregistration.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-host-server-connect.ts b/src/main/ipc/ssh-host-server-connect.ts new file mode 100644 index 00000000000..4d1ff5ba22c --- /dev/null +++ b/src/main/ipc/ssh-host-server-connect.ts @@ -0,0 +1,162 @@ +/** The connect path's managed-server step: decide the host's server, and publish a managed connect. */ +import { getAppEnvironment } from '../../shared/app-environment' +import type { + SshConnectionState, + SshManagedServerServingNote, + SshManagedServerUpdateNote, + SshTarget +} from '../../shared/ssh-types' +import type { HostServerOnConnectResult } from '../ssh/ssh-host-server-on-connect' +import { relayServerStatus, shouldToastManagedServerMove } from '../ssh/ssh-host-server-move-offer' +import { + clearSshHostServerStatus, + getSshHostServerStatus, + setSshHostServerStatus +} from '../ssh/ssh-host-server-status' +import { trackSshHostServerMove } from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { allowsDirectSshRelay } from '../ssh/ssh-connection-store' +import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context' +import { + broadcastSshState, + clearRelayStateOverride, + getPublicSshState +} from './ssh-renderer-broadcast' +import { isAuthError } from '../ssh/ssh-connection-utils' + +/** Resolves null when the decision couldn't run on a host that may still use the relay. */ +export async function decideHostServer( + target: SshTarget +): Promise { + try { + // Why lazy: the managed-server graph (deploy, migration, tunnel) loads only when a host connects. + const [{ resolveHostServerOnConnect }, { hostServerOnConnectDeps }] = await Promise.all([ + import('../ssh/ssh-host-server-on-connect'), + import('./ssh-host-server-on-connect-wiring') + ]) + return await resolveHostServerOnConnect( + target, + hostServerOnConnectDeps(getAppEnvironment().getPath('userData')) + ) + } catch (error) { + // A host with no relay fallback surfaces the real failure (auth, unreachable), not a generic one. + if (!allowsDirectSshRelay(getSshTargetRegistryStore()?.getTarget(target.id) ?? target)) { + throw error + } + console.warn('[ssh] Could not decide the managed Orca server for this host:', error) + return null + } +} + +/** Rechecks a host another desktop's update held during this connect, until the fence clears. */ +export function recheckWhenManagedFenceClears(target: SshTarget, environmentId: string): void { + void Promise.all([ + import('../ssh/managed-server-fence-recheck'), + import('./ssh-host-server-on-connect-wiring') + ]).then(([{ recheckFencedManagedServer, scheduleManagedServerFenceRecheck }, wiring]) => { + const deps = wiring.hostServerOnConnectDeps(getAppEnvironment().getPath('userData')) + scheduleManagedServerFenceRecheck(target.id, { + stillCurrent: () => { + const status = getSshHostServerStatus(target.id) + return ( + connectionManager?.getState(target.id)?.status === 'connected' && + status?.kind === 'managed' && + status.environmentId === environmentId + ) + }, + recheck: () => recheckFencedManagedServer(target, environmentId, deps), + publish: (result) => + publishManagedServerConnect(target.id, environmentId, result.update, result.serving) + }) + }) +} + +/** + * A host only its managed server reaches failed to set up: leave 'connecting' and the 'setting + * up' status for the error, as a failed transport connect does. + */ +export function publishHostServerDecisionFailure(targetId: string, error: unknown): void { + const failure = error instanceof Error ? error : new Error(String(error)) + clearSshHostServerStatus(targetId) + clearRelayStateOverride(targetId) + broadcastSshState(getCurrentMainWindow, targetId, { + targetId, + status: isAuthError(failure) ? 'auth-failed' : 'error', + error: failure.message, + reconnectAttempt: 0 + }) +} + +export function publishManagedServerConnect( + targetId: string, + environmentId: string, + update?: SshManagedServerUpdateNote, + serving?: SshManagedServerServingNote +): SshConnectionState { + const managedServer = { + kind: 'managed' as const, + environmentId, + ...(update ? { update } : {}), + ...(serving ? { serving } : {}) + } + setSshHostServerStatus(targetId, managedServer) + const state: SshConnectionState = { + ...(connectionManager!.getState(targetId) ?? { targetId, reconnectAttempt: 0 }), + targetId, + status: 'connected', + error: null, + managedServer + } + broadcastSshState(getCurrentMainWindow, targetId, state) + return getPublicSshState(targetId) ?? state +} + +/** Records why the host keeps the relay; the first live-terminals stop this version offers a move. */ +export function recordRelayDecision( + target: SshTarget, + decision: Extract +): void { + const appVersion = getAppEnvironment().getVersion() + const offerMove = shouldToastManagedServerMove(target, decision, appVersion) + if (offerMove) { + getSshTargetRegistryStore()!.updateTarget(target.id, { + managedServerMoveOffered: { appVersion } + }) + trackSshHostServerMove('offered', knownSshHostPlatform(target.id)) + } + setSshHostServerStatus(target.id, relayServerStatus(decision, offerMove)) +} + +/** After the relay session is up, a terminal the first decision could not ask about may prove live. */ +export async function refineRelayTerminalDecision( + target: SshTarget, + decision: HostServerOnConnectResult | null, + isCurrent: () => boolean +): Promise { + try { + const [ + { relayTerminalsOnceConnected }, + { orcadMigrationRelayPtyLister }, + { censusHostRelayTerminalsFor } + ] = await Promise.all([ + import('../ssh/ssh-host-relay-terminals-once-connected'), + import('../ssh/orcad-migration-relay-pty-lister'), + import('../ssh/ssh-host-relay-census-for-target') + ]) + const refined = await relayTerminalsOnceConnected({ + store: getSshTargetRegistryStore()!.getOrcadMigrationSource(), + targetId: target.id, + decision, + listRelayPtyIds: orcadMigrationRelayPtyLister(target.id), + censusHost: censusHostRelayTerminalsFor(target), + isCurrent + }) + if (refined && isCurrent()) { + recordRelayDecision(target, refined) + } + } catch (error) { + // The first decision's status stands; it already keeps the host on the relay. + console.warn('[ssh] Could not re-check relay terminals after connecting:', error) + } +} diff --git a/src/main/ipc/ssh-host-server-decision-failure.test.ts b/src/main/ipc/ssh-host-server-decision-failure.test.ts new file mode 100644 index 00000000000..ff74ae9a41e --- /dev/null +++ b/src/main/ipc/ssh-host-server-decision-failure.test.ts @@ -0,0 +1,62 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' + +const mocks = vi.hoisted(() => ({ + getTarget: vi.fn(), + resolve: vi.fn() +})) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getPath: () => '/tmp/user-data', getVersion: () => '1.5.0' }) +})) +vi.mock('../ssh/ssh-target-registry', () => ({ + getSshTargetRegistryStore: () => ({ getTarget: mocks.getTarget }) +})) +vi.mock('../ssh/ssh-host-server-on-connect', () => ({ + resolveHostServerOnConnect: mocks.resolve +})) +vi.mock('./ssh-host-server-on-connect-wiring', () => ({ hostServerOnConnectDeps: () => ({}) })) +vi.mock('./ssh-ipc-context', () => ({ connectionManager: null, getCurrentMainWindow: () => null })) +vi.mock('./ssh-renderer-broadcast', () => ({ + broadcastSshState: vi.fn(), + clearRelayStateOverride: vi.fn(), + getPublicSshState: vi.fn() +})) + +const { decideHostServer, publishHostServerDecisionFailure } = + await import('./ssh-host-server-connect') +const { broadcastSshState } = await import('./ssh-renderer-broadcast') +const { getSshHostServerStatus, setSshHostServerStatus } = + await import('../ssh/ssh-host-server-status') + +const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } +const fenced: SshTarget = { + ...target, + orcadFence: { environmentId: 'env-1' } +} + +beforeEach(() => vi.clearAllMocks()) + +describe('a managed-server decision that fails', () => { + it('keeps the relay path on a host that may still use it', async () => { + mocks.getTarget.mockReturnValue(target) + mocks.resolve.mockRejectedValue(new Error('All configured authentication methods failed')) + await expect(decideHostServer(target)).resolves.toBeNull() + }) + + it('surfaces the real error on a host only its managed server can reach', async () => { + mocks.getTarget.mockReturnValue(fenced) + const auth = new Error('All configured authentication methods failed') + mocks.resolve.mockRejectedValue(auth) + await expect(decideHostServer(target)).rejects.toBe(auth) + }) + + it('replaces a stuck setting-up status with the error once setup fails', () => { + setSshHostServerStatus('ssh-1', { kind: 'setting-up', phase: 'connecting' }) + publishHostServerDecisionFailure('ssh-1', new Error('listen EADDRINUSE')) + expect(getSshHostServerStatus('ssh-1')).toBeUndefined() + expect(vi.mocked(broadcastSshState).mock.calls.at(-1)?.[2]).toMatchObject({ + status: 'error', + error: 'listen EADDRINUSE' + }) + }) +}) diff --git a/src/main/ipc/ssh-host-server-on-connect-wiring.test.ts b/src/main/ipc/ssh-host-server-on-connect-wiring.test.ts new file mode 100644 index 00000000000..4b15671de24 --- /dev/null +++ b/src/main/ipc/ssh-host-server-on-connect-wiring.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { OrcadHostUnsupportedError } from '../ssh/orcad-host-unavailable' +import { allowsDirectSshRelay, SshConnectionStore } from '../ssh/ssh-connection-store' +import { resolveHostServerOnConnect } from '../ssh/ssh-host-server-on-connect' + +const mocks = vi.hoisted(() => { + const registry: { current: unknown } = { current: null } + return { + registry, + deploy: vi.fn(), + convert: vi.fn(), + tunnel: vi.fn(async () => undefined), + broadcast: vi.fn() + } +}) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getVersion: () => '1.5.0', getPath: () => '/tmp/unused' }) +})) +vi.mock('../ssh/ssh-target-registry', () => ({ + getSshTargetRegistryStore: () => mocks.registry.current, + hasRegisteredDirectSshAuthority: () => false +})) +vi.mock('../ssh/orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('../ssh/orcad-runtime-conversion', () => ({ + convertSshTargetToManagedOrcad: mocks.convert +})) +vi.mock('../ssh/orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.tunnel })) +vi.mock('../ssh/orcad-artifact-materializer', () => ({ hasOrcadTemplate: () => true })) +vi.mock('./ssh-renderer-broadcast', () => ({ broadcastSshState: mocks.broadcast })) +vi.mock('./ssh-ipc-context', () => ({ getCurrentMainWindow: () => null })) +vi.mock('./ssh-session-teardown', () => ({ disconnectRegisteredSshTarget: vi.fn() })) + +const { hostServerOnConnectDeps } = await import('./ssh-host-server-on-connect-wiring') +const { connectInFlight } = await import('./ssh-connect-attempt-registry') +const { getSshHostServerStatus } = await import('../ssh/ssh-host-server-status') + +const TARGET: SshTarget = { + id: 'ssh-box', + label: 'Box', + host: 'box.example.com', + port: 22, + username: 'me', + generation: 3 +} + +let userDataPath: string +let store: Store + +beforeEach(() => { + vi.clearAllMocks() + userDataPath = mkdtempSync(join(tmpdir(), 'host-server-wiring-')) + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + mocks.registry.current = new SshConnectionStore(store as never) +}) + +afterEach(async () => { + await closeTestStores() + rmSync(userDataPath, { recursive: true, force: true }) +}) + +const target = (): SshTarget => store.getSshTarget(TARGET.id)! + +describe('connect-time server decision against the real profile', () => { + it('releases an empty host claim when orcad cannot run, records why, and keeps the relay', async () => { + mocks.deploy.mockImplementation(async () => { + // The real deploy claims the host before it finds the template missing the target. + store.updateSshTarget(TARGET.id, { + orcadFence: { environmentId: 'env-new' }, + orcadProvisioning: { requestId: 'env-new', name: 'Box' } + }) + throw new OrcadHostUnsupportedError('Packaged orcad template does not support x') + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await expect( + resolveHostServerOnConnect(target(), hostServerOnConnectDeps(userDataPath)) + ).resolves.toEqual({ route: 'relay', reason: 'orcad_unavailable', detail: 'unsupported_host' }) + warn.mockRestore() + expect(target().orcadFence).toBeUndefined() + expect(target().managedServerUnavailable).toEqual({ + reason: 'unsupported_host', + appVersion: '1.5.0' + }) + expect(allowsDirectSshRelay(target())).toBe(true) + + // Not retried on the next connect of the same build. + mocks.deploy.mockClear() + await resolveHostServerOnConnect(target(), hostServerOnConnectDeps(userDataPath)) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('retries a host an older build kept on the relay', async () => { + const deps = hostServerOnConnectDeps(userDataPath) + store.updateSshTarget(TARGET.id, { + managedServerUnavailable: { reason: 'unsupported_host', appVersion: '1.4.0' } + }) + expect(deps.recordedUnavailable(target())).toBeNull() + }) + + it('keeps the relay while a saved relay terminal is unproven, without converting', async () => { + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + store.upsertSshRemotePtyLease({ targetId: TARGET.id, ptyId: 'pty-1', state: 'expired' }) + await expect( + resolveHostServerOnConnect(target(), hostServerOnConnectDeps(userDataPath)) + ).resolves.toEqual({ route: 'relay', reason: 'relay_terminals_unverifiable', terminals: 1 }) + expect(mocks.convert).not.toHaveBeenCalled() + }) + + it('drops progress from a decision whose connect was cancelled', () => { + const deps = hostServerOnConnectDeps(userDataPath) + deps.progress(target(), 'deploying') + expect(getSshHostServerStatus(TARGET.id)).toBeUndefined() + expect(mocks.broadcast).not.toHaveBeenCalled() + + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: progress reads only presence. + connectInFlight.set(TARGET.id, {} as never) + try { + deps.progress(target(), 'deploying') + expect(getSshHostServerStatus(TARGET.id)).toEqual({ kind: 'setting-up', phase: 'deploying' }) + expect(mocks.broadcast).toHaveBeenCalledTimes(1) + } finally { + connectInFlight.delete(TARGET.id) + } + }) +}) diff --git a/src/main/ipc/ssh-host-server-on-connect-wiring.ts b/src/main/ipc/ssh-host-server-on-connect-wiring.ts new file mode 100644 index 00000000000..b13a9bc256d --- /dev/null +++ b/src/main/ipc/ssh-host-server-on-connect-wiring.ts @@ -0,0 +1,163 @@ +/** The live collaborators behind each connect's server decision. */ +import { getAppEnvironment } from '../../shared/app-environment' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { findOrcadMigrationSourceCutoverForTarget } from '../ssh/orcad-migration-cutover-journal' +import { orcadMigrationRelayPtyLister } from '../ssh/orcad-migration-relay-pty-lister' +import { abandonOrcadConversion } from '../ssh/orcad-conversion-abandon' +import { retainOrcadMigrationSource } from '../ssh/orcad-migration-source-retention' +import { hasOrcadTemplate } from '../ssh/orcad-artifact-materializer' +import { managedServerUpdateDeps } from '../ssh/managed-server-update-deps' +import { ensureOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' +import { verifyOrcadManagedServing } from '../ssh/orcad-managed-serving-verify' +import { convertSshTargetToManagedOrcad } from '../ssh/orcad-runtime-conversion' +import { orcadMigrationDestinationFor } from '../ssh/orcad-runtime-conversion-wiring' +import { createManagedOrcadEnvironment } from '../ssh/orcad-runtime-deployment' +import type { HostServerOnConnectDeps } from '../ssh/ssh-host-server-on-connect' +import { + censusSshHostRelaysBeforeSession, + relayTerminalsOnConnect +} from '../ssh/ssh-host-relay-terminals-on-connect' +import { requireManagedOrcadInfrastructure } from '../ssh/orcad-managed-runtime-context' +import { censusHostRelayTerminalsFor } from '../ssh/ssh-host-relay-census-for-target' +import { setSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { trackSshHostServerEvent } from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' +import { knownOrcadTunnelTransport } from '../ssh/orcad-tunnel-transport-memo' +import { + getSshTargetRegistryStore, + hasRegisteredDirectSshAuthority +} from '../ssh/ssh-target-registry' +import { releaseUnreachableOrcadSetup } from '../ssh/orcad-unreachable-setup-release' +import { getCurrentMainWindow } from './ssh-ipc-context' +import { broadcastSshState } from './ssh-renderer-broadcast' +import { disconnectRegisteredSshTarget } from './ssh-session-teardown' +import { connectInFlight } from './ssh-connect-attempt-registry' + +export function hostServerOnConnectDeps(userDataPath: string): HostServerOnConnectDeps { + const registry = getSshTargetRegistryStore()! + const claims = registry.getOrcadRuntimeClaims() + const store = registry.getOrcadMigrationSource() + const appVersion = getAppEnvironment().getVersion() + const isRegistered = (environmentId: string): boolean => + listEnvironments(userDataPath).some((entry) => entry.id === environmentId) + return { + // Why registered only: a fence whose server never registered is an unfinished setup, not a server. + managedEnvironmentId: (target) => { + const environmentId = getManagedOrcadFenceEnvironmentId(target) + return environmentId && isRegistered(environmentId) ? environmentId : null + }, + ensureTunnel: async (environmentId) => { + await ensureOrcadManagedTunnel(userDataPath, environmentId) + }, + ensureServing: (environmentId) => verifyOrcadManagedServing(userDataPath, environmentId), + retainCommittedSource: (target) => { + const head = findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) + if (head?.phase === 'destination-committed') { + retainOrcadMigrationSource(userDataPath, head.migrationId) + } + }, + hasTemplate: hasOrcadTemplate, + recordedUnavailable: (target) => + target.managedServerUnavailable?.appVersion === appVersion + ? target.managedServerUnavailable.reason + : null, + recordUnavailable: (target, reason) => { + registry.updateTarget(target.id, { managedServerUnavailable: { reason, appVersion } }) + }, + ...managedServerUpdateDeps(userDataPath), + isEmptyHost: (target) => { + // An interrupted empty-host deploy passes its own claim, recorded by its provisioning intent. + const environmentId = getManagedOrcadFenceEnvironmentId(target) + return claims.preflight( + target.id, + environmentId + ? { environmentId, recorded: target.orcadProvisioning !== undefined } + : undefined + ).claimable + }, + relayTerminals: (target) => + relayTerminalsOnConnect({ + store, + targetId: target.id, + listRelayPtyIds: orcadMigrationRelayPtyLister(target.id), + censusHost: async () => + censusSshHostRelaysBeforeSession( + await requireManagedOrcadInfrastructure().connectionManager.connect(target), + target.id + ) + }), + deploy: (target) => + createManagedOrcadEnvironment(userDataPath, { + name: target.orcadProvisioning?.name ?? target.label, + sshTargetId: target.id + }), + convert: (target, hostProof) => + convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: target.id, + name: target.label, + listRelayPtyIds: orcadMigrationRelayPtyLister(target.id), + // The connect's own census already answered; a conversion before any session reuses it. + censusHost: (host) => + hostProof ? Promise.resolve(hostProof) : censusHostRelayTerminalsFor(host)(), + destinationFor: orcadMigrationDestinationFor, + // Why guarded: this runs before the connect registers a session, and an unconditional + // disconnect would cancel the very connect attempt that asked for the conversion. + releaseDirectSession: async (targetId) => { + if (hasRegisteredDirectSshAuthority(targetId)) { + await disconnectRegisteredSshTarget(targetId) + } + } + }), + abandonDeploy: async (target) => { + const environmentId = + getManagedOrcadFenceEnvironmentId(target) ?? + getManagedOrcadFenceEnvironmentId(registry.getTarget(target.id)) + if (environmentId && !isRegistered(environmentId)) { + claims.release(target.id, environmentId) + await claims.flush() + } + }, + hasUnfinishedConversion: (target) => { + const head = findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) + return ( + head !== null && + !head.supersedesMigrationId && + (head.phase === 'source-fenced' || head.phase === 'destination-staged') + ) + }, + abandonConversion: async (target) => { + await abandonOrcadConversion({ + userDataPath, + store, + claims, + targetId: target.id, + destinationFor: orcadMigrationDestinationFor + }) + }, + isFencedBeforeStaging: (target) => + findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id)?.phase === 'source-fenced', + releaseUnreachableSetup: (target) => + releaseUnreachableOrcadSetup({ userDataPath, claims, targetId: target.id }), + // Read at report time: a deploy or conversion learns the platform while the decision runs. + report: (target, event) => + trackSshHostServerEvent( + event, + knownSshHostPlatform(target.id), + knownOrcadTunnelTransport(target.id) + ), + progress: (target, phase) => { + // A cancelled connect's decision keeps running; its progress must not revive the status. + if (!connectInFlight.has(target.id)) { + return + } + setSshHostServerStatus(target.id, { kind: 'setting-up', phase }) + broadcastSshState(getCurrentMainWindow, target.id, { + targetId: target.id, + status: 'connecting', + error: null, + reconnectAttempt: 0 + }) + } + } +} diff --git a/src/main/ipc/ssh-host-server-relay-decision.test.ts b/src/main/ipc/ssh-host-server-relay-decision.test.ts new file mode 100644 index 00000000000..b1677d6d51e --- /dev/null +++ b/src/main/ipc/ssh-host-server-relay-decision.test.ts @@ -0,0 +1,51 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' + +const mocks = vi.hoisted(() => ({ + updateTarget: vi.fn(), + setStatus: vi.fn() +})) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getVersion: () => '1.5.0' }) +})) +vi.mock('../ssh/ssh-target-registry', () => ({ + getSshTargetRegistryStore: () => ({ updateTarget: mocks.updateTarget }) +})) +vi.mock('../ssh/ssh-host-server-status', () => ({ setSshHostServerStatus: mocks.setStatus })) +vi.mock('./ssh-ipc-context', () => ({ connectionManager: null, getCurrentMainWindow: () => null })) +vi.mock('./ssh-renderer-broadcast', () => ({ + broadcastSshState: vi.fn(), + getPublicSshState: vi.fn() +})) + +const { recordRelayDecision } = await import('./ssh-host-server-connect') + +const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } +const live = { route: 'relay' as const, reason: 'relay_terminals_live' as const, terminals: 2 } + +beforeEach(() => vi.clearAllMocks()) + +describe('recording why a connect kept the relay', () => { + it('records the first offer this version and marks the status for the toast', () => { + recordRelayDecision(target, live) + expect(mocks.updateTarget).toHaveBeenCalledWith('ssh-1', { + managedServerMoveOffered: { appVersion: '1.5.0' } + }) + expect(mocks.setStatus).toHaveBeenCalledWith('ssh-1', { + kind: 'relay', + reason: 'relay_terminals_live', + terminals: 2, + offerMove: true + }) + }) + + it('keeps the status line offer but skips the toast once this version offered it', () => { + recordRelayDecision({ ...target, managedServerMoveOffered: { appVersion: '1.5.0' } }, live) + expect(mocks.updateTarget).not.toHaveBeenCalled() + expect(mocks.setStatus).toHaveBeenCalledWith('ssh-1', { + kind: 'relay', + reason: 'relay_terminals_live', + terminals: 2 + }) + }) +}) diff --git a/src/main/ipc/ssh-host-sleep-reconnect.test.ts b/src/main/ipc/ssh-host-sleep-reconnect.test.ts index 1994e8c1a0e..e0f9721e6e0 100644 --- a/src/main/ipc/ssh-host-sleep-reconnect.test.ts +++ b/src/main/ipc/ssh-host-sleep-reconnect.test.ts @@ -4,12 +4,16 @@ const manager = vi.hoisted(() => { const connection = {} return { getConnection: vi.fn(() => connection), reconnect: vi.fn(async () => {}) } }) +const recoverManagedTunnels = vi.hoisted(() => vi.fn(async () => {})) vi.mock('electron', async () => { const { EventEmitter } = await import('node:events') return { powerMonitor: new EventEmitter() } }) vi.mock('./ssh-ipc-context', () => ({ connectionManager: manager })) +vi.mock('../ssh/orcad-managed-tunnel', () => ({ + recoverOrcadManagedTunnelsAfterHostResume: recoverManagedTunnels +})) import { powerMonitor } from 'electron' import { activeSessions } from './ssh-active-relay-sessions' @@ -56,4 +60,24 @@ describe('host sleep reconnect in plain SSH mode', () => { powerMonitor.emit('resume') await vi.waitFor(() => expect(manager.reconnect).toHaveBeenCalledWith('target-1')) }) + + it('recovers managed tunnels with the same probe policy, even with no relay sessions', async () => { + registerPowerMonitorReconnect(() => '/user-data') + powerMonitor.emit('resume') + await vi.waitFor(() => + expect(recoverManagedTunnels).toHaveBeenCalledWith('/user-data', { + attempts: 2, + timeoutMs: 5_000 + }) + ) + }) + + it('leaves managed tunnels alone when the caller supplies no profile path', async () => { + const session = plainSession(async () => true) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the resume path only calls the methods stubbed here. + activeSessions.set('target-1', session as never) + registerPowerMonitorReconnect() + await resumeAndSettle() + expect(recoverManagedTunnels).not.toHaveBeenCalled() + }) }) diff --git a/src/main/ipc/ssh-host-sleep-reconnect.ts b/src/main/ipc/ssh-host-sleep-reconnect.ts index 731946e3cfc..3c559059f45 100644 --- a/src/main/ipc/ssh-host-sleep-reconnect.ts +++ b/src/main/ipc/ssh-host-sleep-reconnect.ts @@ -1,4 +1,5 @@ import { powerMonitor } from 'electron' +import { recoverOrcadManagedTunnelsAfterHostResume } from '../ssh/orcad-managed-tunnel' import type { SshRelaySession } from '../ssh/ssh-relay-session' import { activeSessions } from './ssh-active-relay-sessions' import { connectionManager } from './ssh-ipc-context' @@ -32,7 +33,7 @@ async function isRelayLinkAliveAfterResume(session: SshRelaySession): Promise string): void { powerMonitorUnsubscribe?.() const onSuspend = (): void => { for (const session of activeSessions.values()) { @@ -66,6 +67,19 @@ export function registerPowerMonitorReconnect(): void { } })() } + // Why separate: managed tunnels ride their own connections, not relay sessions. + if (getUserDataPath) { + void recoverOrcadManagedTunnelsAfterHostResume(getUserDataPath(), { + attempts: RESUME_PROBE_ATTEMPTS, + timeoutMs: RESUME_PROBE_TIMEOUT_MS + }).catch((err) => { + console.warn( + `[ssh] Failed to recover a managed Orca tunnel after system resume: ${ + err instanceof Error ? err.message : String(err) + }` + ) + }) + } } powerMonitor.on('suspend', onSuspend) powerMonitor.on('resume', onResume) diff --git a/src/main/ipc/ssh-ipc-mock-shapes.ts b/src/main/ipc/ssh-ipc-mock-shapes.ts index 28efa30ee39..4136aa78512 100644 --- a/src/main/ipc/ssh-ipc-mock-shapes.ts +++ b/src/main/ipc/ssh-ipc-mock-shapes.ts @@ -52,6 +52,7 @@ export type SshPtyProviderMock = { attach: Mock attachForReconnect: Mock shutdown: Mock + listProcesses: Mock providerGeneration: number } @@ -97,6 +98,7 @@ export type SshIpcMockModules = { sshConfigHostPicker: SshIpcMockModule electron: SshIpcMockModule sshPtyOutputIntakeRegistry: SshIpcMockModule + hostServerConnect: SshIpcMockModule sshConnectionStore: SshIpcMockModule sshConnectionManager: SshIpcMockModule sshRelayDeploy: SshIpcMockModule diff --git a/src/main/ipc/ssh-ipc-module-mocks.ts b/src/main/ipc/ssh-ipc-module-mocks.ts index 0855a739fb2..19348813943 100644 --- a/src/main/ipc/ssh-ipc-module-mocks.ts +++ b/src/main/ipc/ssh-ipc-module-mocks.ts @@ -57,6 +57,7 @@ export function createSshIpcMocks(): SshIpcMocks { attach: vi.fn(), attachForReconnect: vi.fn().mockResolvedValue({}), shutdown: vi.fn(), + listProcesses: vi.fn(async () => []), providerGeneration: 0 }, mockFsProvider: {}, @@ -143,7 +144,21 @@ export function createSshIpcMocks(): SshIpcMocks { installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}), installSshPtySourceCancellationPublisher: vi.fn().mockReturnValue(() => {}) }, + // Null keeps today's relay path; the real decision is covered by its own tests. + hostServerConnect: { + decideHostServer: vi.fn(async () => null), + recheckWhenManagedFenceClears: vi.fn(), + publishHostServerDecisionFailure: vi.fn(), + publishManagedServerConnect: vi.fn(), + recordRelayDecision: vi.fn(), + refineRelayTerminalDecision: vi.fn(async () => {}) + }, sshConnectionStore: { + isRuntimeOwnedSshTarget: (target: { owner?: unknown }) => target.owner !== undefined, + isManagedOrcadSshTarget: (target: { orcadFence?: unknown; orcadProvisioning?: unknown }) => + target.orcadFence !== undefined || target.orcadProvisioning !== undefined, + allowsDirectSshRelay: (target: { orcadFence?: unknown; orcadProvisioning?: unknown }) => + target.orcadFence === undefined && target.orcadProvisioning === undefined, SshConnectionStore: class MockSshConnectionStore { constructor() { return mockSshStore diff --git a/src/main/ipc/ssh-ipc-test-harness.ts b/src/main/ipc/ssh-ipc-test-harness.ts index 581fcc916ca..88b83c70119 100644 --- a/src/main/ipc/ssh-ipc-test-harness.ts +++ b/src/main/ipc/ssh-ipc-test-harness.ts @@ -241,6 +241,7 @@ export function createSshIpcHarness(mocks: SshIpcMocks): SshIpcHarness { mockPtyProvider.onReplay.mockReset() mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({}) mockPtyProvider.shutdown.mockReset() + mockPtyProvider.listProcesses.mockReset().mockResolvedValue([]) mockPtyProvider.providerGeneration = 0 mockRegisterSshGitProvider.mockReset() mockPortForwardManager.addForward.mockReset() diff --git a/src/main/ipc/ssh-managed-server-move-conversion.test.ts b/src/main/ipc/ssh-managed-server-move-conversion.test.ts new file mode 100644 index 00000000000..d045165bc48 --- /dev/null +++ b/src/main/ipc/ssh-managed-server-move-conversion.test.ts @@ -0,0 +1,261 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { OrcadMigrationManifest } from '../../shared/orcad-migration-manifest' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { SshManagedServerStatus, SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { fakeOrcadMigrationDestination } from '../ssh/orcad-migration-destination-fake' +import { assessOrcadMigrationTerminals } from '../ssh/orcad-migration-terminal-gate' +import { SshConnectionStore } from '../ssh/ssh-connection-store' + +const mocks = vi.hoisted(() => { + const state: { targetStore: unknown } = { targetStore: null } + return { state, deploy: vi.fn() } +}) +vi.mock('../ssh/ssh-target-registry', () => ({ + getSshConnectionManager: () => ({}), + getSshTargetRegistryStore: () => mocks.state.targetStore, + hasRegisteredDirectSshAuthority: () => false +})) +vi.mock('../ssh/orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('../ssh/orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: vi.fn() })) +vi.mock('./ssh-connect-flow', () => ({ connectTarget: vi.fn() })) +vi.mock('./ssh-terminate-sessions', () => ({ terminateSshTargetSessions: vi.fn() })) +vi.mock('./ssh-session-teardown', () => ({ teardownSshTargetTransport: vi.fn() })) +vi.mock('./ssh-host-server-connect', () => ({ publishRelayTerminalsStatus: vi.fn() })) + +const { convertSshTargetToManagedOrcad } = await import('../ssh/orcad-runtime-conversion') +const { moveSshHostToManagedServer } = await import('./ssh-managed-server-move') + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} +const HOST_ID = `ssh:${TARGET.id}` as const +const WORKTREE = 'repo-1::/srv/app' + +let userDataPath: string +let store: Store +let destination: ReturnType + +beforeEach(() => { + vi.resetAllMocks() + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-move-')) + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + mocks.state.targetStore = new SshConnectionStore(store as never) + destination = fakeOrcadMigrationDestination() + mocks.deploy.mockImplementation(async (path: string, args: { name: string }) => { + const id = getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))! + if (!listEnvironments(path).some((entry) => entry.id === id)) { + addManagedOrcadEnvironment(path, { + id, + name: args.name, + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/', + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: TARGET.id, + sshTargetGeneration: 2, + localPort: 46_768, + remotePort: 6_768 + } + }) + } + return { outcome: 'created', environment: {}, activeVersion: '1.0.0' } + }) +}) + +afterEach(async () => { + await closeTestStores() + rmSync(userDataPath, { recursive: true, force: true }) +}) + +/** Open relay terminal tabs: each tab, layout leaf and pane incarnation names its relay PTY. */ +function openRelayTerminals(count: number): void { + const terminals = Array.from({ length: count }, (_, index) => ({ + tabId: `tab-term-${index + 1}`, + leafId: `11111111-1111-4111-8111-11111111111${index + 1}`, + relayPtyId: `pty2:relay:${index + 1}` + })) + store.setWorkspaceSession( + { + ...store.getWorkspaceSession(HOST_ID), + activeRepoId: 'repo-1', + activeWorktreeId: WORKTREE, + activeTabId: terminals[0].tabId, + tabsByWorktree: { + [WORKTREE]: terminals.map(({ tabId, relayPtyId }, index) => ({ + id: tabId, + ptyId: `${HOST_ID}@@${relayPtyId}`, + worktreeId: WORKTREE, + title: `Terminal ${index + 1}`, + customTitle: null, + color: null, + sortOrder: index, + createdAt: 1 + })) + }, + terminalLayoutsByTabId: Object.fromEntries( + terminals.map(({ tabId, leafId, relayPtyId }) => [ + tabId, + { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [leafId]: `${HOST_ID}@@${relayPtyId}` } + } + ]) + ), + terminalPtyIncarnationsByPaneKey: Object.fromEntries( + terminals.map(({ tabId, leafId }) => [`${tabId}:${leafId}`, `incarnation-${tabId}`]) + ), + activeWorktreeIdsOnShutdown: [WORKTREE] + }, + HOST_ID + ) + for (const { tabId, leafId, relayPtyId } of terminals) { + store.upsertSshRemotePtyLease({ + targetId: TARGET.id, + ptyId: relayPtyId, + worktreeId: WORKTREE, + tabId, + leafId, + state: 'detached' + }) + } +} + +function moveDeps() { + let status: SshManagedServerStatus | undefined + return { + getTarget: (targetId: string) => store.getSshTarget(targetId), + // The relay acknowledged stopping every terminal. + terminate: vi.fn(async (targetId: string) => { + const leases = store.getSshRemotePtyLeases(targetId) + for (const lease of leases) { + store.markSshRemotePtyLease(targetId, lease.ptyId, 'terminated') + } + return { terminated: leases.length, unverifiable: 0 } + }), + // The reconnect's server decision: its census (no relay session, so the host's census, which + // found no relay endpoint, decides with the leases) keeps the relay or lets it convert. + connect: vi.fn(async (targetId: string) => { + const proof = await assessOrcadMigrationTerminals(store, targetId, null, hostIdle) + if (proof.verdict !== 'exited') { + status = { + kind: 'relay', + reason: + proof.verdict === 'live' ? 'relay_terminals_live' : 'relay_terminals_unverifiable', + terminals: proof.ptyIds.length + } + return + } + const result = await convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: targetId, + name: TARGET.label, + listRelayPtyIds: null, + censusHost: hostIdle, + destinationFor: () => destination, + releaseDirectSession: async () => {} + }) + status = + result.outcome === 'converted' + ? { kind: 'managed', environmentId: result.environment.id } + : { kind: 'relay', reason: 'refused', detail: 'reason' in result ? result.reason : '' } + }), + serverStatus: () => status, + report: vi.fn(), + releaseRelay: vi.fn(async () => {}) + } +} + +async function hostIdle() { + return { verdict: 'exited' as const, count: 0 } +} + +function stagedSession() { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: stage receives the migration manifest as its first argument. + const manifest = destination.stage.mock.calls[0]?.[0] as OrcadMigrationManifest + return manifest.payload.dormantState?.workspaceSession +} + +describe('moving a host whose live relay terminals kept it on the relay', () => { + it('stops the terminal, converts, and carries its tab to orcad without the relay PTY', async () => { + openRelayTerminals(1) + const deps = moveDeps() + + await expect(moveSshHostToManagedServer(TARGET.id, deps)).resolves.toMatchObject({ + outcome: 'moved' + }) + expect(deps.terminate).toHaveBeenCalledWith(TARGET.id, expect.any(Function)) + const session = stagedSession() + // No relay PTY id survives, so the tab spawns a fresh shell on the managed server. + expect(Object.values(session?.tabsByWorktree ?? {}).flat()).toMatchObject([ + { id: 'tab-term-1', ptyId: null } + ]) + expect(session?.terminalLayoutsByTabId?.['tab-term-1']?.ptyIdsByLeafId).toBeUndefined() + expect(session?.terminalPtyIncarnationsByPaneKey ?? {}).toEqual({}) + }) + + it('moves two terminals whose stop lost its last reply to a relay that hung up (BUG-15)', async () => { + openRelayTerminals(2) + const deps = moveDeps() + // Both shells died and their leases recorded it, but the second reply was lost. + deps.terminate.mockImplementationOnce(async (targetId: string) => { + store.markSshRemotePtyLease(targetId, 'pty2:relay:1', 'terminated') + store.markSshRemotePtyLease(targetId, 'pty2:relay:2', 'terminated') + throw new Error('Failed to terminate SSH host sessions: pty2:relay:2: Multiplexer disposed') + }) + + await expect(moveSshHostToManagedServer(TARGET.id, deps)).resolves.toMatchObject({ + outcome: 'moved' + }) + expect(deps.releaseRelay).toHaveBeenCalledWith(TARGET.id) + expect(Object.values(stagedSession()?.tabsByWorktree ?? {}).flat()).toMatchObject([ + { id: 'tab-term-1', ptyId: null }, + { id: 'tab-term-2', ptyId: null } + ]) + }) + + it('refuses and refreshes the status when a failed stop left a terminal unproven', async () => { + openRelayTerminals(2) + const deps = moveDeps() + deps.terminate.mockImplementationOnce(async (targetId: string) => { + store.markSshRemotePtyLease(targetId, 'pty2:relay:1', 'terminated') + throw new Error('Failed to terminate SSH host sessions: pty2:relay:2: Multiplexer disposed') + }) + + await expect(moveSshHostToManagedServer(TARGET.id, deps)).resolves.toEqual({ + outcome: 'refused', + verdict: 'unverifiable', + terminals: 1 + }) + // Reconnected on the relay, whose own gate refuses the conversion the same way. + expect(deps.connect).toHaveBeenCalledWith(TARGET.id) + expect(destination.stage).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/ssh-managed-server-move-stop-retention.test.ts b/src/main/ipc/ssh-managed-server-move-stop-retention.test.ts new file mode 100644 index 00000000000..ff26ef17b04 --- /dev/null +++ b/src/main/ipc/ssh-managed-server-move-stop-retention.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = await vi.hoisted(async () => { + const { createSshIpcMocks } = await import('./ssh-ipc-module-mocks') + return createSshIpcMocks() +}) + +vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) +vi.mock('electron', () => mocks.electron) +vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) +vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) +vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) +vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) +vi.mock( + '../ssh/ssh-previous-relay-terminals', + () => import('../ssh/ssh-previous-relay-census-test-double') +) +vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) +vi.mock('../ssh/ssh-channel-multiplexer', () => mocks.sshChannelMultiplexer) +vi.mock('../providers/ssh-pty-provider', () => mocks.sshPtyProvider) +vi.mock('../providers/ssh-filesystem-provider', () => mocks.sshFilesystemProvider) +vi.mock('./pty', () => mocks.pty) +vi.mock('../providers/ssh-filesystem-dispatch', () => mocks.sshFilesystemDispatch) +vi.mock('../providers/ssh-git-provider', () => mocks.sshGitProvider) +vi.mock('../providers/ssh-git-dispatch', () => mocks.sshGitDispatch) +vi.mock('../ssh/ssh-port-forward', () => mocks.sshPortForward) +vi.mock('../ssh/ssh-port-scanner', () => mocks.sshPortScanner) + +import { getDefaultWorkspaceSession } from '../../shared/constants' +import type { Repo } from '../../shared/repo-types' +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import { OrcaRuntimeService } from '../runtime/orca-runtime' +import { withDurableRuntimeStore } from '../runtime/runtime-durable-store-fixture' +import { getSshPtyProvider, getPtyIdsForConnection } from './pty' +import { ptyIncarnationById } from './pty/provider/ownership-state' +import { setCurrentRuntime } from './ssh-ipc-context' +import { createSshIpcHarness } from './ssh-ipc-test-harness' +import { terminateSshTargetSessions } from './ssh-terminate-sessions' + +const { mockSshStore, mockPtyProvider } = mocks +const worktreeId = 'repo::/srv/repo' +const leafId = '11111111-1111-4111-8111-111111111111' +const stoppedShell = 'ssh:ssh-1@@pty2:epoch:1' +const userExitedShell = 'ssh:ssh-1@@pty2:epoch:2' +const repo: Repo = { + id: 'repo', + path: '/srv/repo', + connectionId: 'ssh-1', + displayName: 'repo', + badgeColor: 'gray', + addedAt: 1 +} + +function sessionWith(ptyIds: Record): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [worktreeId]: Object.entries(ptyIds).map(([tabId, ptyId], sortOrder) => ({ + id: tabId, + worktreeId, + ptyId, + title: `Terminal ${sortOrder + 1}`, + customTitle: null, + color: null, + sortOrder, + createdAt: 1 + })) + }, + terminalLayoutsByTabId: Object.fromEntries( + Object.entries(ptyIds).map(([tabId, ptyId]) => [ + tabId, + { + root: { type: 'leaf', leafId: `${leafId.slice(0, -1)}${tabId.at(-1)}` }, + activeLeafId: `${leafId.slice(0, -1)}${tabId.at(-1)}`, + expandedLeafId: null, + ptyIdsByLeafId: { [`${leafId.slice(0, -1)}${tabId.at(-1)}`]: ptyId } + } + ]) + ) + } +} + +describe("a move's stop keeps the saved SSH session it is about to convert", () => { + const harness = createSshIpcHarness(mocks) + beforeEach(harness.reset) + afterEach(() => setCurrentRuntime(undefined)) + + it('keeps the stopped tab through its real exit, certifies the death, and closes an earlier exit', async () => { + let session = sessionWith({ 'tab-1': stoppedShell, 'tab-2': userExitedShell }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fake supplies the session and repo methods exit retirement reads. + const store = withDurableRuntimeStore({ + getRepos: () => [repo], + getRepo: () => repo, + getWorkspaceSessionHostIds: () => ['ssh:ssh-1'], + getWorkspaceSession: () => session, + setWorkspaceSession: (next: WorkspaceSessionState) => { + session = next + }, + flushOrThrow: vi.fn() + }) as never + const runtime = new OrcaRuntimeService(store) + for (const [tabId, ptyId] of [ + ['tab-1', stoppedShell], + ['tab-2', userExitedShell] + ] as const) { + const tabLeafId = `${leafId.slice(0, -1)}${tabId.at(-1)}` + runtime.registerPty(ptyId, worktreeId, 'ssh-1', { + tabId, + leafId: tabLeafId, + incarnationId: `inc-${tabId}` + }) + ptyIncarnationById.set(ptyId, `inc-${tabId}`) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a real OrcaRuntimeService. + setCurrentRuntime(runtime as never) + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + }) + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([stoppedShell]) + + // The user's own exit lands before the move stops anything: it closes its tab as always. + await runtime.onPtyExit(userExitedShell, 0, 'inc-tab-2', { hostExitConfirmed: true }) + expect(session.tabsByWorktree[worktreeId]?.map((tab) => tab.id)).toEqual(['tab-1']) + + // The relay reports the stopped shell's real exit while its shutdown is in flight. + mockPtyProvider.shutdown.mockImplementation(async (ptyId: string) => { + await runtime.onPtyExit(ptyId, 0, 'inc-tab-1', { hostExitConfirmed: true }) + }) + const stopped: string[] = [] + await terminateSshTargetSessions('ssh-1', { + intentionalStop: 'replaced', + onStopped: (ptyId) => stopped.push(ptyId) + }) + + expect(stopped).toEqual([stoppedShell]) + expect(session.tabsByWorktree[worktreeId]?.map((tab) => tab.id)).toEqual(['tab-1']) + expect(runtime.getPtyLivenessVerdict(stoppedShell)?.status).toBe('exited') + }, 120_000) +}) diff --git a/src/main/ipc/ssh-managed-server-move.test.ts b/src/main/ipc/ssh-managed-server-move.test.ts new file mode 100644 index 00000000000..fb30f525a3d --- /dev/null +++ b/src/main/ipc/ssh-managed-server-move.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it, vi } from 'vitest' +import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' +import type { SshManagedServerStatus, SshTarget } from '../../shared/ssh-types' + +vi.mock('./ssh-connect-flow', () => ({ connectTarget: vi.fn() })) +vi.mock('./ssh-terminate-sessions', () => ({ terminateSshTargetSessions: vi.fn() })) +vi.mock('./ssh-session-teardown', () => ({ teardownSshTargetTransport: vi.fn() })) + +const { moveSshHostToManagedServer } = await import('./ssh-managed-server-move') + +const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } +const live = (terminals: number): SshManagedServerStatus => ({ + kind: 'relay', + reason: 'relay_terminals_live', + terminals +}) + +/** `afterConnect` is what the reconnect's own decision (and its census) recorded. */ +function deps(options: { unverifiable?: number; afterConnect?: SshManagedServerStatus } = {}) { + const calls: string[] = [] + let status: SshManagedServerStatus | undefined = live(2) + return { + calls, + getTarget: vi.fn(() => target), + terminate: vi.fn(async (_targetId: string, _onStopped: (appPtyId: string) => void) => { + calls.push('terminate') + return { terminated: 2, unverifiable: options.unverifiable ?? 0 } + }), + connect: vi.fn(async () => { + calls.push('connect') + status = options.afterConnect ?? { kind: 'managed', environmentId: 'env-1' } + }), + serverStatus: vi.fn(() => status), + report: vi.fn(), + releaseRelay: vi.fn(async () => { + calls.push('release') + }) + } +} + +describe('moving an SSH host to its managed server on request', () => { + it('names the shells it stopped, even when a later stop fails, so only those tabs restart', async () => { + const move = deps({ afterConnect: live(1) }) + move.terminate.mockImplementationOnce(async (_targetId, onStopped) => { + onStopped('ssh:ssh-1@@pty-1') + throw new Error('Failed to terminate SSH host sessions: pty-2: mux down') + }) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ + outcome: 'refused', + verdict: 'live', + terminals: 1, + stoppedPtyIds: ['ssh:ssh-1@@pty-1'] + }) + }) + + it('stops the relay terminals, then lets the reconnect prove them exited and convert', async () => { + const move = deps() + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ + outcome: 'moved', + environmentId: 'env-1' + }) + // No census of its own: the connect's decision runs it while holding the raw 'connected'. + expect(move.calls).toEqual(['terminate', 'connect']) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'moved') + }) + + it('refuses without converting when the stop could not reach every terminal', async () => { + const move = deps({ + unverifiable: 1, + afterConnect: { kind: 'relay', reason: 'relay_terminals_unverifiable', terminals: 1 } + }) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ + outcome: 'refused', + verdict: 'unverifiable', + terminals: 1 + }) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'refused_unverifiable') + }) + + it('reports the reconnect census verdict when it keeps the relay', async () => { + const stillLive = deps({ afterConnect: live(1) }) + await expect(moveSshHostToManagedServer('ssh-1', stillLive)).resolves.toEqual({ + outcome: 'refused', + verdict: 'live', + terminals: 1 + }) + expect(stillLive.report).toHaveBeenCalledWith('ssh-1', 'refused_live') + + const unproven = deps({ + afterConnect: { kind: 'relay', reason: 'relay_terminals_unverifiable', terminals: 3 } + }) + await expect(moveSshHostToManagedServer('ssh-1', unproven)).resolves.toEqual({ + outcome: 'refused', + verdict: 'unverifiable', + terminals: 3 + }) + }) + + it('reports a connect that kept the relay for another reason', async () => { + const move = deps({ afterConnect: { kind: 'relay', reason: 'refused', detail: 'blocked' } }) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ outcome: 'stayed' }) + expect(move.report).toHaveBeenCalledWith('ssh-1', 'stayed') + }) + + it('reattaches the relay first when preserved terminals need one to be stopped', async () => { + const move = deps() + move.terminate.mockRejectedValueOnce( + new Error(`${SSH_TERMINATE_RECONNECT_REQUIRED}: reconnect`) + ) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toMatchObject({ + outcome: 'moved' + }) + expect(move.calls).toEqual(['connect', 'terminate', 'connect']) + }) + + it('detaches a relay a failed stop left up, so the reconnect decides again', async () => { + const move = deps() + // BUG-15: the second terminal's reply was lost to a relay that hung up on its last exit. + move.terminate.mockRejectedValueOnce( + new Error('Failed to terminate SSH host sessions: pty2:a:3: Multiplexer disposed') + ) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toMatchObject({ + outcome: 'moved' + }) + expect(move.calls).toEqual(['release', 'connect']) + }) + + it('leaves the relay of a stop that succeeded to the reconnect', async () => { + const move = deps() + await moveSshHostToManagedServer('ssh-1', move) + expect(move.releaseRelay).not.toHaveBeenCalled() + }) + + it('reports the stop’s own unverifiable count when the reconnect itself fails', async () => { + const move = deps({ unverifiable: 2 }) + move.connect.mockRejectedValueOnce(new Error('auth failed')) + await expect(moveSshHostToManagedServer('ssh-1', move)).resolves.toEqual({ + outcome: 'refused', + verdict: 'unverifiable', + terminals: 2 + }) + }) +}) diff --git a/src/main/ipc/ssh-managed-server-move.ts b/src/main/ipc/ssh-managed-server-move.ts new file mode 100644 index 00000000000..3b89256c516 --- /dev/null +++ b/src/main/ipc/ssh-managed-server-move.ts @@ -0,0 +1,144 @@ +/** + * The user's "Move to managed server": stop the host's relay terminals, then reconnect so the + * connect-time decision proves them exited with its own census and runs the conversion. + */ +import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' +import type { SshManagedServerMoveResult } from '../../shared/ssh-managed-server-move' +import type { + SshManagedServerStatus, + SshTarget, + SshTerminateSessionsResult +} from '../../shared/ssh-types' +import { getSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { + trackSshHostServerMove, + type SshHostServerMoveOutcome +} from '../ssh/ssh-host-server-telemetry' +import { knownSshHostPlatform } from '../ssh/ssh-host-platform-memo' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { connectTarget } from './ssh-connect-flow' +import { teardownSshTargetTransport } from './ssh-session-teardown' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' +import { terminateSshTargetSessions } from './ssh-terminate-sessions' + +export type SshManagedServerMoveDeps = { + getTarget: (targetId: string) => SshTarget | undefined + /** Stops the relay terminals as main's own restart, reporting each shell it stopped. */ + terminate: ( + targetId: string, + onStopped: (appPtyId: string) => void + ) => Promise + connect: (targetId: string) => Promise + serverStatus: (targetId: string) => SshManagedServerStatus | undefined + report: (targetId: string, outcome: SshHostServerMoveOutcome) => void + /** Detaches a relay session a failed stop left up, keeping its terminals' leases. */ + releaseRelay: (targetId: string) => Promise +} + +export async function moveSshHostToManagedServer( + targetId: string, + deps: SshManagedServerMoveDeps = defaultMoveDeps() +): Promise { + let result: SshManagedServerMoveResult | null = null + try { + result = await moveHost(targetId, deps) + return result + } finally { + deps.report(targetId, moveOutcome(result)) + } +} + +function moveOutcome(result: SshManagedServerMoveResult | null): SshHostServerMoveOutcome { + if (!result) { + return 'failed' + } + return result.outcome === 'refused' ? `refused_${result.verdict}` : result.outcome +} + +async function moveHost( + targetId: string, + deps: SshManagedServerMoveDeps +): Promise { + if (!deps.getTarget(targetId)) { + throw new Error(`SSH target "${targetId}" not found`) + } + // Why the reconnect's census decides, not the stop: a stop can fail after its shells died (a + // relay that hung up on its last exit), and the connect's decision runs the census the + // conversion trusts, while that connect holds the raw transport's 'connected'. + const stoppedPtyIds = new Set() + const stopped = await stopRelayTerminals(targetId, deps, (appPtyId) => + stoppedPtyIds.add(appPtyId) + ).catch((error: unknown) => { + console.warn('[ssh] Stopping relay terminals for the move failed; asking the census:', error) + return null + }) + if (!stopped) { + // The failed stop left the relay up; a live session would make the reconnect a no-op refresh. + await deps.releaseRelay(targetId) + } + const status = await deps.connect(targetId).then( + () => deps.serverStatus(targetId), + (error: unknown) => { + console.warn('[ssh] Reconnecting for the move failed:', error) + return undefined + } + ) + if (status?.kind === 'managed') { + return { outcome: 'moved', environmentId: status.environmentId } + } + const restart = stoppedPtyIds.size > 0 ? { stoppedPtyIds: [...stoppedPtyIds] } : {} + // Why: an unreached shell is never evidence that it exited (ssh-execution-boundary.md). + if (stopped && stopped.unverifiable > 0) { + return { + outcome: 'refused', + verdict: 'unverifiable', + terminals: stopped.unverifiable, + ...restart + } + } + if (status?.kind === 'relay' && status.reason === 'relay_terminals_live') { + return { outcome: 'refused', verdict: 'live', terminals: status.terminals ?? 0, ...restart } + } + if (status?.kind === 'relay' && status.reason === 'relay_terminals_unverifiable') { + return { + outcome: 'refused', + verdict: 'unverifiable', + terminals: status.terminals ?? 0, + ...restart + } + } + return { outcome: 'stayed', ...restart } +} + +/** Mirrors the renderer's terminate: preserved shells need a fresh relay before they can be stopped. */ +async function stopRelayTerminals( + targetId: string, + deps: SshManagedServerMoveDeps, + onStopped: (appPtyId: string) => void +): Promise { + try { + return await deps.terminate(targetId, onStopped) + } catch (error) { + if (!(error instanceof Error) || !error.message.includes(SSH_TERMINATE_RECONNECT_REQUIRED)) { + throw error + } + await deps.connect(targetId) + return deps.terminate(targetId, onStopped) + } +} + +function defaultMoveDeps(): SshManagedServerMoveDeps { + return { + getTarget: (targetId) => getSshTargetRegistryStore()!.getTarget(targetId), + // Why 'replaced': the move restarts these terminals, so main and every viewer keep their tabs. + terminate: (targetId, onStopped) => + terminateSshTargetSessions(targetId, { intentionalStop: 'replaced', onStopped }), + connect: connectTarget, + serverStatus: getSshHostServerStatus, + report: (targetId, outcome) => trackSshHostServerMove(outcome, knownSshHostPlatform(targetId)), + releaseRelay: (targetId) => + runTargetLifecycle(targetId, () => + teardownSshTargetTransport(targetId, (session) => session.detachAndPersist()) + ) + } +} diff --git a/src/main/ipc/ssh-managed-server-unlink-broadcast.test.ts b/src/main/ipc/ssh-managed-server-unlink-broadcast.test.ts new file mode 100644 index 00000000000..43fb243f54d --- /dev/null +++ b/src/main/ipc/ssh-managed-server-unlink-broadcast.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + send: vi.fn(), + notify: vi.fn(), + getState: vi.fn() +})) + +vi.mock('./ssh-ipc-context', () => ({ + connectionManager: { getState: mocks.getState }, + currentRuntime: { notifySshStateChanged: mocks.notify }, + getCurrentMainWindow: () => ({ isDestroyed: () => false, webContents: { send: mocks.send } }), + persistedStore: null, + portForwardManager: null +})) +vi.mock('../ssh/ssh-target-registry', async (importOriginal) => ({ + ...(await importOriginal>()), + getSshTargetRegistryStore: () => null +})) + +const { clearPublishedManagedServer, relayStateOverrides } = + await import('./ssh-renderer-broadcast') +const { getSshHostServerStatus, setSshHostServerStatus } = + await import('../ssh/ssh-host-server-status') + +describe('clearing a host’s managed server once it is unlinked', () => { + beforeEach(() => { + vi.clearAllMocks() + relayStateOverrides.clear() + }) + + it('forgets the environment and republishes the connection without it', () => { + setSshHostServerStatus('ssh-1', { kind: 'managed', environmentId: 'env-1' }) + mocks.getState.mockReturnValue({ + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + clearPublishedManagedServer('ssh-1') + expect(getSshHostServerStatus('ssh-1')).toBeUndefined() + // The runtime's notify is what drops the host's cached worktree scans. + const [, published] = mocks.notify.mock.calls[0] ?? [] + expect(published).toMatchObject({ targetId: 'ssh-1', status: 'connected' }) + expect(published).not.toHaveProperty('managedServer') + expect(mocks.send).toHaveBeenCalledWith( + 'ssh:state-changed', + expect.objectContaining({ targetId: 'ssh-1' }) + ) + }) + + it('strips the environment from a relay override too', () => { + relayStateOverrides.set('ssh-1', { + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0, + managedServer: { kind: 'managed', environmentId: 'env-1' } + }) + clearPublishedManagedServer('ssh-1') + expect(relayStateOverrides.get('ssh-1')).not.toHaveProperty('managedServer') + expect(mocks.notify.mock.calls[0]?.[1]).not.toHaveProperty('managedServer') + }) + + it('only clears when the host has no connection to republish', () => { + setSshHostServerStatus('ssh-1', { kind: 'managed', environmentId: 'env-1' }) + mocks.getState.mockReturnValue(undefined) + clearPublishedManagedServer('ssh-1') + expect(getSshHostServerStatus('ssh-1')).toBeUndefined() + expect(mocks.notify).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/ssh-plain-ssh-connect.test.ts b/src/main/ipc/ssh-plain-ssh-connect.test.ts index 4e33f77f976..974eb2eab18 100644 --- a/src/main/ipc/ssh-plain-ssh-connect.test.ts +++ b/src/main/ipc/ssh-plain-ssh-connect.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-pty-consumer-identity.test.ts b/src/main/ipc/ssh-pty-consumer-identity.test.ts index 23e698491bc..de2d3abb4c5 100644 --- a/src/main/ipc/ssh-pty-consumer-identity.test.ts +++ b/src/main/ipc/ssh-pty-consumer-identity.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-relay-reset-resume.test.ts b/src/main/ipc/ssh-relay-reset-resume.test.ts index ea460d777f3..b93d9f43099 100644 --- a/src/main/ipc/ssh-relay-reset-resume.test.ts +++ b/src/main/ipc/ssh-relay-reset-resume.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) diff --git a/src/main/ipc/ssh-renderer-broadcast.ts b/src/main/ipc/ssh-renderer-broadcast.ts index 999b8e9a346..112ad70d6ff 100644 --- a/src/main/ipc/ssh-renderer-broadcast.ts +++ b/src/main/ipc/ssh-renderer-broadcast.ts @@ -12,8 +12,11 @@ import { enrichSshForwardEntries, getWorktreeIdsForConnection } from '../ports/ssh-advertised-url-enrichment' +import { isRuntimeOwnedSshTarget } from '../ssh/ssh-connection-store' import { getSshProviderAuthority } from '../ssh/ssh-provider-authority' import { getSshPlainSshMode } from '../ssh/ssh-plain-ssh-mode' +import { clearSshHostServerStatus, getSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' import { activeSessions } from './ssh-active-relay-sessions' import { connectionManager, @@ -30,8 +33,9 @@ export function broadcastSshState( targetId: string, state: SshConnectionState ): void { - // Why: runtime-owned (ephemeral-VM) targets are hidden from the renderer, so broadcasting their state only triggers wasted listTargets() lookups. - if (isRuntimeOwnedSshTargetId(targetId)) { + const target = getSshTargetRegistryStore()?.getTarget(targetId) + // Why: owned targets are hidden from clients; broadcasting them leaks internal transport state into persisted reconnect hints. + if (isRuntimeOwnedSshTargetId(targetId) || (target && isRuntimeOwnedSshTarget(target))) { currentRuntime?.invalidateSshWorktreeScanCache?.(targetId) return } @@ -48,13 +52,15 @@ function withSshRemotePlatform(targetId: string, state: SshConnectionState): Ssh const remotePlatform = activeSessions.get(targetId)?.getHostPlatform()?.os const authority = getSshProviderAuthority(targetId) const plainSsh = state.status === 'connected' ? getSshPlainSshMode(targetId) : undefined + const managedServer = state.managedServer ?? getSshHostServerStatus(targetId) return { ...state, targetId, providerEpoch: authority.providerEpoch, connectionGeneration: authority.connectionGeneration, ...(remotePlatform ? { remotePlatform } : {}), - ...(plainSsh ? { plainSsh } : {}) + ...(plainSsh ? { plainSsh } : {}), + ...(managedServer ? { managedServer } : {}) } } @@ -79,6 +85,24 @@ export function connectionSupportsFolderDownload(targetId: string): boolean { return connectionManager?.getConnection(targetId)?.usesSystemSshTransport?.() !== true } +/** + * Forgets a host's managed-server decision once its server is unlinked, and republishes the + * connection without it. The republish is also what drops the host's cached worktree scans, so + * listings stop naming the removed server without waiting for a reconnect. + */ +export function clearPublishedManagedServer(targetId: string): void { + clearSshHostServerStatus(targetId) + const override = relayStateOverrides.get(targetId) + if (override?.managedServer) { + const { managedServer: _removed, ...rest } = override + relayStateOverrides.set(targetId, rest) + } + const state = relayStateOverrides.get(targetId) ?? connectionManager?.getState(targetId) + if (state) { + broadcastSshState(getCurrentMainWindow, targetId, state) + } +} + export function getPublicSshState(targetId: string): SshConnectionState | undefined { const state = relayStateOverrides.get(targetId) ?? connectionManager!.getState(targetId) return state ? withSshRemotePlatform(targetId, state) : undefined diff --git a/src/main/ipc/ssh-session-teardown.ts b/src/main/ipc/ssh-session-teardown.ts index 5ac34a11ad5..e8e46f32cb6 100644 --- a/src/main/ipc/ssh-session-teardown.ts +++ b/src/main/ipc/ssh-session-teardown.ts @@ -1,8 +1,15 @@ +import type { DirectSshAuthority } from '../../shared/ssh-types' import type { SshConnection } from '../ssh/ssh-connection' import type { SshRelaySession } from '../ssh/ssh-relay-session' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' +import { clearSshHostServerStatus } from '../ssh/ssh-host-server-status' +import { isSshConnectionSolelyOwnedBy } from '../ssh/ssh-connection-attribution' import { activeSessions } from './ssh-active-relay-sessions' -import { invalidateConnectAttempt } from './ssh-connect-attempt-registry' +import { + connectInFlight, + invalidateConnectAttempt, + isCurrentConnectAttempt +} from './ssh-connect-attempt-registry' import { connectionManager, persistedStore, portForwardManager } from './ssh-ipc-context' import { clearRelayLostBackoff } from './ssh-relay-lost-backoff' import { clearRelayStateOverride } from './ssh-renderer-broadcast' @@ -10,6 +17,7 @@ import { runTargetLifecycle } from './ssh-target-lifecycle-queue' export async function disconnectRegisteredSshTarget(targetId: string): Promise { invalidateConnectAttempt(targetId) + clearSshHostServerStatus(targetId) await runTargetLifecycle(targetId, () => teardownSshTargetTransport(targetId, (session) => session.detachAndPersist()) ) @@ -21,6 +29,7 @@ export async function removeRegisteredSshTarget(targetId: string): Promise return } invalidateConnectAttempt(targetId) + clearSshHostServerStatus(targetId) await runTargetLifecycle(targetId, async () => { try { // Why: removal is destructive; dispose so remote PTYs cannot reattach to a deleted target. @@ -165,3 +174,36 @@ export async function abandonCancelledConnectAttempt( ) } } + +/** + * A cancelled connect, or one whose server decision failed, closes the transport its own decision + * opened (a census, deploy or conversion), and only while nothing newer took it over: a completed + * replacement connect or a managed tunnel adopts it, and a pending replacement may be about to. + */ +export async function abandonDecisionTransport( + targetId: string, + owner: symbol, + authority: DirectSshAuthority +): Promise { + const opened = connectionManager!.getConnection(targetId) + const newer = connectInFlight.get(targetId) + // A pending replacement may be about to reuse it, though it has not adopted it yet. While this + // attempt is still current the in-flight entry is its own, so nothing newer can exist. + if ( + !opened || + !isSshConnectionSolelyOwnedBy(opened, owner) || + (newer && + isCurrentConnectAttempt(targetId, newer.authority) && + !isCurrentConnectAttempt(targetId, authority)) + ) { + return + } + try { + await connectionManager!.disconnectConnection(targetId, opened) + } catch (error) { + // Why: the caller is about to throw the cancellation; a teardown throw must not replace it. + console.warn( + `[ssh] Failed to close the transport a cancelled decision opened for ${targetId}: ${error instanceof Error ? error.message : String(error)}` + ) + } +} diff --git a/src/main/ipc/ssh-shutdown-drain.ts b/src/main/ipc/ssh-shutdown-drain.ts index f66db066c01..11e9b75c71d 100644 --- a/src/main/ipc/ssh-shutdown-drain.ts +++ b/src/main/ipc/ssh-shutdown-drain.ts @@ -69,7 +69,10 @@ function sshShutdownTasks(targetIds: readonly string[]): SshShutdownTask[] { targetId, promise: teardownActiveSshSession(targetId, (session) => session.detachAndPersist()) })), - { targetId: '*transports', promise: connectionManager?.disconnectAll() ?? Promise.resolve() } + { + targetId: '*transports', + promise: connectionManager?.disconnectAll() ?? Promise.resolve() + } ] } diff --git a/src/main/ipc/ssh-state-broadcast-fanout.test.ts b/src/main/ipc/ssh-state-broadcast-fanout.test.ts index e7f6480c261..5d3ef9c4a65 100644 --- a/src/main/ipc/ssh-state-broadcast-fanout.test.ts +++ b/src/main/ipc/ssh-state-broadcast-fanout.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) @@ -175,6 +176,41 @@ describe('SSH IPC handlers', () => { expect(runtime.notifySshStateChanged).not.toHaveBeenCalled() }) + it('keeps claimed managed-orcad SSH state off clients under the original target id', () => { + const runtime = { + onPtyData: vi.fn(), + onPtyExit: vi.fn(), + invalidateSshWorktreeScanCache: vi.fn(), + notifySshStateChanged: vi.fn() + } + registerSshHandlers(mockStore as never, () => mockWindow as never, runtime as never) + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Managed orcad host', + host: 'example.com', + port: 22, + username: 'deploy', + owner: { type: 'on-demand-runtime', runtimeId: 'managed-orcad:environment-1' } + } satisfies SshTarget) + const callbacks = mockConnectionManager.callbacksRef.current as { + onStateChange: (targetId: string, state: SshConnectionState) => void + } + + callbacks.onStateChange('ssh-1', { + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + + expect(runtime.invalidateSshWorktreeScanCache).toHaveBeenCalledWith('ssh-1') + expect(runtime.notifySshStateChanged).not.toHaveBeenCalled() + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith( + 'ssh:state-changed', + expect.anything() + ) + }) + it('invalidates runtime scans from hidden SSH state broadcasts', () => { const runtime = { onPtyData: vi.fn(), diff --git a/src/main/ipc/ssh-target-crud-handlers-ownership.test.ts b/src/main/ipc/ssh-target-crud-handlers-ownership.test.ts new file mode 100644 index 00000000000..630615f001c --- /dev/null +++ b/src/main/ipc/ssh-target-crud-handlers-ownership.test.ts @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' + +const mocks = vi.hoisted(() => { + const state: { target?: SshTarget } = {} + return { + handle: vi.fn(), + remove: vi.fn(), + state, + addTarget: vi.fn(), + updateTarget: vi.fn(), + closeTunnel: vi.fn(async () => {}), + disconnect: vi.fn(async () => {}) + } +}) + +vi.mock('electron', () => ({ ipcMain: { handle: mocks.handle } })) +vi.mock('../ssh/ssh-target-registry', () => ({ + getSshTargetRegistryStore: () => ({ + getTarget: () => mocks.state.target, + addTarget: mocks.addTarget, + updateTarget: mocks.updateTarget, + lastRepoReadoptions: [] + }) +})) +vi.mock('./ssh-session-teardown', () => ({ removeRegisteredSshTarget: mocks.remove })) +vi.mock('../ssh/orcad-managed-tunnel', () => ({ closeOrcadManagedTunnel: mocks.closeTunnel })) +vi.mock('./ssh-ipc-context', () => ({ + getCurrentMainWindow: () => null, + connectionManager: { disconnect: mocks.disconnect } +})) + +const { registerSshTargetCrudHandlers } = await import('./ssh-target-crud-handlers') + +function handler(channel: string): (_event: unknown, args: unknown) => unknown { + const registration = mocks.handle.mock.calls.find(([name]) => name === channel) + if (!registration) { + throw new Error(`${channel} handler was not registered`) + } + return registration[1] +} + +const target: SshTarget = { id: 'ssh-1', label: 'host', host: 'host', port: 22, username: 'dev' } + +describe('SSH target CRUD against managed orcad targets', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.state.target = { ...target, orcadFence: { environmentId: 'environment-1' } } + registerSshTargetCrudHandlers() + }) + + it("edits a managed host's connection, keeping its fence and redialing tunnel and transport", async () => { + mocks.updateTarget.mockReturnValue({ ...mocks.state.target, host: 'elsewhere' }) + handler('ssh:updateTarget')(null, { + id: 'ssh-1', + updates: { host: 'elsewhere', orcadFence: undefined, generation: 9 } + }) + expect(mocks.updateTarget).toHaveBeenCalledWith('ssh-1', { host: 'elsewhere' }) + await vi.waitFor(() => expect(mocks.disconnect).toHaveBeenCalledWith('ssh-1')) + expect(mocks.closeTunnel).toHaveBeenCalledWith('environment-1') + }) + + it('leaves the relay session of a host an older build changed alone when its connection is edited', async () => { + mocks.state.target = { + ...target, + orcadFence: { environmentId: 'environment-1', sourceChangedAt: '2026-10-01T00:00:00.000Z' } + } + mocks.updateTarget.mockReturnValue({ ...mocks.state.target, port: 2222 }) + handler('ssh:updateTarget')(null, { id: 'ssh-1', updates: { port: 2222 } }) + await vi.waitFor(() => expect(mocks.closeTunnel).toHaveBeenCalledWith('environment-1')) + expect(mocks.disconnect).not.toHaveBeenCalled() + }) + + it('keeps the SSH transport when only a label changes', async () => { + mocks.updateTarget.mockReturnValue({ ...mocks.state.target, label: 'renamed' }) + handler('ssh:updateTarget')(null, { id: 'ssh-1', updates: { label: 'renamed' } }) + await vi.waitFor(() => expect(mocks.closeTunnel).toHaveBeenCalledWith('environment-1')) + expect(mocks.disconnect).not.toHaveBeenCalled() + }) + + it('refuses to remove a managed host, pointing at Stop instead', async () => { + await expect(handler('ssh:removeTarget')(null, { id: 'ssh-1' })).rejects.toThrow( + 'Settings › Managed servers' + ) + mocks.state.target = { + ...target, + orcadProvisioning: { requestId: 'request-1', name: 'Managed' } + } + await expect(handler('ssh:removeTarget')(null, { id: 'ssh-1' })).rejects.toThrow( + 'managed Orca server' + ) + expect(mocks.remove).not.toHaveBeenCalled() + }) + + it('never lets the renderer write a provisioning intent', () => { + mocks.state.target = target + handler('ssh:updateTarget')(null, { + id: 'ssh-1', + updates: { label: 'renamed', orcadProvisioning: { requestId: 'x', name: 'y' } } + }) + handler('ssh:addTarget')(null, { + target: { ...target, orcadProvisioning: { requestId: 'x', name: 'y' } } + }) + expect(mocks.updateTarget).toHaveBeenCalledWith('ssh-1', { label: 'renamed' }) + expect(mocks.addTarget.mock.calls[0]?.[0]).not.toHaveProperty('orcadProvisioning') + }) +}) diff --git a/src/main/ipc/ssh-target-crud-handlers.ts b/src/main/ipc/ssh-target-crud-handlers.ts index e8b7a95c478..f87593ef1c4 100644 --- a/src/main/ipc/ssh-target-crud-handlers.ts +++ b/src/main/ipc/ssh-target-crud-handlers.ts @@ -9,9 +9,18 @@ import { listUserSshConfigHostSummaries, resolveUserSshConfigHost } from '../ssh/ssh-config-host-picker' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { closeOrcadManagedTunnel } from '../ssh/orcad-managed-tunnel' import { rotateSshProviderAuthority } from '../ssh/ssh-provider-authority' import { getSshTargetRegistryStore } from '../ssh/ssh-target-registry' -import { getCurrentMainWindow } from './ssh-ipc-context' +import { + allowsDirectSshRelay, + isManagedOrcadSshTarget, + isRuntimeOwnedSshTarget +} from '../ssh/ssh-connection-store' +import { connectionManager, getCurrentMainWindow } from './ssh-ipc-context' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' +import { fingerprintRuntimeSshTarget } from '../ssh/runtime-ssh-access' import { removeRegisteredSshTarget } from './ssh-session-teardown' // Why: add/import can re-adopt workspaces orphaned on a removed target id (see ssh-target-readoption); the renderer must refresh its repo list to surface them. @@ -34,18 +43,60 @@ function takeRepoReadoptions(): SshRepoReadoption[] { return repoReadoptions } -// Why: generations and the runtime ladder cache are main-owned; a renderer must not forge them. +// Why: generations, provisioning, the server fence and the runtime ladder cache are main-owned; +// a renderer must not forge them. function omitRendererSshTargetGeneration< - T extends { generation?: unknown; remoteRuntimeResolution?: unknown } ->(value: T): Omit { + T extends { + generation?: unknown + orcadProvisioning?: unknown + orcadFence?: unknown + managedServerUnavailable?: unknown + managedServerMoveOffered?: unknown + managedServerUpdateFailure?: unknown + remoteRuntimeResolution?: unknown + } +>( + value: T +): Omit< + T, + | 'generation' + | 'orcadProvisioning' + | 'orcadFence' + | 'managedServerUnavailable' + | 'managedServerMoveOffered' + | 'managedServerUpdateFailure' + | 'remoteRuntimeResolution' +> { const { generation: _generation, + orcadProvisioning: _orcadProvisioning, + orcadFence: _orcadFence, + managedServerUnavailable: _managedServerUnavailable, + managedServerMoveOffered: _managedServerMoveOffered, + managedServerUpdateFailure: _managedServerUpdateFailure, remoteRuntimeResolution: _remoteRuntimeResolution, ...rest } = value return rest } +function assertNotRuntimeOwned(targetId: string, action: string): void { + const target = getSshTargetRegistryStore()!.getTarget(targetId) + if (target && isRuntimeOwnedSshTarget(target)) { + throw new Error(`Managed runtime SSH targets cannot be ${action} from SSH settings.`) + } +} + +/** Removing a managed host would strand its server; Stop removes both and proves the exit. */ +export function assertNotManagedServerHost(targetId: string): void { + const target = getSshTargetRegistryStore()!.getTarget(targetId) + if (target && isManagedOrcadSshTarget(target)) { + throw new Error( + 'This host runs a managed Orca server. Use Stop… under Settings › Managed servers to stop the server and remove it first.' + ) + } +} + export function registerSshTargetCrudHandlers(): void { ipcMain.handle('ssh:listTargets', () => { return getSshTargetRegistryStore()!.listTargets() @@ -67,14 +118,35 @@ export function registerSshTargetCrudHandlers(): void { ipcMain.handle( 'ssh:updateTarget', (_event, args: { id: string; updates: SshTargetUpdateInput }) => { - return getSshTargetRegistryStore()!.updateTarget( + assertNotRuntimeOwned(args.id, 'edited') + const before = getSshTargetRegistryStore()!.getTarget(args.id) + // The fence and generation are stripped, so a managed host keeps its server binding. + const updated = getSshTargetRegistryStore()!.updateTarget( args.id, omitRendererSshTargetGeneration(args.updates) ) + const environmentId = getManagedOrcadFenceEnvironmentId(updated ?? undefined) + if (environmentId && updated) { + // Why: the tunnel and the SSH transport under it were built from the old fields, so both go + // and the next use dials the edited target. Only a host reached through its managed server: + // one an older build changed runs on the relay directly, whose session owns the transport. + const redial = + !allowsDirectSshRelay(updated) && + (!before || fingerprintRuntimeSshTarget(before) !== fingerprintRuntimeSshTarget(updated)) + void runTargetLifecycle(args.id, async () => { + await closeOrcadManagedTunnel(environmentId) + if (redial) { + await connectionManager?.disconnect(args.id) + } + }).catch(() => undefined) + } + return updated } ) ipcMain.handle('ssh:removeTarget', async (_event, args: { id: string }) => { + assertNotRuntimeOwned(args.id, 'removed') + assertNotManagedServerHost(args.id) await removeRegisteredSshTarget(args.id) }) diff --git a/src/main/ipc/ssh-target-lifecycle-queue.ts b/src/main/ipc/ssh-target-lifecycle-queue.ts index 014a5037175..f6c5221aa5c 100644 --- a/src/main/ipc/ssh-target-lifecycle-queue.ts +++ b/src/main/ipc/ssh-target-lifecycle-queue.ts @@ -1,25 +1,27 @@ // Serializes disconnect/remove/terminate/reset for a single SSH target so they cannot interleave. export const targetLifecycleInFlight = new Map>() -export function runTargetLifecycle( - targetId: string, - operation: () => Promise -): Promise { +export function runTargetLifecycle(targetId: string, operation: () => Promise): Promise { const prior = targetLifecycleInFlight.get(targetId) const operationPromise = (async () => { if (prior) { await prior.catch(() => undefined) } - await operation() + return operation() })() let trackedPromise!: Promise - trackedPromise = operationPromise.finally(() => { - if (targetLifecycleInFlight.get(targetId) === trackedPromise) { - targetLifecycleInFlight.delete(targetId) - } - }) + trackedPromise = operationPromise + .then( + () => undefined, + () => undefined + ) + .finally(() => { + if (targetLifecycleInFlight.get(targetId) === trackedPromise) { + targetLifecycleInFlight.delete(targetId) + } + }) targetLifecycleInFlight.set(targetId, trackedPromise) - return trackedPromise + return operationPromise } export async function awaitTargetLifecycle(targetId: string): Promise { diff --git a/src/main/ipc/ssh-target-registry.test.ts b/src/main/ipc/ssh-target-registry.test.ts index 17f82683832..9aa205d7a04 100644 --- a/src/main/ipc/ssh-target-registry.test.ts +++ b/src/main/ipc/ssh-target-registry.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) @@ -52,6 +53,7 @@ describe('SSH IPC handlers', () => { expect(channels).toContain('ssh:connect') expect(channels).toContain('ssh:disconnect') expect(channels).toContain('ssh:terminateSessions') + expect(channels).toContain('ssh:moveToManagedServer') expect(channels).toContain('ssh:resetRelay') expect(channels).toContain('ssh:getState') expect(channels).toContain('ssh:testConnection') diff --git a/src/main/ipc/ssh-terminate-sessions-previous-relay.test.ts b/src/main/ipc/ssh-terminate-sessions-previous-relay.test.ts new file mode 100644 index 00000000000..4d9cbe130b8 --- /dev/null +++ b/src/main/ipc/ssh-terminate-sessions-previous-relay.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = await vi.hoisted(async () => { + const { createSshIpcMocks } = await import('./ssh-ipc-module-mocks') + return createSshIpcMocks() +}) + +vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) +vi.mock('electron', () => mocks.electron) +vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) +vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) +vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) +vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) +vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) +vi.mock('../ssh/ssh-channel-multiplexer', () => mocks.sshChannelMultiplexer) +vi.mock('../providers/ssh-pty-provider', () => mocks.sshPtyProvider) +vi.mock('../providers/ssh-filesystem-provider', () => mocks.sshFilesystemProvider) +vi.mock('./pty', () => mocks.pty) +vi.mock('../providers/ssh-filesystem-dispatch', () => mocks.sshFilesystemDispatch) +vi.mock('../providers/ssh-git-provider', () => mocks.sshGitProvider) +vi.mock('../providers/ssh-git-dispatch', () => mocks.sshGitDispatch) +vi.mock('../ssh/ssh-port-forward', () => mocks.sshPortForward) +vi.mock('../ssh/ssh-port-scanner', () => mocks.sshPortScanner) +const { listPreviousRelayPtyIds } = vi.hoisted(() => ({ + listPreviousRelayPtyIds: vi.fn(async (): Promise => null) +})) +vi.mock('../ssh/ssh-legacy-relay-routing', async (importOriginal) => ({ + ...(await importOriginal>()), + listPreviousRelayPtyIds +})) +vi.mock('../ssh/ssh-previous-relay-terminals', () => ({ + isReattachHeldByPreviousRelay: vi.fn(async () => true), + startPreviousRelayCensus: vi.fn(), + clearPreviousRelayCensus: vi.fn() +})) + +import type { SshTarget } from '../../shared/ssh-types' +import { getSshPtyProvider, getPtyIdsForConnection } from './pty' +import { createSshIpcHarness } from './ssh-ipc-test-harness' + +const { mockSshStore, mockConnectionManager, mockPtyProvider } = mocks + +describe('ssh:terminateSessions while an older relay may hold terminals', () => { + const harness = createSshIpcHarness(mocks) + const { handlers, mockStore } = harness + + beforeEach(() => { + harness.reset() + listPreviousRelayPtyIds.mockReset().mockResolvedValue(null) + }) + + it('keeps a not-found terminal the previous relay may run and reports it unverifiable', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.connect.mockResolvedValue({}) + mockConnectionManager.getState.mockReturnValue({ + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + mockStore.getSshRemotePtyLeases.mockReturnValue([ + { targetId: 'ssh-1', ptyId: 'pty-held', state: 'detached' } + ]) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the shared IPC mock provider implements the shutdown this path calls. + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + mockPtyProvider.shutdown.mockRejectedValue(new Error('PTY "pty-held" not found')) + + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + await expect( + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1' }) + ).resolves.toEqual({ terminated: 0, unverifiable: 1 }) + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + 'ssh-1', + 'pty-held', + 'terminated' + ) + // The final teardown must not bulk-mark the held lease terminated either. + expect(mockStore.markSshRemotePtyLeasesAsync).not.toHaveBeenCalledWith('ssh-1', 'terminated') + expect(mockStore.markSshRemotePtyLeasesAsync).toHaveBeenCalledWith('ssh-1', 'detached') + }) + + // B4: a shell a respawn superseded on its tab still runs on the previous relay, leaseless. + it('stops a shell only an older relay lists, through the provider that routes it there', async () => { + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + }) + mockConnectionManager.connect.mockResolvedValue({}) + mockStore.getSshRemotePtyLeases.mockReturnValue([]) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the shared IPC mock provider implements the shutdown this path calls. + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + listPreviousRelayPtyIds.mockResolvedValue(['ssh:ssh-1@@pty2:old:1']) + mockPtyProvider.shutdown.mockResolvedValue(undefined) + + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + await expect( + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1' }) + ).resolves.toEqual({ terminated: 1, unverifiable: 0 }) + expect(mockPtyProvider.shutdown).toHaveBeenCalledWith('ssh:ssh-1@@pty2:old:1', { + immediate: true, + keepHistory: false + }) + }) +}) diff --git a/src/main/ipc/ssh-terminate-sessions.test.ts b/src/main/ipc/ssh-terminate-sessions.test.ts index d8a588a9bad..0fc752831ac 100644 --- a/src/main/ipc/ssh-terminate-sessions.test.ts +++ b/src/main/ipc/ssh-terminate-sessions.test.ts @@ -9,8 +9,13 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) +vi.mock( + '../ssh/ssh-previous-relay-terminals', + () => import('../ssh/ssh-previous-relay-census-test-double') +) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) vi.mock('../ssh/ssh-channel-multiplexer', () => mocks.sshChannelMultiplexer) vi.mock('../providers/ssh-pty-provider', () => mocks.sshPtyProvider) @@ -23,6 +28,7 @@ vi.mock('../ssh/ssh-port-forward', () => mocks.sshPortForward) vi.mock('../ssh/ssh-port-scanner', () => mocks.sshPortScanner) import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' +import { SshPtyHeldByPreviousRelayError } from '../providers/ssh-pty-errors' import type { SshConnectionState, SshTarget } from '../../shared/ssh-types' import { clearProviderPtyState, @@ -31,6 +37,7 @@ import { getPtyIdsForConnection } from './pty' import { createSshIpcHarness } from './ssh-ipc-test-harness' +import { setCurrentRuntime } from './ssh-ipc-context' const { mockSshStore, mockConnectionManager, mockPtyProvider, mockPortForwardManager } = mocks @@ -40,6 +47,23 @@ describe('SSH IPC handlers', () => { beforeEach(harness.reset) + it('ssh:terminateSessions refuses a removal of a managed server host before ending anything', async () => { + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy', + orcadFence: { environmentId: 'env-1' } + }) + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) + expect(() => + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1', forRemoval: true }) + ).toThrow('Settings › Managed servers') + expect(mockPtyProvider.shutdown).not.toHaveBeenCalled() + }) + it('ssh:terminateSessions preserves tracking when relay shutdown fails', async () => { const target: SshTarget = { id: 'ssh-1', @@ -279,6 +303,33 @@ describe('SSH IPC handlers', () => { ) }) + it('ssh:terminateSessions stops a shell the relay runs without any lease here', async () => { + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + }) + mockConnectionManager.connect.mockResolvedValue({}) + mockStore.getSshRemotePtyLeases.mockReturnValue([]) + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + // A CLI-created terminal: the relay lists it, but no lease or local pane knows it. + mockPtyProvider.listProcesses.mockResolvedValue([{ id: 'ssh:ssh-1@@pty-cli' }]) + mockPtyProvider.shutdown.mockResolvedValue(undefined) + + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + await expect( + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1' }) + ).resolves.toMatchObject({ terminated: 1, unverifiable: 0 }) + + expect(mockPtyProvider.shutdown).toHaveBeenCalledWith('ssh:ssh-1@@pty-cli', { + immediate: true, + keepHistory: false + }) + }) + it('ssh:terminateSessions tombstones an expired lease the relay reports gone', async () => { const target: SshTarget = { id: 'ssh-1', @@ -312,6 +363,39 @@ describe('SSH IPC handlers', () => { ) }) + it('ssh:terminateSessions keeps a lease an older relay holds but no route can stop', async () => { + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + }) + mockConnectionManager.connect.mockResolvedValue({}) + mockConnectionManager.getState.mockReturnValue({ + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + mockStore.getSshRemotePtyLeases.mockReturnValue([ + { targetId: 'ssh-1', ptyId: 'pty2:old:1', state: 'detached' } + ]) + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + mockPtyProvider.shutdown.mockRejectedValue(new SshPtyHeldByPreviousRelayError('pty2:old:1')) + + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + await expect( + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1' }) + ).resolves.toEqual({ terminated: 0, unverifiable: 1 }) + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + 'ssh-1', + 'pty2:old:1', + 'terminated' + ) + }) + it('ssh:terminateSessions leaves leases it already proved terminated alone', async () => { const target: SshTarget = { id: 'ssh-1', @@ -375,4 +459,37 @@ describe('SSH IPC handlers', () => { 'terminated' ) }) + + it('ssh:terminateSessions tells the runtime a stopped shell is no longer connected', async () => { + mockSshStore.getTarget.mockReturnValue({ + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + }) + mockStore.getSshRemotePtyLeases.mockReturnValue([]) + vi.mocked(getSshPtyProvider).mockReturnValue(mockPtyProvider as never) + vi.mocked(getPtyIdsForConnection).mockReturnValue([ + 'ssh:ssh-1@@pty-lost-exit', + 'ssh:ssh-1@@pty-reported-exit' + ]) + mockPtyProvider.shutdown.mockResolvedValue(undefined) + const runtime = { + getPtyLivenessVerdict: vi.fn((ptyId: string) => + ptyId.endsWith('pty-reported-exit') ? { status: 'exited' } : null + ), + onPtyExit: vi.fn() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: terminate reads only these two runtime methods. + setCurrentRuntime(runtime as never) + + await expect( + handlers.get('ssh:terminateSessions')!(null, { targetId: 'ssh-1' }) + ).resolves.toEqual({ terminated: 2, unverifiable: 0 }) + + // The stop sentinel, never a certified death; an exit the relay already reported stands. + expect(runtime.onPtyExit.mock.calls).toEqual([['ssh:ssh-1@@pty-lost-exit', -1]]) + setCurrentRuntime(undefined) + }) }) diff --git a/src/main/ipc/ssh-terminate-sessions.ts b/src/main/ipc/ssh-terminate-sessions.ts new file mode 100644 index 00000000000..9abc7d4ba12 --- /dev/null +++ b/src/main/ipc/ssh-terminate-sessions.ts @@ -0,0 +1,189 @@ +import { + sshRemotePtyLeaseAllowsReattach, + type SshTerminateSessionsResult +} from '../../shared/ssh-types' +import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants' +import { UNVERIFIED_PROCESS_EXIT_CODE } from '../../shared/terminal-exit-cause' +import { isSshPtyNotFoundError, SshPtyHeldByPreviousRelayError } from '../providers/ssh-pty-errors' +import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' +import { isReattachHeldByPreviousRelay } from '../ssh/ssh-previous-relay-terminals' +import { listPreviousRelayPtyIds } from '../ssh/ssh-legacy-relay-routing' +import { + clearProviderPtyState, + deletePtyOwnership, + getPtyIdsForConnection, + getSshPtyProvider +} from './pty' +import { invalidateConnectAttempt } from './ssh-connect-attempt-registry' +import { currentRuntime, persistedStore } from './ssh-ipc-context' +import { ptyIncarnationById } from './pty/provider/ownership-state' +import type { TerminalIntentionalStopKind } from '../runtime/terminal-intentional-stops' +import { teardownSshTargetTransport } from './ssh-session-teardown' +import { runTargetLifecycle } from './ssh-target-lifecycle-queue' + +export type SshTerminateSessionsOptions = { + /** Records each stop as main's own, so every viewer keeps the tab through its exit. */ + intentionalStop?: TerminalIntentionalStopKind + /** Each shell this call stopped, reported even when a later one fails. */ + onStopped?: (appPtyId: string) => void +} + +/** Stops every relay terminal on the target and closes its transport (`ssh:terminateSessions`). */ +export async function terminateSshTargetSessions( + targetId: string, + options: SshTerminateSessionsOptions = {} +): Promise { + invalidateConnectAttempt(targetId) + // Why (#12661): an offline sweep tears down local transport only. The caller must be able to tell + // "the host stopped these" from "nobody asked the host", so carry the verdict out of the lifecycle queue. + let outcome: SshTerminateSessionsResult = { terminated: 0, unverifiable: 0 } + await runTargetLifecycle(targetId, async () => { + const provider = getSshPtyProvider(targetId) + const leases = persistedStore!.getSshRemotePtyLeases(targetId) + const ptyIdsByRelayId = new Map() + // Why: only leases the app still believes it owns may force a reconnect; a lease whose route + // died for good is swept opportunistically instead, so a target that can no longer answer + // never blocks its own removal (issue #2626, and the renderer tolerates the refusal there). + const ownedRelayIds = new Set() + const trackPtyId = (ptyId: string, owned: boolean): void => { + const relayPtyId = toRelaySshPtyId(targetId, ptyId) + if (!ptyIdsByRelayId.has(relayPtyId)) { + ptyIdsByRelayId.set(relayPtyId, toAppSshPtyId(targetId, ptyId)) + } + if (owned) { + ownedRelayIds.add(relayPtyId) + } + } + for (const ptyId of getPtyIdsForConnection(targetId)) { + trackPtyId(ptyId, true) + } + for (const lease of leases) { + if (lease.state === 'terminated') { + continue + } + // Why the predicate and not `state !== 'expired'`: an `expired` lease carrying no + // retirement mark records only that reattach gave up, never that the remote shell died, so + // it is exactly the orphan the user's terminate must reach — and reaching it needs the + // relay, which is what the fence below demands. Only `supersededBy` / `relayIdRecycled` + // prove the route is dead for good, and those stay unowned. + trackPtyId(lease.ptyId, sshRemotePtyLeaseAllowsReattach(lease)) + } + // A shell a relay runs without any lease here (a CLI-created terminal, or one a respawn + // superseded on its tab) is still this host's; shutdown stops an earlier relay's held one there. + for (const ptyId of await listRelayPtyIdsToStop(targetId, provider)) { + trackPtyId(ptyId, false) + } + const ptyIds = Array.from(ptyIdsByRelayId, ([relayPtyId, appPtyId]) => ({ + relayPtyId, + appPtyId + })) + + if (ownedRelayIds.size > 0 && !provider) { + throw new Error( + `${SSH_TERMINATE_RECONNECT_REQUIRED}: SSH relay is not connected; reconnect before terminating remote sessions.` + ) + } + const shutdownResults = provider + ? await Promise.allSettled( + ptyIds.map(({ appPtyId }) => + shutdownAs(options, appPtyId, () => + provider.shutdown(appPtyId, { immediate: true, keepHistory: false }) + ) + ) + ) + : [] + if (!provider) { + // Nothing observed these remote shells, so their state is unknown — not "nothing to do". + outcome = { terminated: 0, unverifiable: ptyIds.length } + } + const shutdownFailures: string[] = [] + for (const [index, result] of shutdownResults.entries()) { + const { appPtyId, relayPtyId } = ptyIds[index] + if ( + result.status !== 'fulfilled' && + (result.reason instanceof SshPtyHeldByPreviousRelayError || + (await isReattachHeldByPreviousRelay(targetId, result.reason))) + ) { + // Not found here is not absence while an older build's relay may still run it (#25124). + outcome = { ...outcome, unverifiable: outcome.unverifiable + 1 } + continue + } + if (result.status !== 'fulfilled' && !isSshPtyNotFoundError(result.reason)) { + shutdownFailures.push( + `${relayPtyId}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}` + ) + continue + } + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + persistedStore!.markSshRemotePtyLease(targetId, relayPtyId, 'terminated') + reportStoppedPtyToRuntime(appPtyId) + if (result.status === 'fulfilled') { + options.onStopped?.(appPtyId) + } + outcome = { ...outcome, terminated: outcome.terminated + 1 } + } + if (shutdownFailures.length > 0) { + // Why: a failed relay shutdown can leave the remote process alive in the grace window; keep the lease/session so the user can retry. + throw new Error(`Failed to terminate SSH host sessions: ${shutdownFailures.join('; ')}`) + } + // Disposal marks every remaining lease terminated; an unverifiable PTY keeps its lease detached, + // since only the leases this run proved terminated were marked so above. + await teardownSshTargetTransport(targetId, (session) => + outcome.unverifiable > 0 ? session.detachAndPersist() : session.disposeAndPersist() + ) + }) + return outcome +} + +/** An unanswered listing adds nothing here; the move's census after the stop still asks the host. */ +async function listRelayPtyIdsToStop( + targetId: string, + provider: ReturnType +): Promise { + if (!provider) { + return [] + } + const [current, previous] = await Promise.all([ + provider.listProcesses().then( + (rows) => rows.map((row) => row.id), + () => [] + ), + listPreviousRelayPtyIds(targetId).catch(() => null) + ]) + return [...current, ...(previous ?? [])] +} + +/** Marks exactly this shell, from just before its shutdown, so earlier exits close normally. */ +async function shutdownAs( + options: SshTerminateSessionsOptions, + appPtyId: string, + shutdown: () => Promise +): Promise { + const settle = options.intentionalStop + ? currentRuntime?.intentionalPtyStops.mark( + appPtyId, + options.intentionalStop, + ptyIncarnationById.get(appPtyId) ?? null + ) + : undefined + let stopped = false + try { + await shutdown() + stopped = true + } finally { + settle?.(stopped) + } +} + +/** + * Why: a relay that hangs up after its last shell stops never sends that exit, which left + * `terminal list` showing the stopped shell as connected. The relay accepting the kill is not an + * observed exit, so this is the stop sentinel, and a real exit already reported is kept. + */ +function reportStoppedPtyToRuntime(appPtyId: string): void { + if (currentRuntime?.getPtyLivenessVerdict(appPtyId)?.status === 'exited') { + return + } + currentRuntime?.onPtyExit(appPtyId, UNVERIFIED_PROCESS_EXIT_CODE) +} diff --git a/src/main/ipc/ssh-test-connection-probe-registry.test.ts b/src/main/ipc/ssh-test-connection-probe-registry.test.ts new file mode 100644 index 00000000000..c8b75e1c1ea --- /dev/null +++ b/src/main/ipc/ssh-test-connection-probe-registry.test.ts @@ -0,0 +1,98 @@ +import { expect, it, vi } from 'vitest' +import { + awaitSshTestConnectionProbes, + credentialRequestedForTarget, + runSshTestConnectionProbe, + testConnectionProbes, + testingTargets +} from './ssh-connect-attempt-registry' + +function deferred() { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +it('publishes global and target tracking before callback-capable work', async () => { + const close = deferred() + let joined!: Promise + const finished = vi.fn() + const operation = vi.fn(async () => { + expect(testConnectionProbes.has(probe)).toBe(true) + expect(testingTargets.has('published')).toBe(true) + joined = awaitSshTestConnectionProbes('published').then(finished) + await close.promise + return 'connected' + }) + const probe = runSshTestConnectionProbe('published', operation) + expect(operation).not.toHaveBeenCalled() + expect(testConnectionProbes.has(probe)).toBe(true) + await Promise.resolve() + expect(finished).not.toHaveBeenCalled() + close.resolve() + await expect(probe).resolves.toBe('connected') + await joined + expect(finished).toHaveBeenCalledTimes(1) + expect(testConnectionProbes.has(probe)).toBe(false) + expect(testingTargets.has('published')).toBe(false) +}) + +it('joins every admitted probe and retains callback suppression until the final settlement', async () => { + const first = deferred() + const second = deferred() + const a = runSshTestConnectionProbe('multiple', () => first.promise) + const b = runSshTestConnectionProbe('multiple', () => second.promise) + credentialRequestedForTarget.add('multiple') + const finished = vi.fn() + const joined = awaitSshTestConnectionProbes('multiple').then(finished) + first.resolve() + await a + expect(testingTargets.has('multiple')).toBe(true) + expect(credentialRequestedForTarget.has('multiple')).toBe(true) + expect(finished).not.toHaveBeenCalled() + second.resolve() + await b + await joined + expect(testingTargets.has('multiple')).toBe(false) + expect(credentialRequestedForTarget.has('multiple')).toBe(false) +}) + +it('joins failed probes without mistaking rejection for unfinished work', async () => { + const failure = new Error('connection failed') + const probe = runSshTestConnectionProbe('failed', async () => { + throw failure + }) + const result = expect(probe).rejects.toBe(failure) + await expect(awaitSshTestConnectionProbes('failed')).resolves.toBeUndefined() + await result + expect(testConnectionProbes.has(probe)).toBe(false) + expect(testingTargets.has('failed')).toBe(false) +}) + +it('does not wait for another target', async () => { + const close = deferred() + const probe = runSshTestConnectionProbe('other', () => close.promise) + await awaitSshTestConnectionProbes('absent') + expect(testConnectionProbes.has(probe)).toBe(true) + close.resolve() + await probe +}) + +it('rechecks the target after a joined snapshot settles', async () => { + const first = deferred() + const second = deferred() + const a = runSshTestConnectionProbe('later', () => first.promise) + const finished = vi.fn() + const joined = awaitSshTestConnectionProbes('later').then(finished) + const b = runSshTestConnectionProbe('later', () => second.promise) + first.resolve() + await a + await Promise.resolve() + expect(finished).not.toHaveBeenCalled() + second.resolve() + await b + await joined + expect(finished).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/ipc/ssh-test-probe-presence.test.ts b/src/main/ipc/ssh-test-probe-presence.test.ts new file mode 100644 index 00000000000..73287c4798f --- /dev/null +++ b/src/main/ipc/ssh-test-probe-presence.test.ts @@ -0,0 +1,36 @@ +import { expect, it } from 'vitest' +import { + hasSshTestConnectionProbes, + runSshTestConnectionProbe, + testingTargets +} from './ssh-connect-attempt-registry' + +it('publishes target-scoped presence before callbacks and retains it until settlement', async () => { + const target = 'probe-presence-success' + const work = Promise.withResolvers() + const probe = runSshTestConnectionProbe(target, async () => { + expect(hasSshTestConnectionProbes(target)).toBe(true) + await work.promise + }) + expect(hasSshTestConnectionProbes(target)).toBe(true) + expect(hasSshTestConnectionProbes('unrelated')).toBe(false) + testingTargets.delete(target) + expect(hasSshTestConnectionProbes(target)).toBe(true) + work.resolve() + await probe + expect(hasSshTestConnectionProbes(target)).toBe(false) +}) + +it('keeps other probes visible after a failed probe settles', async () => { + const target = 'probe-presence-failure' + const work = Promise.withResolvers() + const pending = runSshTestConnectionProbe(target, () => work.promise) + const failed = runSshTestConnectionProbe(target, async () => { + throw new Error('failed') + }) + await expect(failed).rejects.toThrow('failed') + expect(hasSshTestConnectionProbes(target)).toBe(true) + work.resolve() + await pending + expect(hasSshTestConnectionProbes(target)).toBe(false) +}) diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index 0e0f7bd9de0..8b07ca103b2 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -9,6 +9,7 @@ vi.mock('../ssh/ssh-config-host-picker', () => mocks.sshConfigHostPicker) vi.mock('electron', () => mocks.electron) vi.mock('./ssh-pty-output-intake-registry', () => mocks.sshPtyOutputIntakeRegistry) vi.mock('../ssh/ssh-connection-store', () => mocks.sshConnectionStore) +vi.mock('./ssh-host-server-connect', () => mocks.hostServerConnect) vi.mock('../ssh/ssh-connection-manager', () => mocks.sshConnectionManager) vi.mock('../ssh/ssh-relay-deploy', () => mocks.sshRelayDeploy) vi.mock('../ssh/ssh-relay-reset', () => mocks.sshRelayReset) @@ -26,6 +27,8 @@ import { RelayVersionMismatchError } from '../ssh/ssh-relay-version-mismatch-err import type { SshConnectionState, SshConnectionStatus, SshTarget } from '../../shared/ssh-types' import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation' import { createSshIpcHarness } from './ssh-ipc-test-harness' +import { decideHostServer } from './ssh-host-server-connect' +import { beginSshHostCensus } from '../ssh/ssh-connection-attribution' const { mockSshStore, @@ -375,6 +378,106 @@ describe('SSH IPC handlers', () => { expect(connectedBroadcasts).toEqual([]) }) + it('holds a raw connected the server decision causes before any relay session exists', async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const conn = {} + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.connect.mockResolvedValue(conn) + mockConnectionManager.getConnection.mockReturnValue(conn) + let broadcastsDuringDecision: unknown[] = [] + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + await Promise.resolve() + // The census dials the shared pool before doConnect has a session. + const callbacks = mockConnectionManager.callbacksRef.current as { + onStateChange: (targetId: string, state: SshConnectionState) => void + } + callbacks.onStateChange('ssh-1', { + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + broadcastsDuringDecision = mockWindow.webContents.send.mock.calls + .filter(([channel]) => channel === 'ssh:state-changed') + .map(([, payload]) => payload) + return null + }) + + await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) + expect(broadcastsDuringDecision).not.toContainEqual( + expect.objectContaining({ state: expect.objectContaining({ status: 'connected' }) }) + ) + expect(broadcastsDuringDecision.at(-1)).toMatchObject({ state: { status: 'connecting' } }) + }) + + it("never broadcasts 'connected' while a move's census runs with no relay session", async () => { + const target: SshTarget = { + id: 'ssh-1', + label: 'Server', + host: 'example.com', + port: 22, + username: 'deploy' + } + const conn = {} + mockSshStore.getTarget.mockReturnValue(target) + mockConnectionManager.connect.mockResolvedValue(conn) + mockConnectionManager.getConnection.mockReturnValue(undefined) + harness.mockStore.getSshRemotePtyLeases.mockReturnValue([]) + let broadcastsDuringCensus: unknown[] = [] + vi.mocked(decideHostServer).mockImplementationOnce(async () => { + await Promise.resolve() + // The move's census is the reconnect's decision, dialing the pool before any session. + const callbacks = mockConnectionManager.callbacksRef.current as { + onStateChange: (targetId: string, state: SshConnectionState) => void + } + callbacks.onStateChange('ssh-1', { + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + broadcastsDuringCensus = mockWindow.webContents.send.mock.calls + .filter(([channel]) => channel === 'ssh:state-changed') + .map(([, payload]) => payload) + return null + }) + + await handlers.get('ssh:moveToManagedServer')!(null, { targetId: 'ssh-1' }) + expect(vi.mocked(decideHostServer)).toHaveBeenCalled() + expect(broadcastsDuringCensus.length).toBeGreaterThan(0) + expect(broadcastsDuringCensus).not.toContainEqual( + expect.objectContaining({ state: expect.objectContaining({ status: 'connected' }) }) + ) + }) + + it("holds a census's raw 'connected' outside any connect", () => { + const end = beginSshHostCensus('ssh-1') + try { + const callbacks = mockConnectionManager.callbacksRef.current as { + onStateChange: (targetId: string, state: SshConnectionState) => void + } + mockWindow.webContents.send.mockClear() + callbacks.onStateChange('ssh-1', { + targetId: 'ssh-1', + status: 'connected', + error: null, + reconnectAttempt: 0 + }) + expect(mockWindow.webContents.send).toHaveBeenCalledWith( + 'ssh:state-changed', + expect.objectContaining({ state: expect.objectContaining({ status: 'connecting' }) }) + ) + } finally { + end() + } + }) + // Why: guards the fix's scope. A relay version mismatch during a relay reconnect // strands the session 'idle' in activeSessions (only doConnect deletes it). A later // transport blip then delivers a raw 'connected' with NO connect in flight — the diff --git a/src/main/ipc/ssh.ts b/src/main/ipc/ssh.ts index 9ccf4ad16f6..cb993f03e2a 100644 --- a/src/main/ipc/ssh.ts +++ b/src/main/ipc/ssh.ts @@ -41,6 +41,7 @@ import { } from '../ssh/ssh-connection-generation' import { resetSshProviderAuthorities } from '../ssh/ssh-provider-authority' import { activeSessions } from './ssh-active-relay-sessions' +import { installManagedOrcadStartStatus } from './runtime-environment-managed-tunnel' import { registerAdvertisedUrlRefresh, unregisterAdvertisedUrlRefresh @@ -77,6 +78,10 @@ import { broadcastPortForwards, relayStateOverrides } from './ssh-renderer-broad import { resetSshShutdownDrain } from './ssh-shutdown-drain' import { registerSshTargetCrudHandlers } from './ssh-target-crud-handlers' import { targetLifecycleInFlight } from './ssh-target-lifecycle-queue' +import { disposeOrcadManagedTunnels } from '../ssh/orcad-managed-tunnel' +import { reconcileManagedOrcadSshTargets } from '../ssh/orcad-retained-source' +import { installOrcadMigrationScrollbackRetention } from '../ssh/orcad-migration-scrollback-retention-wiring' +import { getAppEnvironment } from '../../shared/app-environment' const SSH_IPC_CHANNELS = [ 'ssh:listTargets', @@ -90,6 +95,7 @@ const SSH_IPC_CHANNELS = [ 'ssh:connect', 'ssh:disconnect', 'ssh:terminateSessions', + 'ssh:moveToManagedServer', 'ssh:resetRelay', 'ssh:getState', 'ssh:needsPassphrasePrompt', @@ -184,7 +190,10 @@ export function registerSshHandlers( setCurrentRuntime(runtime) setSshTargetRegistryStore(new SshConnectionStore(store)) setPersistedStore(store) + reconcileManagedOrcadSshTargets(getAppEnvironment().getPath('userData'), store) + installOrcadMigrationScrollbackRetention(getAppEnvironment().getPath('userData'), store) registerAdvertisedUrlRefresh(getCurrentMainWindow) + installManagedOrcadStartStatus() registerCredentialHandler() @@ -209,7 +218,7 @@ export function registerSshHandlers( } }) refreshActiveRelaySessions() - registerPowerMonitorReconnect() + registerPowerMonitorReconnect(() => getAppEnvironment().getPath('userData')) registerSshBrowseHandler(() => connectionManager) setSshConnectionManagerResolver(() => connectionManager) @@ -253,6 +262,7 @@ export async function resetSshHandlerStateForTests(): Promise { resetSshShutdownDrain() await connectionManager?.disconnectAll() + disposeOrcadManagedTunnels() portForwardManager?.dispose() setConnectionManager(null) setSshConnectionManagerResolver(null) diff --git a/src/main/ipc/telemetry.ts b/src/main/ipc/telemetry.ts index 83cc6579591..164a1ef2ac9 100644 --- a/src/main/ipc/telemetry.ts +++ b/src/main/ipc/telemetry.ts @@ -31,7 +31,11 @@ const MAIN_OWNED_TELEMETRY_EVENTS = new Set([ 'daemon_pty_cwd_readable', 'star_nag_outcome', 'feature_interaction_usage_bucket_reached', - 'ssh_remote_runtime_resolved' + 'ssh_remote_runtime_resolved', + 'ssh_host_server_decided', + 'ssh_host_server_conversion', + 'ssh_host_server_deploy_failed', + 'ssh_host_server_move' ]) /** diff --git a/src/main/ipc/worktrees-ssh-provider-authority.test.ts b/src/main/ipc/worktrees-ssh-provider-authority.test.ts index 8d14755ab18..4e50260cbaa 100644 --- a/src/main/ipc/worktrees-ssh-provider-authority.test.ts +++ b/src/main/ipc/worktrees-ssh-provider-authority.test.ts @@ -95,6 +95,92 @@ describe('registerWorktreeHandlers', () => { setupWorktreeHandlers() }) + it.each(['resolve', 'reject'] as const)( + 'answers a canceled SSH listing without writing when the provider settles late: %s', + async (outcome) => { + const targetId = `continuation-cancel-${outcome}` + const repo = { + id: `repo-${targetId}`, + path: '/remote/repo', + displayName: 'repo', + badgeColor: '#000', + addedAt: 0, + connectionId: targetId + } + let resolveProvider!: (value: GitWorktreeInfo[]) => void + let rejectProvider!: (error: Error) => void + const provider = { + listWorktrees: vi.fn( + () => + new Promise((resolve, reject) => { + resolveProvider = resolve + rejectProvider = reject + }) + ) + } + store.getRepos.mockReturnValue([repo]) + getSshGitProviderMock.mockReturnValue(provider) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: request ids are minted by the renderer; any unique string is a valid id here. + const providerRequestId = `request-${targetId}` as ProviderRequestId + const pending = handlers['worktrees:listDetected'](ipcEvent, { + providerRequestId, + repoId: repo.id, + executionHostId: toSshExecutionHostId(targetId), + expectedAuthority: getSshProviderAuthority(targetId) + }) + await Promise.resolve() + expect(provider.listWorktrees).toHaveBeenCalledOnce() + handlers['worktrees:cancelListDetected'](ipcEvent, { providerRequestId }) + await expect(pending).resolves.toMatchObject({ status: 'canceled' }) + if (outcome === 'resolve') { + resolveProvider([]) + } else { + rejectProvider(new Error('late provider failure')) + } + await new Promise((resolve) => setImmediate(resolve)) + expect(store.setWorktreeMeta).not.toHaveBeenCalled() + expect(store.removeWorktreeLineage).not.toHaveBeenCalled() + } + ) + + it('lists a local folder while a legacy SSH listing is still pending', async () => { + const targetId = 'continuation-legacy' + const repo = { + id: 'repo-legacy', + path: '/remote/repo', + displayName: 'repo', + badgeColor: '#000', + addedAt: 0, + connectionId: targetId + } + const localRepo: Repo = { + id: 'repo-local', + path: '/local/repo', + displayName: 'local', + badgeColor: '#000', + addedAt: 0, + kind: 'folder' + } + let resolveProvider!: (value: GitWorktreeInfo[]) => void + const provider = { + listWorktrees: vi.fn( + () => + new Promise((resolve) => { + resolveProvider = resolve + }) + ) + } + store.getRepos.mockReturnValue([repo, localRepo]) + getSshGitProviderMock.mockReturnValue(provider) + const pending = handlers['worktrees:listDetected'](ipcEvent, { repoId: repo.id }) + await expect( + handlers['worktrees:listDetected'](ipcEvent, { repoId: localRepo.id }) + ).resolves.toMatchObject({ repoId: localRepo.id, authoritative: true }) + expect(provider.listWorktrees).toHaveBeenCalledOnce() + resolveProvider([]) + await expect(pending).resolves.toMatchObject({ repoId: repo.id, authoritative: true }) + }) + it.each([ ['malformed', 'ssh:%'], ['contradictory', toSshExecutionHostId('target-b')] diff --git a/src/main/ipc/worktrees/listing/register-detected-worktree-handlers.ts b/src/main/ipc/worktrees/listing/register-detected-worktree-handlers.ts index 442a70985c8..26b865b64fb 100644 --- a/src/main/ipc/worktrees/listing/register-detected-worktree-handlers.ts +++ b/src/main/ipc/worktrees/listing/register-detected-worktree-handlers.ts @@ -61,7 +61,9 @@ export function registerDetectedWorktreeHandlers(context: WorktreeIpcContext): v status: timedOut ? 'timed-out' : 'canceled' }) } - controller.signal.addEventListener('abort', onAbort, { once: true }) + controller.signal.addEventListener('abort', onAbort, { + once: true + }) removeAbortListener = () => controller.signal.removeEventListener('abort', onAbort) }) : undefined diff --git a/src/main/memory/hydrate-local-pty-registry-folder-workspaces.test.ts b/src/main/memory/hydrate-local-pty-registry-folder-workspaces.test.ts new file mode 100644 index 00000000000..c67e4772bee --- /dev/null +++ b/src/main/memory/hydrate-local-pty-registry-folder-workspaces.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import type { FolderWorkspace } from '../../shared/folder-workspace-types' +import type { SessionInfo } from '../daemon/types' +import type { Store } from '../persistence' + +const getDaemonProviderMock = vi.fn() +vi.mock('../daemon/daemon-init', () => ({ + getDaemonProvider: () => getDaemonProviderMock() +})) +vi.mock('../project-runtime-git-options', () => ({ + getLocalProjectWorktreeGitOptions: () => ({}) +})) +const listLocalRepoWorktreesStrictMock = vi.fn() +vi.mock('../repo-worktrees', () => ({ + listLocalRepoWorktreesStrict: (...args: unknown[]) => listLocalRepoWorktreesStrictMock(...args) +})) + +function makeStore(folderWorkspaces: FolderWorkspace[]): Store { + const store: Partial = { + getRepos: () => [], + getFolderWorkspaces: () => folderWorkspaces, + getProjectGroups: () => [], + getAllWorktreeMeta: () => ({}), + getAllWorktreeMetaForHost: () => ({}) + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: hydration reads only the store members stubbed here. + return store as Store +} + +// Why fresh modules: the hydrator memoizes its pass and the registry is module-scoped. +async function loadFresh() { + vi.resetModules() + const hydrateMod = await import('./hydrate-local-pty-registry') + const registryMod = await import('./pty-registry') + return { + hydrate: hydrateMod.hydrateLocalPtyRegistryAtBoot, + listRegisteredPtys: registryMod.listRegisteredPtys + } +} + +beforeEach(() => { + getDaemonProviderMock.mockReset() + listLocalRepoWorktreesStrictMock.mockReset() +}) + +it('hydrates surviving true folder workspace PTYs without enumerating Git', async () => { + const { hydrate, listRegisteredPtys } = await loadFresh() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: ownership reads only id and executionHostId. + const workspace = { id: 'folder-workspace-1', executionHostId: 'local' } as FolderWorkspace + const session = { sessionId: 'folder:folder-workspace-1@@cafebabe', pid: 4242 } + getDaemonProviderMock.mockReturnValue({ listSessions: vi.fn().mockResolvedValue([session]) }) + + await hydrate(makeStore([workspace])) + + expect(listRegisteredPtys()).toEqual([ + expect.objectContaining({ + ptyId: 'folder:folder-workspace-1@@cafebabe', + worktreeId: 'folder:folder-workspace-1', + pid: 4242 + }) + ]) + expect(listLocalRepoWorktreesStrictMock).not.toHaveBeenCalled() +}) + +it.each(['deleted', 'remote', 'ssh'] as const)( + 'rechecks folder ownership after inventory when the catalog becomes %s', + async (change) => { + const { hydrate, listRegisteredPtys } = await loadFresh() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: ownership reads only id and executionHostId. + const folders = [{ id: 'folder-1', executionHostId: 'local' } as FolderWorkspace] + getDaemonProviderMock.mockReturnValue({ + listSessions: vi.fn().mockImplementation(async (): Promise[]> => { + if (change === 'deleted') { + folders.splice(0) + } else { + folders[0].executionHostId = change === 'ssh' ? 'ssh:target-1' : 'runtime:environment-1' + } + return [{ sessionId: 'folder:folder-1@@cafebabe', pid: 4242 }] + }) + }) + + await hydrate(makeStore(folders)) + + expect(listRegisteredPtys()).toEqual([]) + expect(listLocalRepoWorktreesStrictMock).not.toHaveBeenCalled() + } +) diff --git a/src/main/memory/hydrate-local-pty-registry.test.ts b/src/main/memory/hydrate-local-pty-registry.test.ts index b7dd5e1d233..ffa2f703e03 100644 --- a/src/main/memory/hydrate-local-pty-registry.test.ts +++ b/src/main/memory/hydrate-local-pty-registry.test.ts @@ -60,7 +60,8 @@ function makeStore( kind?: Repo['kind'] path?: string }[] = [], - worktreeMeta: Record = {} + worktreeMeta: Record = {}, + folderWorkspaces: FolderWorkspace[] = [] ): Store { const built: Repo[] = repos.map((r) => ({ id: r.id, @@ -72,15 +73,18 @@ function makeStore( executionHostId: r.executionHostId ?? null, kind: r.kind })) - return { + const store: Partial = { getRepos: () => built, - getFolderWorkspaces: (): FolderWorkspace[] => [], + getFolderWorkspaces: () => folderWorkspaces, + getProjectGroups: () => [], getAllWorktreeMeta: () => worktreeMeta, getAllWorktreeMetaForHost: (hostId) => Object.fromEntries( Object.entries(worktreeMeta).filter(([, meta]) => !meta.hostId || meta.hostId === hostId) ) - } as Store + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: hydration reads only the store members stubbed above. + return store as Store } function makeProvider(sessions: SessionInfo[]): Pick { diff --git a/src/main/memory/hydrate-local-pty-registry.ts b/src/main/memory/hydrate-local-pty-registry.ts index 4e5063b5fc1..6c70dc4c0e6 100644 --- a/src/main/memory/hydrate-local-pty-registry.ts +++ b/src/main/memory/hydrate-local-pty-registry.ts @@ -2,7 +2,6 @@ import { getRepoExecutionHostId, LOCAL_EXECUTION_HOST_ID } from '../../shared/ex import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../shared/abort-signal-reason' import { mapSettledWithConcurrency } from '../../shared/map-with-concurrency' import { parsePtySessionId } from '../../shared/pty-session-id-format' -import { folderWorkspaceToWorktree } from '../../shared/folder-workspace-worktree' import { isFolderRepo } from '../../shared/repo-kind' import type { Repo } from '../../shared/repo-types' import { splitWorktreeId, worktreeIdComparisonKey } from '../../shared/worktree/id' @@ -17,6 +16,7 @@ import { readAllWorktreeMetaForHost } from '../persistence/host-qualified-worktr import { getLocalProjectWorktreeGitOptions } from '../project-runtime-git-options' import { listLocalRepoWorktreesStrict } from '../repo-worktrees' import { listRegisteredPtys, registerPty } from './pty-registry' +import { getVerifiedLocalFolderWorkspaceKeys } from './verified-local-folder-workspaces' type HydrationStore = Store @@ -165,6 +165,11 @@ async function hydrateLocalPtyRegistry( } throwIfSignalAborted(signal) + const verifiedLocalFolderKeys = getVerifiedLocalFolderWorkspaceKeys({ + folderWorkspaces: store.getFolderWorkspaces(), + projectGroups: store.getProjectGroups(), + repos: store.getRepos() + }) for (const info of inventory.sessions) { throwIfSignalAborted(signal) if (alreadyRegistered.has(info.sessionId)) { @@ -186,7 +191,7 @@ async function hydrateLocalPtyRegistry( return complete function isVerifiedLocalWorktree(worktreeId: string): boolean { - if (verifiedFolderWorktreeIds.has(worktreeId)) { + if (verifiedLocalFolderKeys.has(worktreeId) || verifiedFolderWorktreeIds.has(worktreeId)) { return true } const key = worktreeIdComparisonKey(worktreeId) @@ -220,17 +225,6 @@ function getVerifiedFolderWorktreeIds( repoCatalog: LocalRepoCatalog ): Set { const verified = new Set() - const folders = store.getFolderWorkspaces() - const counts = new Map() - for (const folder of folders) { - counts.set(folder.id, (counts.get(folder.id) ?? 0) + 1) - } - for (const folder of folders) { - const worktree = folderWorkspaceToWorktree(folder) - if (counts.get(folder.id) === 1 && worktree.hostId === LOCAL_EXECUTION_HOST_ID) { - verified.add(worktree.id) - } - } const metadata = readAllWorktreeMetaForHost(store, LOCAL_EXECUTION_HOST_ID) for (const [worktreeId, meta] of Object.entries(metadata)) { const parsed = splitWorktreeId(worktreeId) diff --git a/src/main/memory/verified-local-folder-workspaces.test.ts b/src/main/memory/verified-local-folder-workspaces.test.ts new file mode 100644 index 00000000000..594e701561b --- /dev/null +++ b/src/main/memory/verified-local-folder-workspaces.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest' +import type { FolderWorkspaceHostState } from '../../shared/folder-workspace-execution-host' +import type { FolderWorkspace } from '../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../shared/project-group-types' +import type { Repo } from '../../shared/repo-types' +import { getVerifiedLocalFolderWorkspaceKeys } from './verified-local-folder-workspaces' + +const folder = (patch: Partial = {}): FolderWorkspace => ({ + id: 'folder-1', + projectGroupId: 'group-1', + name: 'Workspace', + folderPath: '/workspace', + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + createdAt: 0, + lastActivityAt: 0, + updatedAt: 0, + ...patch +}) +const repo = (patch: Partial = {}): Repo => ({ + id: 'repo-1', + path: '/workspace/repo', + displayName: 'repo', + badgeColor: '#000000', + addedAt: 0, + ...patch +}) +const projectGroup = (patch: Partial = {}): ProjectGroup => ({ + id: 'group-1', + name: 'Group', + parentPath: null, + parentGroupId: null, + createdFrom: 'manual', + tabOrder: 0, + isCollapsed: false, + color: null, + createdAt: 0, + updatedAt: 0, + ...patch +}) +const state = (patch: Partial = {}): FolderWorkspaceHostState => ({ + folderWorkspaces: [folder()], + projectGroups: [], + repos: [], + ...patch +}) + +describe('verified local folder workspace keys', () => { + it.each([undefined, null, 'local'] as const)('accepts local ownership %s', (executionHostId) => { + expect( + getVerifiedLocalFolderWorkspaceKeys( + state({ + folderWorkspaces: [folder({ executionHostId })] + }) + ) + ).toEqual(new Set(['folder:folder-1'])) + }) + + it.each(['ssh:target-1', 'runtime:environment-1', 'invalid', ''])( + 'rejects nonlocal or invalid ownership %s', + (executionHostId) => { + expect( + getVerifiedLocalFolderWorkspaceKeys( + state({ + folderWorkspaces: [ + // Why Object.assign: invalid stored ids must reach the parser without widening the type. + Object.assign(folder(), { executionHostId }) + ] + }) + ) + ).toEqual(new Set()) + } + ) + + it('honors explicit local ownership over legacy scope', () => { + expect( + getVerifiedLocalFolderWorkspaceKeys( + state({ + folderWorkspaces: [folder({ executionHostId: 'local', connectionId: 'target-1' })], + repos: [repo({ executionHostId: 'runtime:environment-1' })] + }) + ) + ).toEqual(new Set(['folder:folder-1'])) + }) + + it.each([ + state({ folderWorkspaces: [] }), + state({ folderWorkspaces: [folder(), folder()] }), + state({ + folderWorkspaces: [ + folder({ executionHostId: 'local' }), + folder({ executionHostId: 'runtime:environment-1' }) + ] + }), + state({ folderWorkspaces: [folder({ connectionId: 'target-1' })] }), + state({ projectGroups: [projectGroup({ connectionId: 'target-1' })] }), + state({ repos: [repo({ executionHostId: 'runtime:environment-1' })] }), + state({ repos: [repo({ executionHostId: 'ssh:target-1' })] }), + state({ repos: [repo(), repo({ id: 'repo-2', connectionId: 'target-1' })] }), + state({ repos: [repo(), repo({ id: 'repo-2', executionHostId: 'runtime:environment-1' })] }) + ])('rejects missing, duplicate, remote, or ambiguous scope %#', (catalog) => { + expect(getVerifiedLocalFolderWorkspaceKeys(catalog)).toEqual(new Set()) + }) + + it('accepts local inferred scope without unrelated runtime repos affecting it', () => { + expect( + getVerifiedLocalFolderWorkspaceKeys( + state({ + repos: [ + repo(), + repo({ id: 'remote', path: '/elsewhere', executionHostId: 'runtime:environment-1' }) + ] + }) + ) + ).toEqual(new Set(['folder:folder-1'])) + }) +}) diff --git a/src/main/memory/verified-local-folder-workspaces.ts b/src/main/memory/verified-local-folder-workspaces.ts new file mode 100644 index 00000000000..eae3ee39967 --- /dev/null +++ b/src/main/memory/verified-local-folder-workspaces.ts @@ -0,0 +1,40 @@ +import { getRepoExecutionHostId, parseExecutionHostId } from '../../shared/execution-host' +import { + findFolderWorkspaceCandidateRepos, + resolveFolderWorkspaceHost, + type FolderWorkspaceHostState +} from '../../shared/folder-workspace-execution-host' +import { folderWorkspaceKey } from '../../shared/workspace-scope' + +export function getVerifiedLocalFolderWorkspaceKeys(state: FolderWorkspaceHostState): Set { + const counts = new Map() + for (const workspace of state.folderWorkspaces) { + counts.set(workspace.id, (counts.get(workspace.id) ?? 0) + 1) + } + const keys = new Set() + for (const workspace of state.folderWorkspaces) { + if (counts.get(workspace.id) !== 1) { + continue + } + const pin = parseExecutionHostId(workspace.executionHostId) + if (workspace.executionHostId != null) { + if (pin?.kind === 'local') { + keys.add(folderWorkspaceKey(workspace.id)) + } + continue + } + if (resolveFolderWorkspaceHost(state, workspace.id).kind !== 'local') { + continue + } + // The shared legacy resolver projects runtime ownership as local; attribution cannot. + if ( + findFolderWorkspaceCandidateRepos(state, workspace.id).some( + (repo) => getRepoExecutionHostId(repo) !== 'local' + ) + ) { + continue + } + keys.add(folderWorkspaceKey(workspace.id)) + } + return keys +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-corruption.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-corruption.test.ts index bd7d735abbc..9a8b5b62a8d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-corruption.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-corruption.test.ts @@ -51,6 +51,8 @@ const stop = (turnEnvelope = envelope('agentSession.cancel', { turnId: 'turn-1' // T-corrupt-midsession. it('refuses a send as corrupt when SQLite reports damage, and still stops the agent', async () => { await attach() + // The attach's restart-offer withdrawal holds a lock file until it ends; snapshot after it. + await hostTestRecoveryCapsuleSettled() const files = await readdir(root, { recursive: true }) const damaged = sqliteError('database disk image is malformed', 11) vi.spyOn(openTestJournalHostDatabase(root), 'transaction').mockImplementation(() => { @@ -78,7 +80,6 @@ it('refuses a send as corrupt when SQLite reports damage, and still stops the ag sessionId: expect.any(String), error: expect.objectContaining({ message: 'database disk image is malformed' }) }) - // The restart-offer withdrawal the attach started holds its lock until it ends. await hostTestRecoveryCapsuleSettled() expect(await readdir(root, { recursive: true })).toEqual(files) }) diff --git a/src/main/network/transient-download-error.ts b/src/main/network/transient-download-error.ts new file mode 100644 index 00000000000..1e95d7b120e --- /dev/null +++ b/src/main/network/transient-download-error.ts @@ -0,0 +1,27 @@ +/** Download failures a later attempt may not hit: dropped connections, timeouts, 5xx. */ +export type HttpStatusError = Error & { + httpStatusCode?: number + retryAfterMs?: number + retryable?: boolean +} + +const RETRYABLE_NET_ERROR = + /net::ERR_(CONTENT_LENGTH_MISMATCH|INCOMPLETE_CHUNKED_ENCODING|CONNECTION_(RESET|CLOSED|ABORTED|REFUSED|TIMED_OUT)|EMPTY_RESPONSE|NETWORK_CHANGED|TIMED_OUT|INTERNET_DISCONNECTED|ADDRESS_UNREACHABLE|NAME_NOT_RESOLVED|SOCKET_NOT_CONNECTED|HTTP2_PROTOCOL_ERROR|QUIC_PROTOCOL_ERROR)\b/ +const RETRYABLE_HTTP_STATUSES = new Set([408, 416, 425, 429, 500, 502, 503, 504]) + +export function isRetryableDownloadError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false + } + const downloadError: HttpStatusError = error + if (downloadError.retryable === true) { + return true + } + const statusCode = downloadError.httpStatusCode + if (statusCode !== undefined) { + return RETRYABLE_HTTP_STATUSES.has(statusCode) + } + return ( + RETRYABLE_NET_ERROR.test(error.message) || error.message.includes('without network activity') + ) +} diff --git a/src/main/orca-profiles/profile-project-session-field-disposition.test.ts b/src/main/orca-profiles/profile-project-session-field-disposition.test.ts index daedc20a111..ce509767119 100644 --- a/src/main/orca-profiles/profile-project-session-field-disposition.test.ts +++ b/src/main/orca-profiles/profile-project-session-field-disposition.test.ts @@ -66,6 +66,120 @@ describe('client-hosted rows in the repo-removal and transfer paths', () => { expect(result.clientHostedBrowserPagesByWorktree).toBeUndefined() }) + it('rekeys markdown frontmatter visibility with the open file identity', () => { + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: { + [REMOVED_WORKTREE_ID]: [ + { + filePath: '/tmp/worktree-a/README.md', + relativePath: 'README.md', + worktreeId: REMOVED_WORKTREE_ID, + language: 'markdown' + } + ] + }, + markdownFrontmatterVisible: { + '/tmp/worktree-a/README.md': false + } + } + + const result = extractSessionForTransfer(session, REMOVED_REPO_ID, TRANSFER_TARGET_REPO_ID) + + expect(result.markdownFrontmatterVisible).toEqual({ '/tmp/worktree-a/README.md': false }) + }) + + it('omits stale markdown visibility entries that no longer name an open file', () => { + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: { + [REMOVED_WORKTREE_ID]: [ + { + filePath: '/tmp/worktree-a/README.md', + relativePath: 'README.md', + worktreeId: REMOVED_WORKTREE_ID, + language: 'markdown' + } + ] + }, + markdownFrontmatterVisible: { + '/tmp/worktree-a/README.md': false, + '/tmp/worktree-a/closed.md': false + } + } + + const result = extractSessionForTransfer(session, REMOVED_REPO_ID, TRANSFER_TARGET_REPO_ID) + + expect(result.markdownFrontmatterVisible).toEqual({ '/tmp/worktree-a/README.md': false }) + }) + + it('prunes markdown visibility for files in a removed repo', () => { + const session = { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: { + [REMOVED_WORKTREE_ID]: [ + { + filePath: '/tmp/worktree-a/README.md', + relativePath: 'README.md', + worktreeId: REMOVED_WORKTREE_ID, + language: 'markdown' + } + ], + [RETAINED_WORKTREE_ID]: [ + { + filePath: '/tmp/worktree-b/README.md', + relativePath: 'README.md', + worktreeId: RETAINED_WORKTREE_ID, + language: 'markdown' + } + ] + }, + markdownFrontmatterVisible: { + '/tmp/worktree-a/README.md': false, + '/tmp/worktree-b/README.md': false + } + } + + const result = removeRepoFromWorkspaceSession(session, REMOVED_REPO_ID) + + expect(result.markdownFrontmatterVisible).toEqual({ '/tmp/worktree-b/README.md': false }) + }) + + it('keeps a unified-only terminal layout attached during transfer', () => { + const session = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: {}, + unifiedTabs: { + [REMOVED_WORKTREE_ID]: [ + { + id: 'terminal-tab-1', + entityId: 'terminal-tab-1', + groupId: 'group-1', + worktreeId: REMOVED_WORKTREE_ID, + contentType: 'terminal' as const, + label: 'Terminal', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'terminal-tab-1': { + root: { type: 'leaf' as const, leafId: 'leaf-1' }, + activeLeafId: 'leaf-1', + expandedLeafId: null, + titlesByLeafId: { 'leaf-1': 'shell' } + } + } + } + + const result = extractSessionForTransfer(session, REMOVED_REPO_ID, TRANSFER_TARGET_REPO_ID) + + expect(result.terminalLayoutsByTabId).toEqual(session.terminalLayoutsByTabId) + }) + it('removes the transferred repo rows from the source it left', () => { const source = removeRepoFromWorkspaceSession( sessionWithClientHostedRows(), diff --git a/src/main/orca-profiles/profile-project-session-field-disposition.ts b/src/main/orca-profiles/profile-project-session-field-disposition.ts index 02a7ce77ca6..219ef63d8ca 100644 --- a/src/main/orca-profiles/profile-project-session-field-disposition.ts +++ b/src/main/orca-profiles/profile-project-session-field-disposition.ts @@ -66,7 +66,10 @@ export const WORKSPACE_SESSION_FIELD_DISPOSITION = { }, openFilesByWorktree: { onRepoRemoval: 'prunedByOwnerKey', onTransfer: 'copiedByBespokeRule' }, activeFileIdByWorktree: { onRepoRemoval: 'prunedByOwnerKey', onTransfer: 'copiedByOwnerKey' }, - markdownFrontmatterVisible: { onRepoRemoval: 'notRepoScoped', onTransfer: 'notTransferred' }, + markdownFrontmatterVisible: { + onRepoRemoval: 'prunedByBespokeRule', + onTransfer: 'copiedByBespokeRule' + }, browserTabsByWorktree: { onRepoRemoval: 'prunedByBespokeRule', onTransfer: 'copiedByBespokeRule' diff --git a/src/main/orca-profiles/profile-project-session-state.ts b/src/main/orca-profiles/profile-project-session-state.ts index 0c1ea243b3e..b1ccc6c488f 100644 --- a/src/main/orca-profiles/profile-project-session-state.ts +++ b/src/main/orca-profiles/profile-project-session-state.ts @@ -3,6 +3,7 @@ import type { ExecutionHostId } from '../../shared/execution-host' import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' import { parseWorkspaceKey } from '../../shared/workspace-scope' import { SESSION_FIELDS_PRUNED_BY_OWNER_KEY } from './profile-project-session-field-disposition' +import { markdownFileIdCandidates } from './profile-session-markdown-transfer' import { isRepoWorktreeId, ownerKeyBelongsToRepo, @@ -44,6 +45,10 @@ export function mergeWorkspaceSessions( const base = existing ?? getDefaultWorkspaceSession() return { ...base, + // Why: the focus scalars below fall back to incoming together, so a borrowed worktree keeps its repo and host. + activeRepoId: base.activeRepoId ?? incoming.activeRepoId, + activeWorkspaceExecutionHostId: + base.activeWorkspaceExecutionHostId ?? incoming.activeWorkspaceExecutionHostId, tabsByWorktree: { ...base.tabsByWorktree, ...incoming.tabsByWorktree }, terminalLayoutsByTabId: { ...base.terminalLayoutsByTabId, @@ -58,6 +63,10 @@ export function mergeWorkspaceSessions( } : {}), openFilesByWorktree: { ...base.openFilesByWorktree, ...incoming.openFilesByWorktree }, + markdownFrontmatterVisible: { + ...base.markdownFrontmatterVisible, + ...incoming.markdownFrontmatterVisible + }, browserTabsByWorktree: { ...base.browserTabsByWorktree, ...incoming.browserTabsByWorktree @@ -106,6 +115,14 @@ export function mergeWorkspaceSessions( ...base.terminalSurfaceTombstonesByPaneKey, ...incoming.terminalSurfaceTombstonesByPaneKey }, + ...(base.sleepingAgentSessionsByPaneKey || incoming.sleepingAgentSessionsByPaneKey + ? { + sleepingAgentSessionsByPaneKey: { + ...base.sleepingAgentSessionsByPaneKey, + ...incoming.sleepingAgentSessionsByPaneKey + } + } + : {}), activeWorktreeIdsOnShutdown: [ ...(base.activeWorktreeIdsOnShutdown ?? []), ...(incoming.activeWorktreeIdsOnShutdown ?? []) @@ -133,6 +150,17 @@ export function removeRepoFromWorkspaceSession( ): WorkspaceSessionState { const next = structuredClone(session ?? getDefaultWorkspaceSession()) const removedTerminalTabIds = new Set() + const removedMarkdownFileIds = new Set() + for (const [ownerKey, files] of Object.entries(next.openFilesByWorktree ?? {})) { + if (!ownerKeyBelongsToRepo(ownerKey, repoId)) { + continue + } + for (const file of files) { + markdownFileIdCandidates(file.filePath, ownerKey, file.runtimeEnvironmentId).forEach((id) => + removedMarkdownFileIds.add(id) + ) + } + } for (const [ownerKey, tabs] of Object.entries(next.tabsByWorktree)) { if (!ownerKeyBelongsToRepo(ownerKey, repoId)) { continue @@ -162,6 +190,13 @@ export function removeRepoFromWorkspaceSession( const record = next[field] as Record | undefined ;(next as Record)[field] = removeRepoWorktreeRecord(record, repoId) } + if (next.markdownFrontmatterVisible) { + next.markdownFrontmatterVisible = Object.fromEntries( + Object.entries(next.markdownFrontmatterVisible).filter( + ([fileId]) => !removedMarkdownFileIds.has(fileId) + ) + ) + } if (next.terminalSurfaceTombstonesByPaneKey) { next.terminalSurfaceTombstonesByPaneKey = Object.fromEntries( Object.entries(next.terminalSurfaceTombstonesByPaneKey).filter( diff --git a/src/main/orca-profiles/profile-project-session-transfer.ts b/src/main/orca-profiles/profile-project-session-transfer.ts index 3018f07173c..39c3b3b6189 100644 --- a/src/main/orca-profiles/profile-project-session-transfer.ts +++ b/src/main/orca-profiles/profile-project-session-transfer.ts @@ -1,24 +1,10 @@ -import { getDefaultWorkspaceSession } from '../../shared/constants' import type { ExecutionHostId } from '../../shared/execution-host' -import type { - BrowserPage, - BrowserPageDocLocation, - BrowserWorkspace -} from '../../shared/browser-workspace-types' -import { remapBrowserPageDocLocation } from '../../shared/browser-page-doc-location' -import type { Tab, TabGroup } from '../../shared/tab-types' -import type { TerminalTab } from '../../shared/terminal-tab-types' -import type { - PersistedOpenFile, - WorkspaceSessionState -} from '../../shared/workspace-session-state-types' -import { parseWorkspaceKey, worktreeWorkspaceKey } from '../../shared/workspace-scope' -import { SESSION_FIELDS_COPIED_BY_OWNER_KEY } from './profile-project-session-field-disposition' +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import { rekeyOwnerKey, rekeyWorktreeId } from './profile-project-worktree-identity' import { - isRepoWorktreeId, - rekeyOwnerKey, - rekeyWorktreeId -} from './profile-project-worktree-identity' + extractSessionOwnersForTransfer, + hasTransferredSessionState +} from './profile-session-owner-transfer' export function extractHostSessionsForTransfer( sessions: Partial> | undefined, @@ -38,210 +24,13 @@ export function extractHostSessionsForTransfer( return next } -function hasTransferredSessionState(session: WorkspaceSessionState): boolean { - return ( - Object.keys(session.tabsByWorktree).length > 0 || - Object.keys(session.openFilesByWorktree ?? {}).length > 0 || - Object.keys(session.browserTabsByWorktree ?? {}).length > 0 || - Object.keys(session.unifiedTabs ?? {}).length > 0 || - Object.keys(session.tabGroups ?? {}).length > 0 || - Object.keys(session.lastVisitedAtByWorktreeId ?? {}).length > 0 || - Object.keys(session.terminalTopologyRevisionByRepoId ?? {}).length > 0 - ) -} - export function extractSessionForTransfer( session: WorkspaceSessionState | undefined, oldRepoId: string, newRepoId: string ): WorkspaceSessionState { - const source = session ?? getDefaultWorkspaceSession() - const transferred = getDefaultWorkspaceSession() - const copiedTerminalTabIds = new Set() - const copiedBrowserWorkspaceIds = new Set() - const mapOwnerRecord = ( - record: Record | undefined, - mapValue: (value: T) => T - ): Record => { - const next: Record = {} - for (const [ownerKey, value] of Object.entries(record ?? {})) { - const nextOwnerKey = rekeyOwnerKey(oldRepoId, newRepoId, ownerKey) - if (nextOwnerKey) { - next[nextOwnerKey] = mapValue(value) - } - } - return next - } - transferred.tabsByWorktree = mapOwnerRecord(source.tabsByWorktree, (tabs) => - tabs.map((tab) => { - copiedTerminalTabIds.add(tab.id) - return rekeyTerminalTab(tab, oldRepoId, newRepoId) - }) - ) - transferred.openFilesByWorktree = mapOwnerRecord(source.openFilesByWorktree, (files) => - files.map((file) => rekeyOpenFile(file, oldRepoId, newRepoId)) - ) - transferred.browserTabsByWorktree = mapOwnerRecord(source.browserTabsByWorktree, (tabs) => - tabs.map((tab) => { - copiedBrowserWorkspaceIds.add(tab.id) - return rekeyBrowserWorkspace(tab, oldRepoId, newRepoId) - }) - ) - transferred.browserPagesByWorkspace = copyBrowserPages( - source.browserPagesByWorkspace, - copiedBrowserWorkspaceIds, - oldRepoId, - newRepoId - ) - // Driven by the census so a field cannot be added to the session type and forgotten here. The - // census is also where a field's deliberate non-transfer is recorded -- notably the runtime's - // client-hosted rows, which name a paired device this payload does not carry. - for (const field of SESSION_FIELDS_COPIED_BY_OWNER_KEY) { - const record = source[field] as Record | undefined - ;(transferred as Record)[field] = mapOwnerRecord(record, (value) => - structuredClone(value) - ) - } - transferred.unifiedTabs = mapOwnerRecord(source.unifiedTabs, (tabs) => - tabs.map((tab) => rekeyUnifiedTab(tab, oldRepoId, newRepoId)) - ) - transferred.tabGroups = mapOwnerRecord(source.tabGroups, (groups) => - groups.map((group) => rekeyTabGroup(group, oldRepoId, newRepoId)) - ) - transferred.terminalLayoutsByTabId = {} - for (const tabId of copiedTerminalTabIds) { - const layout = source.terminalLayoutsByTabId[tabId] - if (layout) { - transferred.terminalLayoutsByTabId[tabId] = structuredClone(layout) - } - } - if (source.localOnlyScrollbackByTabId) { - transferred.localOnlyScrollbackByTabId = Object.fromEntries( - Object.entries(source.localOnlyScrollbackByTabId) - .filter(([tabId]) => copiedTerminalTabIds.has(tabId)) - .map(([tabId, buffers]) => [tabId, structuredClone(buffers)]) - ) - } - transferred.terminalPtyIncarnationsByPaneKey = Object.fromEntries( - Object.entries(source.terminalPtyIncarnationsByPaneKey ?? {}).filter(([paneKey]) => { - const separator = paneKey.lastIndexOf(':') - return separator > 0 && copiedTerminalTabIds.has(paneKey.slice(0, separator)) - }) - ) - transferred.terminalSurfaceTombstonesByPaneKey = Object.fromEntries( - Object.entries(source.terminalSurfaceTombstonesByPaneKey ?? {}).flatMap( - ([paneKey, tombstone]) => - isRepoWorktreeId(oldRepoId, tombstone.worktreeId) - ? [ - [ - paneKey, - { - ...structuredClone(tombstone), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, tombstone.worktreeId) - } - ] as const - ] - : [] - ) - ) - transferred.activeWorktreeIdsOnShutdown = source.activeWorktreeIdsOnShutdown - ?.filter((worktreeId) => isRepoWorktreeId(oldRepoId, worktreeId)) - .map((worktreeId) => rekeyWorktreeId(oldRepoId, newRepoId, worktreeId)) - if (source.activeWorktreeId && isRepoWorktreeId(oldRepoId, source.activeWorktreeId)) { - transferred.activeWorktreeId = rekeyWorktreeId(oldRepoId, newRepoId, source.activeWorktreeId) - } - const activeScope = source.activeWorkspaceKey - ? parseWorkspaceKey(source.activeWorkspaceKey) - : null - if (activeScope?.type === 'worktree' && isRepoWorktreeId(oldRepoId, activeScope.worktreeId)) { - transferred.activeWorkspaceKey = worktreeWorkspaceKey( - rekeyWorktreeId(oldRepoId, newRepoId, activeScope.worktreeId) - ) - } - return transferred -} - -function rekeyTerminalTab(tab: TerminalTab, oldRepoId: string, newRepoId: string): TerminalTab { - return { - ...structuredClone(tab), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, tab.worktreeId) - } -} - -function rekeyOpenFile( - file: PersistedOpenFile, - oldRepoId: string, - newRepoId: string -): PersistedOpenFile { - return { - ...structuredClone(file), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, file.worktreeId) - } -} - -function rekeyBrowserWorkspace( - workspace: BrowserWorkspace, - oldRepoId: string, - newRepoId: string -): BrowserWorkspace { - return { - ...structuredClone(workspace), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, workspace.worktreeId), - ...(workspace.docLocation - ? { docLocation: rekeyBrowserDocLocation(workspace.docLocation, oldRepoId, newRepoId) } - : {}), - // Why: both the session profile and the resolved partition string are - // source-profile-scoped; carrying either across would point the restored - // pane at a partition the target profile's allowlist rejects. - sessionProfileId: null, - sessionPartition: null - } -} - -function rekeyBrowserPage(page: BrowserPage, oldRepoId: string, newRepoId: string): BrowserPage { - return { - ...structuredClone(page), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, page.worktreeId), - ...(page.docLocation - ? { docLocation: rekeyBrowserDocLocation(page.docLocation, oldRepoId, newRepoId) } - : {}) - } -} - -function rekeyBrowserDocLocation( - location: BrowserPageDocLocation, - oldRepoId: string, - newRepoId: string -): BrowserPageDocLocation { - const nextWorktreeId = rekeyWorktreeId(oldRepoId, newRepoId, location.worktreeId) - return remapBrowserPageDocLocation(location, location.worktreeId, nextWorktreeId) -} - -function copyBrowserPages( - pagesByWorkspace: Record | undefined, - workspaceIds: ReadonlySet, - oldRepoId: string, - newRepoId: string -): Record { - const next: Record = {} - for (const [workspaceId, pages] of Object.entries(pagesByWorkspace ?? {})) { - if (workspaceIds.has(workspaceId)) { - next[workspaceId] = pages.map((page) => rekeyBrowserPage(page, oldRepoId, newRepoId)) - } - } - return next -} - -function rekeyUnifiedTab(tab: Tab, oldRepoId: string, newRepoId: string): Tab { - return { - ...structuredClone(tab), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, tab.worktreeId) - } -} - -function rekeyTabGroup(group: TabGroup, oldRepoId: string, newRepoId: string): TabGroup { - return { - ...structuredClone(group), - worktreeId: rekeyWorktreeId(oldRepoId, newRepoId, group.worktreeId) - } + return extractSessionOwnersForTransfer(session, { + mapOwnerKey: (ownerKey) => rekeyOwnerKey(oldRepoId, newRepoId, ownerKey), + mapWorktreeId: (worktreeId) => rekeyWorktreeId(oldRepoId, newRepoId, worktreeId) + }) } diff --git a/src/main/orca-profiles/profile-session-markdown-transfer.ts b/src/main/orca-profiles/profile-session-markdown-transfer.ts new file mode 100644 index 00000000000..98d877fcbe4 --- /dev/null +++ b/src/main/orca-profiles/profile-session-markdown-transfer.ts @@ -0,0 +1,80 @@ +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import { ownedEditorFileId } from '../../shared/workspace-session-pane-ownership' + +type OwnerProjection = { + mapOwnerKey: (ownerKey: string) => string | null + mapWorktreeId: (worktreeId: string) => string +} + +/** Map file-keyed markdown visibility along with the persisted open-file identity it belongs to. */ +export function mapMarkdownFrontmatterVisible( + visibleByFileId: Record | undefined, + filesByOwner: WorkspaceSessionState['openFilesByWorktree'] | undefined, + projection: OwnerProjection +): Record { + const next: Record = {} + const sourceToDestination = buildMarkdownFrontmatterIdMap(filesByOwner, projection) + for (const [sourceFileId, visible] of Object.entries(visibleByFileId ?? {})) { + if (visible) { + // Visible is the hydration default; preserving only hidden overrides avoids carrying stale + // positive entries while retaining the exact rendered result. + continue + } + const destinationId = sourceToDestination.get(sourceFileId) + if (destinationId) { + next[destinationId] = false + } + } + return next +} + +/** + * Builds the file-id mapping used by hydration, marking a source id null when two transferred + * files claim it. An ambiguous visibility override is never guessed at by the migration. + */ +export function buildMarkdownFrontmatterIdMap( + filesByOwner: WorkspaceSessionState['openFilesByWorktree'] | undefined, + projection: OwnerProjection +): ReadonlyMap { + const result = new Map() + for (const [sourceOwnerKey, files] of Object.entries(filesByOwner ?? {})) { + if (!projection.mapOwnerKey(sourceOwnerKey)) { + continue + } + const destinationWorktreeId = projection.mapWorktreeId(sourceOwnerKey) + const sourceWorktreeIds = [sourceOwnerKey] + const separator = sourceOwnerKey.indexOf('|') + if (separator > 0) { + sourceWorktreeIds.push(sourceOwnerKey.slice(separator + 1)) + } + for (const file of files) { + const destinationId = ownedEditorFileId( + file.filePath, + destinationWorktreeId, + file.runtimeEnvironmentId + ) + for (const sourceWorktreeId of sourceWorktreeIds) { + const candidates = markdownFileIdCandidates( + file.filePath, + sourceWorktreeId, + file.runtimeEnvironmentId + ) + for (const sourceId of candidates) { + const mapped = sourceId === file.filePath ? file.filePath : destinationId + const existing = result.get(sourceId) + result.set(sourceId, existing === undefined || existing === mapped ? mapped : null) + } + } + } + } + return result +} + +/** Returns the IDs hydration can assign to a persisted file before any runtime data is loaded. */ +export function markdownFileIdCandidates( + filePath: string, + worktreeId: string, + runtimeEnvironmentId: string | null | undefined +): ReadonlySet { + return new Set([filePath, ownedEditorFileId(filePath, worktreeId, runtimeEnvironmentId)]) +} diff --git a/src/main/orca-profiles/profile-session-owner-transfer.test.ts b/src/main/orca-profiles/profile-session-owner-transfer.test.ts new file mode 100644 index 00000000000..ebcbb472032 --- /dev/null +++ b/src/main/orca-profiles/profile-session-owner-transfer.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../shared/constants' +import type { PersistedOpenFile } from '../../shared/workspace-session-state-types' +import { mergeWorkspaceSessions } from './profile-project-session-state' +import { extractSessionForTransfer } from './profile-project-session-transfer' +import { buildMarkdownFrontmatterIdMap } from './profile-session-markdown-transfer' +import { extractSessionOwnersForTransfer } from './profile-session-owner-transfer' + +const SOURCE = 'repo-1::/tmp/a' +const DESTINATION = 'repo-9::/tmp/a' +const editorId = (worktreeId: string, filePath: string, runtime = 'local') => + `editor:${encodeURIComponent(worktreeId)}:${runtime}:${encodeURIComponent(filePath)}` + +function file(filePath: string, worktreeId = SOURCE): PersistedOpenFile { + return { filePath, relativePath: filePath.split('/').at(-1)!, worktreeId, language: 'markdown' } +} + +describe('session owner transfer', () => { + it('moves a hidden override keyed by the owned editor id to the destination worktree', () => { + const session = { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: { [SOURCE]: [file('/tmp/a/README.md')] }, + markdownFrontmatterVisible: { + [editorId(SOURCE, '/tmp/a/README.md')]: false, + // Visible is the hydration default, so it is not carried. + '/tmp/a/README.md': true + } + } + + const result = extractSessionForTransfer(session, 'repo-1', 'repo-9') + + expect(result.markdownFrontmatterVisible).toEqual({ + [editorId(DESTINATION, '/tmp/a/README.md')]: false + }) + }) + + it('refuses to guess an override two transferred files both claim', () => { + // Two host-qualified owners of one worktree each claim its unqualified editor id. + const owners = { [`h1|${SOURCE}`]: 'repo-9::/tmp/h1', [`h2|${SOURCE}`]: 'repo-9::/tmp/h2' } + const projection = { + mapOwnerKey: (key: string) => owners[key] ?? null, + mapWorktreeId: (key: string) => owners[key] ?? key + } + const files = { + [`h1|${SOURCE}`]: [file('/tmp/a/x.md')], + [`h2|${SOURCE}`]: [file('/tmp/a/x.md')] + } + + expect( + buildMarkdownFrontmatterIdMap(files, projection).get(editorId(SOURCE, '/tmp/a/x.md')) + ).toBe(null) + const visibility = extractSessionOwnersForTransfer( + { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: files, + markdownFrontmatterVisible: { [editorId(SOURCE, '/tmp/a/x.md')]: false } + }, + projection + ).markdownFrontmatterVisible + + expect(visibility).toEqual({}) + }) + + it('carries a worktree focus key and lets the destination merge keep its own overrides', () => { + const transferred = extractSessionForTransfer( + { + ...getDefaultWorkspaceSession(), + activeWorkspaceKey: `worktree:${SOURCE}`, + openFilesByWorktree: { [SOURCE]: [file('/tmp/a/README.md')] }, + markdownFrontmatterVisible: { '/tmp/a/README.md': false } + }, + 'repo-1', + 'repo-9' + ) + expect(transferred.activeWorkspaceKey).toBe(`worktree:${DESTINATION}`) + + const merged = mergeWorkspaceSessions( + { ...getDefaultWorkspaceSession(), markdownFrontmatterVisible: { '/other.md': false } }, + transferred + ) + + expect(merged.markdownFrontmatterVisible).toEqual({ + '/other.md': false, + '/tmp/a/README.md': false + }) + }) +}) diff --git a/src/main/orca-profiles/profile-session-owner-transfer.ts b/src/main/orca-profiles/profile-session-owner-transfer.ts new file mode 100644 index 00000000000..2e628e936ae --- /dev/null +++ b/src/main/orca-profiles/profile-session-owner-transfer.ts @@ -0,0 +1,270 @@ +import { getDefaultWorkspaceSession } from '../../shared/constants' +import type { SleepingAgentSessionRecord } from '../../shared/agent-session-resume' +import type { BrowserPage, BrowserWorkspace } from '../../shared/browser-workspace-types' +import { remapBrowserPageDocLocation } from '../../shared/browser-page-doc-location' +import type { Tab, TabGroup } from '../../shared/tab-types' +import type { TerminalTab } from '../../shared/terminal-tab-types' +import type { + PersistedOpenFile, + WorkspaceSessionState +} from '../../shared/workspace-session-state-types' +import { isWorkspaceKey } from '../../shared/workspace-scope' +import { SESSION_FIELDS_COPIED_BY_OWNER_KEY } from './profile-project-session-field-disposition' +import { mapMarkdownFrontmatterVisible } from './profile-session-markdown-transfer' + +export { + buildMarkdownFrontmatterIdMap, + markdownFileIdCandidates +} from './profile-session-markdown-transfer' +import { paneBelongsToTabs } from '../../shared/workspace-session-pane-ownership' + +export type SessionOwnerProjection = { + mapOwnerKey: (ownerKey: string) => string | null + mapWorktreeId: (worktreeId: string) => string + /** Keeps a sleeping agent's resume record when it can still resume after the transfer. */ + projectSleepingAgentSession?: ( + record: SleepingAgentSessionRecord + ) => SleepingAgentSessionRecord | null + /** Projects source-partition focus scalars when the selected entities are dormant. */ + projectSessionFocus?: (args: { + source: WorkspaceSessionState + transferred: WorkspaceSessionState + terminalTabIds: ReadonlySet + }) => void +} + +export function extractSessionOwnersForTransfer( + session: WorkspaceSessionState | undefined, + projection: SessionOwnerProjection +): WorkspaceSessionState { + const source = session ?? getDefaultWorkspaceSession() + const transferred = getDefaultWorkspaceSession() + const terminalTabIds = new Set() + const browserWorkspaceIds = new Set() + const mapOwnerRecord = ( + record: Record | undefined, + mapValue: (value: T) => T + ): Record => { + const next: Record = {} + for (const [ownerKey, value] of Object.entries(record ?? {})) { + const nextOwnerKey = projection.mapOwnerKey(ownerKey) + if (nextOwnerKey) { + next[nextOwnerKey] = mapValue(value) + } + } + return next + } + transferred.tabsByWorktree = mapOwnerRecord(source.tabsByWorktree, (tabs) => + tabs.map((tab) => { + terminalTabIds.add(tab.id) + return mapTerminalTab(tab, projection) + }) + ) + // Newer sessions may persist a terminal only in the unified tab model while the legacy + // terminal map is absent. Keep its layout and pane metadata attached to the transferred tab. + for (const [ownerKey, tabs] of Object.entries(source.unifiedTabs ?? {})) { + if (!projection.mapOwnerKey(ownerKey)) { + continue + } + for (const tab of tabs) { + if (tab.contentType === 'terminal') { + terminalTabIds.add(tab.id) + terminalTabIds.add(tab.entityId) + } + } + } + transferred.openFilesByWorktree = mapOwnerRecord(source.openFilesByWorktree, (files) => + files.map((file) => mapOpenFile(file, projection)) + ) + transferred.markdownFrontmatterVisible = mapMarkdownFrontmatterVisible( + source.markdownFrontmatterVisible, + source.openFilesByWorktree, + projection + ) + transferred.browserTabsByWorktree = mapOwnerRecord(source.browserTabsByWorktree, (tabs) => + tabs.map((tab) => { + browserWorkspaceIds.add(tab.id) + return mapBrowserWorkspace(tab, projection) + }) + ) + transferred.browserPagesByWorkspace = copyBrowserPages( + source.browserPagesByWorkspace, + browserWorkspaceIds, + projection + ) + for (const field of SESSION_FIELDS_COPIED_BY_OWNER_KEY) { + const record = source[field] as Record | undefined + ;(transferred as Record)[field] = mapOwnerRecord(record, (value) => + structuredClone(value) + ) + } + transferred.unifiedTabs = mapOwnerRecord(source.unifiedTabs, (tabs) => + tabs.map((tab) => mapUnifiedTab(tab, projection)) + ) + transferred.tabGroups = mapOwnerRecord(source.tabGroups, (groups) => + groups.map((group) => mapTabGroup(group, projection)) + ) + transferred.terminalLayoutsByTabId = Object.fromEntries( + [...terminalTabIds].flatMap((tabId) => { + const layout = source.terminalLayoutsByTabId?.[tabId] + return layout ? [[tabId, structuredClone(layout)] as const] : [] + }) + ) + if (source.localOnlyScrollbackByTabId) { + transferred.localOnlyScrollbackByTabId = Object.fromEntries( + Object.entries(source.localOnlyScrollbackByTabId) + .filter(([tabId]) => terminalTabIds.has(tabId)) + .map(([tabId, buffers]) => [tabId, structuredClone(buffers)]) + ) + } + transferred.terminalPtyIncarnationsByPaneKey = Object.fromEntries( + Object.entries(source.terminalPtyIncarnationsByPaneKey ?? {}).filter(([paneKey]) => + paneBelongsToTabs(paneKey, terminalTabIds) + ) + ) + transferred.terminalSurfaceTombstonesByPaneKey = Object.fromEntries( + Object.entries(source.terminalSurfaceTombstonesByPaneKey ?? {}).flatMap( + ([paneKey, tombstone]) => + projection.mapOwnerKey(tombstone.worktreeId) + ? [ + [ + paneKey, + { + ...structuredClone(tombstone), + worktreeId: projection.mapWorktreeId(tombstone.worktreeId) + } + ] as const + ] + : [] + ) + ) + projection.projectSessionFocus?.({ source, transferred, terminalTabIds }) + if (projection.projectSleepingAgentSession) { + const sleepingAgentSessionsByPaneKey = Object.fromEntries( + Object.entries(source.sleepingAgentSessionsByPaneKey ?? {}).flatMap(([paneKey, record]) => { + const projected = projection.projectSleepingAgentSession?.(record) + return projected ? [[paneKey, projected] as const] : [] + }) + ) + if (Object.keys(sleepingAgentSessionsByPaneKey).length > 0) { + transferred.sleepingAgentSessionsByPaneKey = sleepingAgentSessionsByPaneKey + } + } + transferred.activeWorktreeIdsOnShutdown = source.activeWorktreeIdsOnShutdown + ?.filter((worktreeId) => projection.mapOwnerKey(worktreeId) !== null) + .map(projection.mapWorktreeId) + const activeWorktreeId = source.activeWorktreeId + ? projection.mapOwnerKey(source.activeWorktreeId) + : null + if (activeWorktreeId) { + transferred.activeWorktreeId = projection.mapWorktreeId(source.activeWorktreeId!) + } + const activeWorkspaceKey = source.activeWorkspaceKey + ? projection.mapOwnerKey(source.activeWorkspaceKey) + : null + if (activeWorkspaceKey && isWorkspaceKey(activeWorkspaceKey)) { + transferred.activeWorkspaceKey = activeWorkspaceKey + } + return transferred +} + +export function hasTransferredSessionState(session: WorkspaceSessionState): boolean { + return ( + Object.keys(session.tabsByWorktree ?? {}).length > 0 || + Object.keys(session.openFilesByWorktree ?? {}).length > 0 || + Object.keys(session.markdownFrontmatterVisible ?? {}).length > 0 || + Object.keys(session.browserTabsByWorktree ?? {}).length > 0 || + Object.keys(session.browserPagesByWorkspace ?? {}).length > 0 || + Object.keys(session.unifiedTabs ?? {}).length > 0 || + Object.keys(session.tabGroups ?? {}).length > 0 || + Object.keys(session.terminalLayoutsByTabId ?? {}).length > 0 || + SESSION_FIELDS_COPIED_BY_OWNER_KEY.some( + (field) => Object.keys(session[field] ?? {}).length > 0 + ) || + Object.keys(session.terminalSurfaceTombstonesByPaneKey ?? {}).length > 0 || + Object.keys(session.terminalPtyIncarnationsByPaneKey ?? {}).length > 0 || + Boolean(session.activeWorktreeId || session.activeWorkspaceKey) || + (session.activeWorktreeIdsOnShutdown?.length ?? 0) > 0 + ) +} + +function mapTerminalTab(tab: TerminalTab, projection: SessionOwnerProjection): TerminalTab { + const { pendingActivationSpawn: _pendingActivationSpawn, ...persisted } = tab + return { + ...structuredClone(persisted), + worktreeId: projection.mapWorktreeId(tab.worktreeId) + } +} + +function mapOpenFile( + file: PersistedOpenFile, + projection: SessionOwnerProjection +): PersistedOpenFile { + return { + ...structuredClone(file), + worktreeId: projection.mapWorktreeId(file.worktreeId) + } +} + +function mapBrowserWorkspace( + workspace: BrowserWorkspace, + projection: SessionOwnerProjection +): BrowserWorkspace { + return { + ...structuredClone(workspace), + worktreeId: projection.mapWorktreeId(workspace.worktreeId), + ...(workspace.docLocation + ? { + docLocation: remapBrowserPageDocLocation( + workspace.docLocation, + workspace.docLocation.worktreeId, + projection.mapWorktreeId(workspace.docLocation.worktreeId) + ) + } + : {}), + // Why: the browser session profile and partition are source-profile-scoped. + sessionProfileId: null, + sessionPartition: null + } +} + +function mapBrowserPage(page: BrowserPage, projection: SessionOwnerProjection): BrowserPage { + return { + ...structuredClone(page), + worktreeId: projection.mapWorktreeId(page.worktreeId), + ...(page.docLocation + ? { + docLocation: remapBrowserPageDocLocation( + page.docLocation, + page.docLocation.worktreeId, + projection.mapWorktreeId(page.docLocation.worktreeId) + ) + } + : {}) + } +} + +function copyBrowserPages( + pagesByWorkspace: Record | undefined, + workspaceIds: ReadonlySet, + projection: SessionOwnerProjection +): Record { + const next: Record = {} + for (const [workspaceId, pages] of Object.entries(pagesByWorkspace ?? {})) { + if (workspaceIds.has(workspaceId)) { + next[workspaceId] = pages.map((page) => mapBrowserPage(page, projection)) + } + } + return next +} + +function mapUnifiedTab(tab: Tab, projection: SessionOwnerProjection): Tab { + return { + ...structuredClone(tab), + worktreeId: projection.mapWorktreeId(tab.worktreeId) + } +} + +function mapTabGroup(group: TabGroup, projection: SessionOwnerProjection): TabGroup { + return { ...structuredClone(group), worktreeId: projection.mapWorktreeId(group.worktreeId) } +} diff --git a/src/main/orcad-migration-export-holds.test.ts b/src/main/orcad-migration-export-holds.test.ts new file mode 100644 index 00000000000..53c5b0e1ee2 --- /dev/null +++ b/src/main/orcad-migration-export-holds.test.ts @@ -0,0 +1,152 @@ +import { mkdtempSync, rmSync, statSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getDefaultWorkspaceSession } from '../shared/constants' +import type { WorkspaceSessionState } from '../shared/workspace-session-state-types' +import { writeFileDurableSync } from './durable-file-write' +import { + getTerminalScrollbackSnapshotPath, + readTerminalScrollbackStoredBytesSync, + writeTerminalScrollbackSnapshotSync +} from './terminal-scrollback-snapshots' +import { deleteRemovedTerminalScrollbackSnapshots } from './persistence/loading-store/terminal-session-cleanup' +import { deleteRemovedTerminalScrollbackSnapshotsAsync } from './terminal-scrollback-snapshot-async-migration' +import { extractSessionOwnersForTransfer } from './orca-profiles/profile-session-owner-transfer' +import type { SleepingAgentSessionRecord } from '../shared/agent-session-resume' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function storage() { + const root = mkdtempSync(join(tmpdir(), 'orcad-export-holds-')) + roots.push(root) + return { root, storage: { snapshotRoot: join(root, 'terminal-scrollback') } } +} + +function sessionWithRef(ref: string | null): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + terminalLayoutsByTabId: ref + ? { + 'tab-1': { + root: null, + activeLeafId: null, + expandedLeafId: null, + scrollbackRefsByLeafId: { 'leaf-1': ref } + } + } + : {} + } +} + +describe('scrollback store reads and retention for migration export', () => { + it('reads stored bytes for a ref and none for an unknown one', () => { + const { storage: store } = storage() + const ref = writeTerminalScrollbackSnapshotSync({ + tabId: 'tab-1', + leafId: 'leaf-1', + buffer: 'saved output', + storage: store + }) + expect(ref).not.toBeNull() + expect(readTerminalScrollbackStoredBytesSync(ref ?? '', store)?.toString('utf8')).toBe( + 'saved output' + ) + expect(readTerminalScrollbackStoredBytesSync(`v1-${'0'.repeat(32)}`, store)).toBeNull() + }) + + it.each(['sync', 'async'] as const)( + 'keeps a %s-removed snapshot a pending export still reads', + async (mode) => { + const { storage: store } = storage() + const ref = + writeTerminalScrollbackSnapshotSync({ + tabId: 'tab-1', + leafId: 'leaf-1', + buffer: 'retained', + storage: store + }) ?? '' + const path = getTerminalScrollbackSnapshotPath(ref, store) ?? '' + const retained = new Set([ref]) + if (mode === 'sync') { + deleteRemovedTerminalScrollbackSnapshots( + sessionWithRef(ref), + sessionWithRef(null), + store, + retained + ) + } else { + await deleteRemovedTerminalScrollbackSnapshotsAsync( + sessionWithRef(ref), + sessionWithRef(null), + store, + retained + ) + } + expect(readFileSync(path, 'utf8')).toBe('retained') + deleteRemovedTerminalScrollbackSnapshots(sessionWithRef(ref), sessionWithRef(null), store) + expect(() => statSync(path)).toThrow() + } + ) +}) + +describe.skipIf(process.platform === 'win32')('durable writes with a creation mode', () => { + it('creates the file with the requested mode', () => { + const { root } = storage() + const finalPath = join(root, 'state.json') + writeFileDurableSync(`${finalPath}.tmp`, finalPath, '{}', 0o600) + expect(statSync(finalPath).mode & 0o777).toBe(0o600) + }) +}) + +describe('session owner projection hooks', () => { + function sleeping(paneKey: string): SleepingAgentSessionRecord { + return { + paneKey, + worktreeId: 'repo-1::/srv/app', + agent: 'claude', + providerSession: { key: 'session_id', id: paneKey }, + prompt: 'resume me', + state: 'done', + capturedAt: 1, + updatedAt: 1 + } + } + + it('lets a transfer keep resumable sleeping agents and project focus scalars', () => { + const source: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + sleepingAgentSessionsByPaneKey: { + 'tab-1:leaf-1': sleeping('tab-1:leaf-1'), + 'tab-2:leaf-1': sleeping('tab-2:leaf-1') + } + } + const projectSessionFocus = vi.fn() + const transferred = extractSessionOwnersForTransfer(source, { + mapOwnerKey: (ownerKey) => ownerKey, + mapWorktreeId: (id) => id, + projectSleepingAgentSession: (record) => (record.paneKey === 'tab-1:leaf-1' ? record : null), + projectSessionFocus + }) + expect(Object.keys(transferred.sleepingAgentSessionsByPaneKey ?? {})).toEqual(['tab-1:leaf-1']) + expect(projectSessionFocus).toHaveBeenCalledWith( + expect.objectContaining({ source, transferred }) + ) + }) + + it('drops sleeping agents when the projection does not opt in', () => { + const transferred = extractSessionOwnersForTransfer( + { + ...getDefaultWorkspaceSession(), + sleepingAgentSessionsByPaneKey: { 'tab-1:leaf-1': sleeping('tab-1:leaf-1') } + }, + { mapOwnerKey: (ownerKey) => ownerKey, mapWorktreeId: (id) => id } + ) + expect(transferred.sleepingAgentSessionsByPaneKey).toBeUndefined() + }) +}) diff --git a/src/main/orcad/electron-serve-browser-process.test.ts b/src/main/orcad/electron-serve-browser-process.test.ts index d3ea8c87e23..b92d6150dfd 100644 --- a/src/main/orcad/electron-serve-browser-process.test.ts +++ b/src/main/orcad/electron-serve-browser-process.test.ts @@ -131,6 +131,42 @@ afterEach(async () => { }) describe('ElectronServeBrowserProcess start-up', () => { + it('does not launch when startup is already cancelled', async () => { + const processHandle = new ElectronServeBrowserProcess(INSTALLED_EXECUTABLE) + started.push(processHandle) + await expect(processHandle.start(AbortSignal.abort())).rejects.toThrow() + expect(spawnProcessMock).not.toHaveBeenCalled() + }) + + it('cancels readiness polling and cleans up the unready sidecar', async () => { + await setControl({ capabilities: [['runtime.v1']] }) + const controller = new AbortController() + const processHandle = new ElectronServeBrowserProcess(INSTALLED_EXECUTABLE) + started.push(processHandle) + const starting = processHandle.start(controller.signal) + const outcome = starting.then( + () => ({ rejected: false }), + () => ({ rejected: true }) + ) + try { + await vi.waitFor(async () => expect(await sidecarRequests()).not.toHaveLength(0), { + timeout: 10_000 + }) + } finally { + controller.abort() + await outcome + } + expect(await outcome).toEqual({ rejected: true }) + expect(processHandle.isAvailable()).toBe(false) + const userDataPath = (spawnSpec().args ?? []) + .find((arg) => arg.startsWith('--user-data-dir='))! + .slice('--user-data-dir='.length) + const metadata = JSON.parse(await readFile(join(userDataPath, 'orca-runtime.json'), 'utf8')) + await processHandle.stop() + expect(existsSync(userDataPath)).toBe(false) + expect(() => process.kill(metadata.pid, 0)).toThrow() + }) + it('launches the installed app in headless serve mode without orcad browser env', async () => { for (const key of AGENT_BROWSER_ENVIRONMENT_KEYS) { vi.stubEnv(key, `leaked-${key}`) diff --git a/src/main/orcad/electron-serve-browser-process.ts b/src/main/orcad/electron-serve-browser-process.ts index d55020551ef..10cbf5bfff3 100644 --- a/src/main/orcad/electron-serve-browser-process.ts +++ b/src/main/orcad/electron-serve-browser-process.ts @@ -104,11 +104,13 @@ export class ElectronServeBrowserProcess { constructor(private readonly executablePath: string) {} - async start(): Promise { + async start(signal?: AbortSignal): Promise { + signal?.throwIfAborted() const temporaryRoot = process.platform === 'win32' ? tmpdir() : '/tmp' const userDataPath = await mkdtemp(join(temporaryRoot, 'orcad-browser-')) this.sidecarDataPath = userDataPath const port = await reserveLoopbackPort() + signal?.throwIfAborted() const child = spawnProcess({ program: this.executablePath, args: [ @@ -132,12 +134,14 @@ export class ElectronServeBrowserProcess { const deadline = Date.now() + START_TIMEOUT_MS let lastError: unknown = null while (Date.now() < deadline) { + signal?.throwIfAborted() const metadata = readRuntimeMetadata(userDataPath) if (metadata) { try { const status = RuntimeStatusResult.parse( await sendOrcadSidecarRequest(metadata, 'status.get', undefined, 5_000) ) + signal?.throwIfAborted() if (status.capabilities?.includes('browser.headless.v1')) { this.metadata = metadata return @@ -150,7 +154,7 @@ export class ElectronServeBrowserProcess { if (child.exitCode !== null || child.signalCode !== null) { break } - await delay(100) + await delay(100, undefined, { signal }) } throw new Error( `Installed Electron browser provider did not become ready: ${ diff --git a/src/main/orcad/external-chromium-browser-process.test.ts b/src/main/orcad/external-chromium-browser-process.test.ts new file mode 100644 index 00000000000..6122a4b2fe4 --- /dev/null +++ b/src/main/orcad/external-chromium-browser-process.test.ts @@ -0,0 +1,46 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { start, stop, initialize, clear } = vi.hoisted(() => ({ + start: vi.fn(), + stop: vi.fn(), + initialize: vi.fn(), + clear: vi.fn() +})) +vi.mock('./external-chromium-browser-session', () => ({ + ExternalChromiumBrowserSession: class { + start = start + stop = stop + } +})) +vi.mock('./external-chromium-tab-registry', () => ({ + ExternalChromiumTabRegistry: class { + initialize = initialize + clear = clear + } +})) + +import { ExternalChromiumBrowserProcess } from './external-chromium-browser-process' + +describe('external Chromium startup cancellation', () => { + beforeEach(() => vi.resetAllMocks()) + + it('does not publish a session that resolves after cancellation and permits cleanup', async () => { + const controller = new AbortController() + start.mockImplementation(async () => { + controller.abort() + return 'tab-live' + }) + const browser = new ExternalChromiumBrowserProcess( + '/opt/orca/agent-browser', + { executablePath: '/opt/orca/chromium', provider: 'chromium' }, + '/state' + ) + await expect(browser.start(controller.signal)).rejects.toMatchObject({ name: 'AbortError' }) + expect(start).toHaveBeenCalledWith(controller.signal) + expect(initialize).not.toHaveBeenCalled() + expect(browser.isAvailable()).toBe(false) + await browser.stop() + expect(stop).toHaveBeenCalledWith() + expect(clear).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/orcad/external-chromium-browser-process.ts b/src/main/orcad/external-chromium-browser-process.ts index 912a44775e7..112278547cd 100644 --- a/src/main/orcad/external-chromium-browser-process.ts +++ b/src/main/orcad/external-chromium-browser-process.ts @@ -37,8 +37,10 @@ export class ExternalChromiumBrowserProcess { this.tabs = new ExternalChromiumTabRegistry(this.session) } - async start(): Promise { - this.tabs.initialize(await this.session.start()) + async start(signal?: AbortSignal): Promise { + const activeTabId = await this.session.start(signal) + signal?.throwIfAborted() + this.tabs.initialize(activeTabId) this.available = true } diff --git a/src/main/orcad/external-chromium-browser-session.test.ts b/src/main/orcad/external-chromium-browser-session.test.ts index 730a3d36e2d..989d3f6f217 100644 --- a/src/main/orcad/external-chromium-browser-session.test.ts +++ b/src/main/orcad/external-chromium-browser-session.test.ts @@ -1,8 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdir } from 'node:fs/promises' -const runProcessMock = vi.fn() +const runProcessMock = vi.fn<(spec: Spec) => Promise>() vi.mock('../../shared/child-process/run-process', () => ({ - runProcess: (spec: unknown) => runProcessMock(spec) + runProcess: (spec: Spec) => runProcessMock(spec) })) vi.mock('node:fs/promises', () => ({ mkdir: vi.fn(async () => undefined), @@ -21,7 +22,7 @@ const BASE = { sessionName: 'orca-orcad-0123456789abcdef' } -type Spec = { args?: readonly string[]; env?: NodeJS.ProcessEnv } +type Spec = { args?: readonly string[]; env?: NodeJS.ProcessEnv; signal?: AbortSignal } function commands(): string[][] { return runProcessMock.mock.calls.map((call) => [...((call[0] as Spec).args ?? [])]) @@ -30,8 +31,60 @@ function commands(): string[][] { describe('orcad external-chromium agent-browser environment', () => { beforeEach(() => { runProcessMock.mockReset() + vi.mocked(mkdir).mockClear() }) + it('does no work when startup is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const session = new ExternalChromiumBrowserSession( + '/opt/orca/agent-browser', + { executablePath: BASE.executablePath, provider: 'chromium' }, + '/state' + ) + await expect(session.start(controller.signal)).rejects.toMatchObject({ name: 'AbortError' }) + expect(mkdir).not.toHaveBeenCalled() + expect(runProcessMock).not.toHaveBeenCalled() + }) + + it.each(['tab', 'close', 'open'])( + 'does not continue startup after abort during %s', + async (phase) => { + const controller = new AbortController() + runProcessMock.mockImplementation(async (spec: Spec) => { + if (spec.args?.includes(phase)) { + controller.abort() + } + return { + code: 0, + signal: null, + stdout: JSON.stringify({ success: true, data: { tabs: [] } }), + stderr: '', + timedOut: false + } + }) + const session = new ExternalChromiumBrowserSession( + '/opt/orca/agent-browser', + { executablePath: BASE.executablePath, provider: 'chromium' }, + '/state' + ) + await expect(session.start(controller.signal)).rejects.toMatchObject({ name: 'AbortError' }) + const specs = runProcessMock.mock.calls.map(([spec]) => spec) + const issued = specs.map((spec) => + spec.args?.find((arg) => ['tab', 'close', 'open'].includes(arg)) + ) + expect(issued).toEqual( + phase === 'tab' ? ['tab'] : phase === 'close' ? ['tab', 'close'] : ['tab', 'close', 'open'] + ) + for (const spec of specs) { + expect(spec.signal).toBe(spec.args?.includes('close') ? undefined : controller.signal) + } + await session.stop() + expect(runProcessMock.mock.lastCall?.[0]).toMatchObject({ signal: undefined }) + expect(commands().at(-1)).toContain('close') + } + ) + // Why: this daemon owns the user's remote Chromium, so an idle bound would close a live browser. it('never bounds the daemon that owns the Chromium tree', () => { const env = externalChromiumAgentBrowserEnvironment({ inheritedEnv: {}, ...BASE }) diff --git a/src/main/orcad/external-chromium-browser-session.ts b/src/main/orcad/external-chromium-browser-session.ts index de72ae8a061..56c7bd47e78 100644 --- a/src/main/orcad/external-chromium-browser-session.ts +++ b/src/main/orcad/external-chromium-browser-session.ts @@ -87,20 +87,25 @@ export class ExternalChromiumBrowserSession { this.profilePath = join(statePath, `browser-${launch.provider}`) } - async start(): Promise { + async start(signal?: AbortSignal): Promise { + signal?.throwIfAborted() await mkdir(this.profilePath, { recursive: true }) + signal?.throwIfAborted() // Why: the session name is stable across runs, so a daemon an earlier orcad left behind is // still driving the user's Chromium. Unlike the pane bridge this session never passes --cdp, // so nothing binds it to the old process — a surviving one is reusable as-is, and closing it // would take the remote user's browser and every tab with it (#16367). - const reusable = await this.readActiveTabId() + const reusable = await this.readActiveTabId(signal) + signal?.throwIfAborted() if (reusable) { return reusable } // Nothing answered, so anything under this name is wedged or half-dead; reclaim it. await this.stop() - await this.run(['open', 'about:blank']) - const opened = await this.readActiveTabId() + signal?.throwIfAborted() + await this.run(['open', 'about:blank'], COMMAND_TIMEOUT_MS, signal) + const opened = await this.readActiveTabId(signal) + signal?.throwIfAborted() if (!opened) { throw new BrowserError( BROWSER_UNAVAILABLE_ERROR_CODE, @@ -110,11 +115,12 @@ export class ExternalChromiumBrowserSession { return opened } - private async readActiveTabId(): Promise { + private async readActiveTabId(signal?: AbortSignal): Promise { try { - const tabs = await this.readTabs() + const tabs = await this.readTabs(signal) return (tabs.find((tab) => tab.active) ?? tabs[0])?.tabId ?? null } catch { + signal?.throwIfAborted() return null } } @@ -131,8 +137,10 @@ export class ExternalChromiumBrowserSession { await this.run(['tab', agentPageId]) } - async readTabs(): Promise { - return AgentBrowserTabsResult.parse(await this.run(['tab'])).tabs ?? [] + async readTabs(signal?: AbortSignal): Promise { + return ( + AgentBrowserTabsResult.parse(await this.run(['tab'], COMMAND_TIMEOUT_MS, signal)).tabs ?? [] + ) } async screenshot(params: Record, full: boolean): Promise { @@ -152,7 +160,12 @@ export class ExternalChromiumBrowserSession { return { data } } - async run(command: readonly string[], timeoutMs = COMMAND_TIMEOUT_MS): Promise { + async run( + command: readonly string[], + timeoutMs = COMMAND_TIMEOUT_MS, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() const args = ['--session', this.sessionName, '--profile', this.profilePath] if (this.launch.browserArgs?.length) { args.push('--args', this.launch.browserArgs.join('\n')) @@ -170,8 +183,10 @@ export class ExternalChromiumBrowserSession { args, env, timeoutMs, + signal, maxOutputBytes: MAX_OUTPUT_BYTES }) + signal?.throwIfAborted() if (result.timedOut) { throw new BrowserError('browser_timeout', 'Browser command timed out.') } diff --git a/src/main/orcad/main-preflight-order.test.ts b/src/main/orcad/main-preflight-order.test.ts index 8a7fc60407f..d6744cdcb10 100644 --- a/src/main/orcad/main-preflight-order.test.ts +++ b/src/main/orcad/main-preflight-order.test.ts @@ -3,6 +3,10 @@ import { ORCAD_PROFILE_PREFLIGHT_FLAG, ORCAD_STARTUP_PREFLIGHT_FLAG } from '../../shared/orcad-profile-preflight' +import { + ORCAD_CANCEL_MANAGED_STOP_FLAG, + ORCAD_COMPLETE_MANAGED_STOP_FLAG +} from '../../shared/orcad-stop-request' /** * The precondition is only worth anything if it runs first. A loader failure is not @@ -38,6 +42,12 @@ vi.mock('./orcad-native-preflight', () => ({ } })) +vi.mock('./orcad-managed-stop-command', () => ({ + runOrcadManagedStopCommandAndExit: async (argv: string[]) => { + order.push(`managed-stop:${argv.join(' ')}`) + } +})) + vi.mock('./orcad-entry', () => ({ main: async () => { order.push('main') @@ -67,4 +77,43 @@ describe('orcad entry', () => { expect(order).toEqual(['profile-admission', 'preflight', 'main']) }) + + it.each([ORCAD_COMPLETE_MANAGED_STOP_FLAG, ORCAD_CANCEL_MANAGED_STOP_FLAG])( + 'runs %s without preflights, the bundled handoff, or a runtime', + async (flag) => { + vi.spyOn(process, 'argv', 'get').mockReturnValue(['runtime', 'orcad.js', flag, '{}']) + await import('./main') + await vi.waitFor(() => expect(order).toHaveLength(1)) + + expect(order).toEqual([`managed-stop:${flag} {}`]) + } + ) + + it('runs the Windows breakaway launcher without preflights or a runtime', async () => { + vi.spyOn(process, 'argv', 'get').mockReturnValue([ + 'runtime', + 'orcad.js', + '--windows-breakaway-launch', + '--stdout-file', + 'out', + '--stderr-file', + 'err', + '--orcad-args', + '--json' + ]) + const write = vi + .spyOn(process.stdout, 'write') + .mockImplementation((_chunk: unknown, callback?: unknown) => { + if (typeof callback === 'function') { + callback() + } + return true + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stub only records the call; nothing reads its never return. + const exit = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never) + await import('./main') + await vi.waitFor(() => expect(exit).toHaveBeenCalled()) + expect(String(write.mock.calls[0]?.[0])).toMatch(/^ORCA_ORCAD_LAUNCH /u) + expect(order).toEqual([]) + }) }) diff --git a/src/main/orcad/main.ts b/src/main/orcad/main.ts index 46f98853c02..325ada702d6 100644 --- a/src/main/orcad/main.ts +++ b/src/main/orcad/main.ts @@ -1,6 +1,7 @@ /** Executable entry for `orcad`. See `./orcad-entry.ts`. */ import process from 'node:process' import { main, resolveOrcadExitCode } from './orcad-entry' +import { reserveServeStdoutForReadiness } from '../server/serve-stdout-boundary' import { runOrcadNativePreflight } from './orcad-native-preflight' import { ORCAD_PROFILE_PREFLIGHT_FLAG, @@ -8,6 +9,19 @@ import { } from '../../shared/orcad-profile-preflight' import { preflightBundledOrcadStartup, runOrcadProfilePreflight } from './orcad-profile-preflight' import { handoffToBundledOrcad } from './orcad-bundled-runtime' +import { + formatOrcadNativePreflightReport, + ORCAD_NATIVE_PREFLIGHT_FLAG +} from '../../shared/orcad-native-preflight-report' +import { + ORCAD_CANCEL_MANAGED_STOP_FLAG, + ORCAD_COMPLETE_MANAGED_STOP_FLAG +} from '../../shared/orcad-stop-request' +import { + ORCAD_WINDOWS_BREAKAWAY_CONTRACT, + WINDOWS_BREAKAWAY_LAUNCH_FLAG +} from '../../shared/windows-breakaway-launch' +import { runWindowsBreakawayLaunchIfRequested } from '../../shared/windows-breakaway-launcher' // Why exit before the preflight: reaching this line means the whole module graph resolved // under plain Node, which is all the build guard needs to prove. Probing natives or @@ -30,28 +44,52 @@ function failStartup(error: unknown): void { process.exit(resolveOrcadExitCode(error)) } -try { - if (!handoffToBundledOrcad()) { - const flag = process.argv[2] - if ( - (flag === ORCAD_PROFILE_PREFLIGHT_FLAG || flag === ORCAD_STARTUP_PREFLIGHT_FLAG) && - process.argv.length === 4 - ) { - void runOrcadProfilePreflight(process.argv[3], { - nativeFeatures: flag === ORCAD_PROFILE_PREFLIGHT_FLAG - }) - // Why exit: the owner reads to EOF, so a lingering native handle must not hold the probe open. - .then(() => process.stdout.write('', () => process.exit(0))) - .catch(failStartup) - } else { - void preflightBundledOrcadStartup() - .then(() => { - runOrcadNativePreflight() - return main() +// Why before the bundled handoff and preflights: launching or completing a stop must not start a runtime. +if (process.argv[2] === WINDOWS_BREAKAWAY_LAUNCH_FLAG) { + runWindowsBreakawayLaunchIfRequested(ORCAD_WINDOWS_BREAKAWAY_CONTRACT, process.argv) +} else if ( + process.argv[2] === ORCAD_COMPLETE_MANAGED_STOP_FLAG || + process.argv[2] === ORCAD_CANCEL_MANAGED_STOP_FLAG +) { + void import('./orcad-managed-stop-command').then(({ runOrcadManagedStopCommandAndExit }) => + runOrcadManagedStopCommandAndExit(process.argv.slice(2)) + ) +} else { + startOrcadProcess() +} + +function startOrcadProcess(): void { + try { + if (!handoffToBundledOrcad()) { + const flag = process.argv[2] + if (flag === ORCAD_NATIVE_PREFLIGHT_FLAG && process.argv.length === 3) { + void import('./node-pty-precondition').then(({ checkNodePtyPrecondition }) => { + const verdict = checkNodePtyPrecondition() + const report = formatOrcadNativePreflightReport(verdict.status, verdict.reason ?? null) + process.stdout.write(`${report}\n`, () => process.exit(0)) + }, failStartup) + } else if ( + (flag === ORCAD_PROFILE_PREFLIGHT_FLAG || flag === ORCAD_STARTUP_PREFLIGHT_FLAG) && + process.argv.length === 4 + ) { + void runOrcadProfilePreflight(process.argv[3], { + nativeFeatures: flag === ORCAD_PROFILE_PREFLIGHT_FLAG }) - .catch(failStartup) + // Why exit: the owner reads to EOF, so a lingering native handle must not hold the probe open. + .then(() => process.stdout.write('', () => process.exit(0))) + .catch(failStartup) + } else { + // Why: stdout is the serve readiness API; incidental diagnostics go to stderr. + reserveServeStdoutForReadiness() + void preflightBundledOrcadStartup() + .then(() => { + runOrcadNativePreflight() + return main() + }) + .catch(failStartup) + } } + } catch (error) { + failStartup(error) } -} catch (error) { - failStartup(error) } diff --git a/src/main/orcad/orcad-artifact-cache-retention.test.ts b/src/main/orcad/orcad-artifact-cache-retention.test.ts new file mode 100644 index 00000000000..edaa0a11a54 --- /dev/null +++ b/src/main/orcad/orcad-artifact-cache-retention.test.ts @@ -0,0 +1,148 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { ORCAD_LOCK_FILE_NAME } from './orcad-instance-lock' +import { + liveLocalOrcadServeVersion, + pruneDesktopOrcadArtifactCache, + pruneOrcadArtifactCache +} from './orcad-artifact-cache-retention' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function cacheRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orcad-artifact-cache-')) + roots.push(root) + return root +} + +/** A slot dir last used `ageMinutes` ago. */ +function slot(root: string, target: string, name: string, ageMinutes: number): string { + const path = join(root, target, name) + mkdirSync(path, { recursive: true }) + writeFileSync(join(path, 'orcad.js'), '') + const at = new Date(Date.now() - ageMinutes * 60_000) + utimesSync(path, at, at) + return path +} + +describe('orcad artifact cache retention', () => { + it('keeps the three most recently used slots per target and evicts older ones', async () => { + const root = cacheRoot() + const kept = [1, 2, 3].map((age) => slot(root, 'linux-x64-glibc', `v${age}`, age)) + const evicted = [4, 5].map((age) => slot(root, 'linux-x64-glibc', `v${age}`, age)) + const otherTarget = slot(root, 'darwin-arm64', 'v9', 9) + + expect((await pruneOrcadArtifactCache(root)).sort()).toEqual(evicted.sort()) + for (const path of [...kept, otherTarget]) { + expect(existsSync(path)).toBe(true) + } + for (const path of evicted) { + expect(existsSync(path)).toBe(false) + } + }) + + it('keeps an in-use version (even a repair copy) plus the two most recent others', async () => { + const root = cacheRoot() + const recent = [1, 2].map((age) => slot(root, 'linux-x64-glibc', `v${age}`, age)) + const third = slot(root, 'linux-x64-glibc', 'v3', 3) + const inUse = slot(root, 'linux-x64-glibc', 'v-old.repair-1', 50) + const staging = slot(root, 'linux-x64-glibc', '.staging-1-abc', 60) + + expect(await pruneOrcadArtifactCache(root, { inUseVersions: new Set(['v-old']) })).toEqual([ + third + ]) + for (const path of [...recent, inUse, staging]) { + expect(existsSync(path)).toBe(true) + } + }) + + it('leaves the runtime archive cache and a missing root alone', async () => { + const root = cacheRoot() + const archives = join(root, 'node', 'archives') + mkdirSync(archives, { recursive: true }) + for (const age of [1, 2, 3, 4]) { + slot(root, 'node', `archive-${age}`, age) + } + expect(await pruneOrcadArtifactCache(root)).toEqual([]) + expect(await pruneOrcadArtifactCache(join(root, 'missing'))).toEqual([]) + }) + + it('names the slot a live local orcad serve runs from, and nothing for the desktop', () => { + const userData = cacheRoot() + const lock = (role: string) => + writeFileSync( + join(userData, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ + pid: process.pid, + startedAtMs: null, + nonce: 'nonce', + identity: 'uid', + version: '0.1.0+abc', + acquiredAt: new Date().toISOString(), + role + }) + ) + lock('orcad') + expect(liveLocalOrcadServeVersion(userData)).toBe('0.1.0+abc') + expect(liveLocalOrcadServeVersion(userData, () => false)).toBeNull() + lock('desktop') + expect(liveLocalOrcadServeVersion(userData)).toBeNull() + }) + + it('keeps both the selected slot and the one a running orcad serve still uses', async () => { + const userData = cacheRoot() + const root = join(userData, 'orcad-artifacts') + const running = slot(root, 'linux-x64-glibc', '0.1.0+old', 90) + const selected = slot(root, 'linux-x64-glibc', '0.4.0+new', 1) + const others = [2, 3, 4].map((age) => slot(root, 'linux-x64-glibc', `v${age}`, age)) + writeFileSync( + join(userData, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ + pid: process.pid, + startedAtMs: null, + nonce: 'nonce', + identity: 'uid', + version: '0.1.0+old', + acquiredAt: new Date().toISOString(), + role: 'orcad' + }) + ) + expect(await pruneDesktopOrcadArtifactCache(userData, ['0.4.0+new'])).toEqual([others[2]]) + expect(existsSync(running)).toBe(true) + expect(existsSync(selected)).toBe(true) + }) + + it('keeps the slot a surviving terminal daemon runs from after orcad exits', async () => { + const userData = cacheRoot() + const root = join(userData, 'orcad-artifacts') + const daemonSlot = slot(root, 'linux-x64-glibc', '0.1.0+old', 90) + const others = [1, 2, 3, 4].map((age) => slot(root, 'linux-x64-glibc', `v${age}`, age)) + mkdirSync(join(userData, 'daemon')) + writeFileSync( + join(userData, 'daemon', 'daemon-v30.pid'), + JSON.stringify({ pid: process.pid, entryPath: join(daemonSlot, 'out', 'daemon-entry.js') }) + ) + expect((await pruneDesktopOrcadArtifactCache(userData)).sort()).toEqual( + [others[2], others[3]].sort() + ) + expect(existsSync(daemonSlot)).toBe(true) + }) + + it('evicts nothing while a daemon record cannot be read', async () => { + const userData = cacheRoot() + const root = join(userData, 'orcad-artifacts') + for (const age of [1, 2, 3, 4, 5]) { + slot(root, 'linux-x64-glibc', `v${age}`, age) + } + mkdirSync(join(userData, 'daemon')) + writeFileSync(join(userData, 'daemon', 'daemon-v30.pid'), '') + expect(await pruneDesktopOrcadArtifactCache(userData)).toEqual([]) + }) +}) diff --git a/src/main/orcad/orcad-artifact-cache-retention.ts b/src/main/orcad/orcad-artifact-cache-retention.ts new file mode 100644 index 00000000000..e09a052b2d2 --- /dev/null +++ b/src/main/orcad/orcad-artifact-cache-retention.ts @@ -0,0 +1,119 @@ +/** + * Bounds the desktop's `/orcad-artifacts` slot cache: per target, keep the most + * recently used slots and evict the rest at startup. The cache survives uninstall (it lives in userData); this is what keeps it small. + * + * Never evicted: a slot this process materialized (an SSH deploy may be reading it), the slot + * a live local orcad serve runs from (named by the profile's instance lock), and the slot a + * surviving terminal daemon runs from (named by its PID record). + */ +import { readdir, rm, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { nodeRuntimeAsset } from '../../shared/node-runtime-pin' +import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock' +import { materializedOrcadArtifactVersions } from '../ssh/orcad-artifact-materializer' +import { isProcessAlive } from '../daemon/daemon-process-inspection' +import { liveDaemonOrcadSlots } from './orcad-daemon-slot-pins' + +/** The in-use slot plus the two most recent others (the newest three when none is in use). */ +export const ORCAD_ARTIFACT_CACHE_KEEP = 3 + +const REPAIR_SUFFIX = /\.repair-\d+$/u + +export type OrcadArtifactCachePruneOptions = { + keep?: number + /** Slot versions to keep whatever their age. */ + inUseVersions?: ReadonlySet +} + +/** Removes the stale slots under `cacheRoot` and returns their paths. Best effort per entry. */ +export async function pruneOrcadArtifactCache( + cacheRoot: string, + options: OrcadArtifactCachePruneOptions = {} +): Promise { + const keep = options.keep ?? ORCAD_ARTIFACT_CACHE_KEEP + const inUse = options.inUseVersions ?? new Set() + const removed: string[] = [] + for (const target of await listNames(cacheRoot)) { + // Only target directories hold slots; `node/` is the runtime archive cache. + if (!nodeRuntimeAsset(target)) { + continue + } + const targetRoot = join(cacheRoot, target) + const slots = await Promise.all( + // Dot entries are staging directories another process may be filling right now. + (await listNames(targetRoot)) + .filter((name) => !name.startsWith('.')) + .map(async (name) => ({ name, mtimeMs: await mtimeOf(join(targetRoot, name)) })) + ) + const present = slots.filter( + (slot): slot is { name: string; mtimeMs: number } => slot.mtimeMs !== null + ) + const isInUse = (name: string): boolean => inUse.has(name.replace(REPAIR_SUFFIX, '')) + // The in-use version counts toward `keep`; with none in use, the newest stands in for it. + const othersToKeep = present.some((slot) => isInUse(slot.name)) ? keep - 1 : keep + const others = present + .filter((slot) => !isInUse(slot.name)) + .sort((left, right) => right.mtimeMs - left.mtimeMs) + for (const [index, slot] of others.entries()) { + if (index < othersToKeep) { + continue + } + const path = join(targetRoot, slot.name) + try { + await rm(path, { recursive: true, force: true }) + removed.push(path) + } catch (error) { + console.warn(`[orcad-artifacts] could not evict ${path}:`, error) + } + } + } + return removed +} + +/** A pass over `/orcad-artifacts` that never evicts a slot something still runs from. */ +export function pruneDesktopOrcadArtifactCache( + userDataPath: string, + alsoInUse: readonly string[] = [] +): Promise { + const cacheRoot = join(userDataPath, 'orcad-artifacts') + const daemonSlots = liveDaemonOrcadSlots(userDataPath, cacheRoot) + if (!daemonSlots) { + // An unreadable daemon record could name any slot; evicting nothing is the safe answer. + return Promise.resolve([]) + } + const live = liveLocalOrcadServeVersion(userDataPath) + return pruneOrcadArtifactCache(cacheRoot, { + inUseVersions: new Set([ + ...materializedOrcadArtifactVersions(), + ...(live ? [live] : []), + ...daemonSlots.map((slot) => slot.replace(REPAIR_SUFFIX, '')), + ...alsoInUse + ]) + }) +} + +/** The slot version a live local orcad serve runs from, or null when none holds the profile. */ +export function liveLocalOrcadServeVersion( + userDataPath: string, + isAlive: (pid: number) => boolean = isProcessAlive +): string | null { + const record = readOrcadInstanceLockRecord(join(userDataPath, ORCAD_LOCK_FILE_NAME)) + return record && record.role !== 'desktop' && isAlive(record.pid) ? record.version : null +} + +async function listNames(directory: string): Promise { + try { + return await readdir(directory) + } catch { + return [] + } +} + +async function mtimeOf(path: string): Promise { + try { + const info = await stat(path) + return info.isDirectory() ? info.mtimeMs : null + } catch { + return null + } +} diff --git a/src/main/orcad/orcad-artifact-identity.ts b/src/main/orcad/orcad-artifact-identity.ts index 86140acdceb..27b7c905daa 100644 --- a/src/main/orcad/orcad-artifact-identity.ts +++ b/src/main/orcad/orcad-artifact-identity.ts @@ -8,13 +8,13 @@ import { ORCAD_VERSION, orcadArtifactFilenames } from '../../shared/orcad-artifacts' -import { SERVER_TARGETS } from '../../shared/node-runtime-pin' +import { COMPAT_SERVER_TARGETS, SERVER_TARGETS } from '../../shared/node-runtime-pin' import { orcadAgentBrowserNativeName } from '../../shared/orcad-agent-browser-name' /** Hash installed bytes in the build's order; a version marker is not proof of delivery. */ export async function readOrcadArtifactIdentity(directory: string): Promise { const target = z - .enum(SERVER_TARGETS) + .enum([...SERVER_TARGETS, ...COMPAT_SERVER_TARGETS]) .parse((await readFile(join(directory, ORCAD_SERVER_TARGET_FILENAME), 'utf8')).trim()) const platform = target.startsWith('win32-') ? 'win32' diff --git a/src/main/orcad/orcad-automations.test.ts b/src/main/orcad/orcad-automations.test.ts new file mode 100644 index 00000000000..5e932d58370 --- /dev/null +++ b/src/main/orcad/orcad-automations.test.ts @@ -0,0 +1,78 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { closeTestStores, createStore, makeRepo, testState } from '../persistence-test-harness' +import type { AutomationService } from '../automations/service' +import { orcadAutomationsKeepHostBusy, startOrcadAutomations } from './orcad-automations' + +vi.mock('electron', () => ({ app: { getPath: () => testState.dir } })) + +function headlessRuntime() { + let bound: AutomationService | null = null + const runtime = { + setAutomationService: vi.fn((service: AutomationService) => { + bound = service + }), + notifyAutomationsChanged: vi.fn(), + createManagedWorktree: vi.fn(async () => ({ + worktree: { id: 'r1::/repo/wt', displayName: 'wt' }, + startupTerminal: { handle: 'term-1', tabId: 'tab-1', paneKey: 'tab-1:1', ptyId: 'pty-1' } + })), + waitForTerminal: vi.fn(async () => ({ satisfied: true })), + readTerminal: vi.fn(async () => ({ tail: ['done'] })), + getTerminalHandleForPaneKey: vi.fn(() => null), + getAgentStatusRowsForPane: vi.fn(() => []) + } + return { runtime, service: () => bound } +} + +describe('orcad automations', () => { + beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orcad-automations-')) + }) + afterEach(async () => { + await closeTestStores() + rmSync(testState.dir, { recursive: true, force: true }) + }) + + it('binds a headless service, so a run dispatches on orcad without a renderer', async () => { + const store = createStore() + store.addRepo(makeRepo()) + const automation = store.createAutomation({ + name: 'Nightly', + prompt: 'Run checks', + agentId: 'claude', + projectId: 'r1', + workspaceMode: 'new_per_run', + timezone: 'UTC', + rrule: 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', + dtstart: Date.parse('2026-05-13T00:00:00Z') + }) + const { runtime, service } = headlessRuntime() + const cleanups: (() => void)[] = [] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the headless dispatcher reaches only the runtime methods stubbed above. + startOrcadAutomations(runtime as never, store, (cleanup) => cleanups.push(cleanup)) + try { + expect(runtime.setAutomationService).toHaveBeenCalledOnce() + const run = await service()!.runNow(automation.id) + expect(runtime.createManagedWorktree).toHaveBeenCalledOnce() + expect(run.status).toBe('dispatched') + expect(orcadAutomationsKeepHostBusy(store)).toBe(true) + } finally { + cleanups.forEach((cleanup) => cleanup()) + } + }) + + it('lets a host with no enabled schedule and no unsettled run idle out', () => { + const store = { listAutomations: () => [], listAutomationRuns: () => [] } + expect(orcadAutomationsKeepHostBusy(store)).toBe(false) + }) + + // Booting orcad here would need its whole runtime; the wiring is pinned by the entry point's text. + it('orcad starts the service with its runtime and feeds it to managed idle exit', () => { + const entry = readFileSync(join(import.meta.dirname, 'orcad-entry.ts'), 'utf8') + expect(entry).toContain('startOrcadAutomations(runtime, profileStore, registerCleanup)') + expect(entry).toContain('automationsBusy: () => orcadAutomationsKeepHostBusy(profileStore)') + }) +}) diff --git a/src/main/orcad/orcad-automations.ts b/src/main/orcad/orcad-automations.ts new file mode 100644 index 00000000000..f025e4ca2d4 --- /dev/null +++ b/src/main/orcad/orcad-automations.ts @@ -0,0 +1,37 @@ +/** + * orcad executes the profile's schedules, as `orca serve` on Electron does. Without this a + * persisted or migrated automation lists fine but never runs, and "Run now" has no service. + */ +import type { Store } from '../persistence' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import { createRuntimeAutomationService } from '../automations/runtime-automation-service' +import { isFinalAutomationRunStatus } from '../../shared/automations-types' + +type AutomationStore = Pick + +let stopScheduler: (() => void) | null = null + +export function startOrcadAutomations( + runtime: OrcaRuntimeService, + store: Store, + registerCleanup: (cleanup: () => void) => void +): void { + const service = createRuntimeAutomationService({ store, runtime, headless: true }) + stopScheduler = () => service.stop() + // Stops before the store flushes, so no run is written after the final profile save. + registerCleanup(() => service.stop()) + service.start() +} + +/** Halts scheduling and in-flight dispatch; a no-op before the service starts. */ +export function stopOrcadAutomationScheduler(): void { + stopScheduler?.() +} + +/** Busy while a schedule may fire or a run has not settled; idling out would skip both. */ +export function orcadAutomationsKeepHostBusy(store: AutomationStore): boolean { + return ( + store.listAutomations().some((automation) => automation.enabled) || + store.listAutomationRuns().some((run) => !isFinalAutomationRunStatus(run.status)) + ) +} diff --git a/src/main/orcad/orcad-browser-provider.ts b/src/main/orcad/orcad-browser-provider.ts index fdb8bfad34b..a350ef8f9f9 100644 --- a/src/main/orcad/orcad-browser-provider.ts +++ b/src/main/orcad/orcad-browser-provider.ts @@ -23,6 +23,7 @@ export type OrcadBrowserProvider = { export type OrcadBrowserProviderOptions = { userDataPath: string + signal?: AbortSignal environment?: NodeJS.ProcessEnv resolveInstalledElectronExecutable?: () => Promise resolveAgentBrowserBinary?: () => string | null @@ -30,6 +31,20 @@ export type OrcadBrowserProviderOptions = { type ExecutableProbe = 'ok' | 'missing' | 'not_executable' +class OrcadBrowserCleanupError extends AggregateError {} + +async function cleanupFailedProvider( + processHandle: { stop(): Promise }, + startupError: unknown +): Promise { + try { + await processHandle.stop() + } catch (cleanupError) { + throw new OrcadBrowserCleanupError([startupError, cleanupError], 'orcad_browser_cleanup_failed') + } + throw startupError +} + /** Splits the two failures apart: a wrong path and a forgotten chmod +x need different fixes. */ async function probeExecutable(path: string): Promise { const mode = process.platform === 'win32' ? constants.F_OK : constants.X_OK @@ -99,14 +114,14 @@ export async function resolveInstalledElectronExecutable(): Promise { const processHandle = new ExternalChromiumBrowserProcess(agentBrowserPath, launch, userDataPath) try { - await processHandle.start() + await processHandle.start(signal) } catch (error) { - await processHandle.stop() - throw error + return cleanupFailedProvider(processHandle, error) } return { kind: launch.provider, @@ -116,13 +131,15 @@ async function startProvider( } } -async function startElectronServeProvider(executablePath: string): Promise { +async function startElectronServeProvider( + executablePath: string, + signal?: AbortSignal +): Promise { const processHandle = new ElectronServeBrowserProcess(executablePath) try { - await processHandle.start() + await processHandle.start(signal) } catch (error) { - await processHandle.stop() - throw error + return cleanupFailedProvider(processHandle, error) } return { kind: 'electron', @@ -137,6 +154,9 @@ export async function resolveOrcadBrowserProvider( options: OrcadBrowserProviderOptions ): Promise { const environment = options.environment ?? process.env + if (options.signal?.aborted) { + return null + } await mkdir(options.userDataPath, { recursive: true, mode: 0o700 }) const declined = (cause: RuntimeBrowserUnavailableCause): null => { @@ -147,14 +167,23 @@ export async function resolveOrcadBrowserProvider( const installedElectronExecutable = await ( options.resolveInstalledElectronExecutable ?? resolveInstalledElectronExecutable )() + if (options.signal?.aborted) { + return null + } // Why held rather than reported now: Chromium may still resolve, and if it does not, its // own concrete fault is the more actionable one for an operator who set the env var. let electronFailure: RuntimeBrowserUnavailableCause | null = null if (installedElectronExecutable) { try { setRuntimeBrowserUnavailableCause(null) - return await startElectronServeProvider(installedElectronExecutable) + return await startElectronServeProvider(installedElectronExecutable, options.signal) } catch (error) { + if (error instanceof OrcadBrowserCleanupError) { + throw error + } + if (options.signal?.aborted) { + return null + } console.warn('[orcad] Installed Electron browser provider unavailable:', error) electronFailure = { reason: 'electron_start_failed', detail: errorDetail(error) } } @@ -174,6 +203,9 @@ export async function resolveOrcadBrowserProvider( } const probe = await probeExecutable(chromiumExecutable) + if (options.signal?.aborted) { + return null + } if (probe !== 'ok') { return declined({ reason: probe === 'missing' ? 'executable_not_found' : 'executable_not_executable', @@ -186,9 +218,16 @@ export async function resolveOrcadBrowserProvider( return await startProvider( agentBrowserPath, { executablePath: chromiumExecutable, provider: 'chromium' }, - options.userDataPath + options.userDataPath, + options.signal ) } catch (error) { + if (error instanceof OrcadBrowserCleanupError) { + throw error + } + if (options.signal?.aborted) { + return null + } console.warn('[orcad] External Chromium browser provider unavailable:', error) return declined({ reason: 'chromium_start_failed', detail: errorDetail(error) }) } diff --git a/src/main/orcad/orcad-browser-startup-cancellation.test.ts b/src/main/orcad/orcad-browser-startup-cancellation.test.ts new file mode 100644 index 00000000000..9eca8d4e1da --- /dev/null +++ b/src/main/orcad/orcad-browser-startup-cancellation.test.ts @@ -0,0 +1,62 @@ +import { beforeEach, expect, it, vi } from 'vitest' +import { resolveOrcadBrowserProvider } from './orcad-browser-provider' + +const { start, stop, chromiumStart } = vi.hoisted(() => ({ + start: vi.fn(), + stop: vi.fn(), + chromiumStart: vi.fn() +})) +vi.mock('node:fs/promises', () => ({ mkdir: vi.fn(), access: vi.fn() })) +vi.mock('./electron-serve-browser-process', () => ({ + ElectronServeBrowserProcess: class { + start = start + stop = stop + } +})) +vi.mock('./external-chromium-browser-process', () => ({ + ExternalChromiumBrowserProcess: class { + start = chromiumStart + } +})) +beforeEach(() => vi.resetAllMocks()) + +const options = { + userDataPath: '/fixture', + resolveInstalledElectronExecutable: async () => '/fixture/electron', + resolveAgentBrowserBinary: () => '/fixture/driver', + environment: { ORCA_BROWSER_EXECUTABLE: '/fixture/chromium' } +} + +it('does not launch a provider after pre-cancellation', async () => { + await expect( + resolveOrcadBrowserProvider({ ...options, signal: AbortSignal.abort() }) + ).resolves.toBeNull() + expect(start).not.toHaveBeenCalled() + expect(chromiumStart).not.toHaveBeenCalled() +}) + +it('cleans cancelled Electron startup without launching a fallback', async () => { + const controller = new AbortController() + start.mockImplementation(async () => { + controller.abort() + controller.signal.throwIfAborted() + }) + await expect( + resolveOrcadBrowserProvider({ ...options, signal: controller.signal }) + ).resolves.toBeNull() + expect(stop).toHaveBeenCalledOnce() + expect(chromiumStart).not.toHaveBeenCalled() +}) + +it('propagates failed cleanup even when startup was cancelled', async () => { + const controller = new AbortController() + start.mockImplementation(async () => { + controller.abort() + controller.signal.throwIfAborted() + }) + stop.mockRejectedValue(new Error('sidecar still owned')) + await expect( + resolveOrcadBrowserProvider({ ...options, signal: controller.signal }) + ).rejects.toThrow('orcad_browser_cleanup_failed') + expect(chromiumStart).not.toHaveBeenCalled() +}) diff --git a/src/main/orcad/orcad-browser-startup.test.ts b/src/main/orcad/orcad-browser-startup.test.ts new file mode 100644 index 00000000000..9f2f5812ac7 --- /dev/null +++ b/src/main/orcad/orcad-browser-startup.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { startOrcadBrowserProvider } from './orcad-browser-startup' +import { resolveOrcadBrowserProvider, type OrcadBrowserProvider } from './orcad-browser-provider' +import { + createRuntimeBrowserCommands, + runtimeBrowserCommandsFactoryIsHeadless, + runtimeBrowserUnavailableCause, + setRuntimeBrowserCommandsFactory, + setRuntimeBrowserUnavailableCause +} from '../runtime/runtime-browser-commands-factory' +import type { + RuntimeBrowserCommandHost, + RuntimeBrowserCommands +} from '../runtime/orca-runtime-browser' + +vi.mock('./orcad-browser-provider', () => ({ resolveOrcadBrowserProvider: vi.fn() })) +afterEach(() => { + vi.restoreAllMocks() + setRuntimeBrowserCommandsFactory(null) + setRuntimeBrowserUnavailableCause(null) +}) + +const host: RuntimeBrowserCommandHost = Object.create(null) +function delayedProvider() { + const pending = Promise.withResolvers() + vi.mocked(resolveOrcadBrowserProvider).mockReturnValueOnce(pending.promise) + const command = vi.fn(() => ({ tabs: [] })) + const provider: OrcadBrowserProvider = { + kind: 'electron', + factory: vi.fn((): RuntimeBrowserCommands => + Object.assign(Object.create(null), { browserTabList: command }) + ), + isAvailable: vi.fn(() => true), + stop: vi.fn(async () => {}) + } + const startup = startOrcadBrowserProvider({ userDataPath: '/private-fixture' }) + return { pending, provider, command, startup } +} + +it('returns before discovery and keeps already-created commands usable after readiness', async () => { + const { pending, provider, command, startup } = delayedProvider() + const commands = createRuntimeBrowserCommands(host) + expect(runtimeBrowserCommandsFactoryIsHeadless()).toBe(false) + expect(() => commands.browserTabList({})).toThrow(/unavailable/) + pending.resolve(provider) + await startup.ready + expect(runtimeBrowserCommandsFactoryIsHeadless()).toBe(true) + commands.browserTabList({}) + commands.browserTabList({}) + expect(provider.factory).toHaveBeenCalledTimes(1) + expect(command).toHaveBeenCalledTimes(2) + await startup.stop() + expect(() => commands.browserTabList({})).toThrow(/unavailable/) + expect(runtimeBrowserCommandsFactoryIsHeadless()).toBe(false) +}) + +it('aborts pending discovery and awaits a late provider cleanup exactly once', async () => { + const { pending, provider, startup } = delayedProvider() + await Promise.resolve() + const signal = vi.mocked(resolveOrcadBrowserProvider).mock.calls.at(-1)![0].signal! + const stopped = startup.stop() + expect(startup.stop()).toBe(stopped) + expect(signal.aborted).toBe(true) + pending.resolve(provider) + await stopped + expect(provider.stop).toHaveBeenCalledTimes(1) + expect(runtimeBrowserCommandsFactoryIsHeadless()).toBe(false) +}) + +it('retains specific provider-unavailable diagnostics after discovery declines', async () => { + const { pending, startup } = delayedProvider() + setRuntimeBrowserUnavailableCause({ reason: 'driver_missing' }) + pending.resolve(null) + await startup.ready + expect(runtimeBrowserUnavailableCause()).toEqual({ reason: 'driver_missing' }) + await startup.stop() +}) + +it('does not hide resolver cleanup failures from the runtime lifetime', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { pending, startup } = delayedProvider() + pending.reject(new Error('cleanup failed')) + await startup.ready + await expect(startup.stop()).rejects.toThrow('cleanup failed') +}) + +it('propagates ready-provider cleanup failures', async () => { + const { pending, provider, startup } = delayedProvider() + vi.mocked(provider.stop).mockRejectedValueOnce(new Error('stop failed')) + pending.resolve(provider) + await startup.ready + await expect(startup.stop()).rejects.toThrow('stop failed') +}) + +it('refuses a member the provider does not implement instead of invoking it', async () => { + const { pending, provider, startup } = delayedProvider() + const commands = createRuntimeBrowserCommands(host) + pending.resolve(provider) + await startup.ready + // The fixture provider implements only browserTabList. + expect(() => commands.browserTabCreate({ worktree: 'wt-a' })).toThrow('Unknown browser command') + await startup.stop() +}) diff --git a/src/main/orcad/orcad-browser-startup.ts b/src/main/orcad/orcad-browser-startup.ts new file mode 100644 index 00000000000..fb7dc03a0ab --- /dev/null +++ b/src/main/orcad/orcad-browser-startup.ts @@ -0,0 +1,95 @@ +import { BrowserError } from '../browser/browser-error' +import { BROWSER_UNAVAILABLE_ERROR_CODE } from '../../shared/runtime-types' +import type { RuntimeBrowserCommands } from '../runtime/orca-runtime-browser' +import { setRuntimeBrowserCommandsFactory } from '../runtime/runtime-browser-commands-factory' +import { + resolveOrcadBrowserProvider, + type OrcadBrowserProvider, + type OrcadBrowserProviderOptions +} from './orcad-browser-provider' + +/** Browser discovery must not hold core RPC readiness hostage to a desktop authorization UI. */ +export function startOrcadBrowserProvider(options: OrcadBrowserProviderOptions): { + ready: Promise + stop(): Promise +} { + const controller = new AbortController() + let provider: OrcadBrowserProvider | null = null + let startupError: unknown + let stopping: Promise | undefined + setRuntimeBrowserCommandsFactory( + (host) => { + let commands: RuntimeBrowserCommands | undefined + // Every member resolves through the getter, so the target needs no members of its own. + const target: RuntimeBrowserCommands = Object.create(null) + return new Proxy(target, { + get: (_target, property) => { + if (property === 'then' || typeof property !== 'string') { + return undefined + } + return (...args: unknown[]) => { + if (controller.signal.aborted || !provider?.isAvailable()) { + throw new BrowserError( + BROWSER_UNAVAILABLE_ERROR_CODE, + 'Browser automation is unavailable on this host.' + ) + } + commands ??= provider.factory(host) + return callBrowserCommand(commands, property, args) + } + } + }) + }, + { headless: true, isAvailable: () => !controller.signal.aborted && !!provider?.isAvailable() } + ) + const ready = Promise.resolve() + .then(() => resolveOrcadBrowserProvider({ ...options, signal: controller.signal })) + .then( + (resolved) => { + provider = resolved + if (!resolved && !controller.signal.aborted) { + setRuntimeBrowserCommandsFactory(null) + } + }, + (error: unknown) => { + startupError = error + if (!controller.signal.aborted) { + setRuntimeBrowserCommandsFactory(null) + console.warn('[orcad] Browser startup failed:', error) + } + } + ) + return { + ready, + stop: () => { + controller.abort() + stopping ??= ready.then(async () => { + await provider?.stop() + // Unexpected resolver errors may include failed cleanup of a partially started provider. + if (startupError) { + throw startupError + } + }) + return stopping + } + } +} + +type BrowserCommandMember = (this: RuntimeBrowserCommands, ...args: unknown[]) => unknown + +function isBrowserCommandMember(value: unknown): value is BrowserCommandMember { + return typeof value === 'function' +} + +/** The proxy forwards by name; anything that is not a command method is refused, not invoked. */ +function callBrowserCommand( + commands: RuntimeBrowserCommands, + name: string, + args: unknown[] +): unknown { + const member: unknown = name in commands ? commands[name] : undefined + if (!isBrowserCommandMember(member)) { + throw new BrowserError(BROWSER_UNAVAILABLE_ERROR_CODE, `Unknown browser command: ${name}`) + } + return member.call(commands, ...args) +} diff --git a/src/main/orcad/orcad-bundled-runtime.test.ts b/src/main/orcad/orcad-bundled-runtime.test.ts index e4328fb833b..9fe9b96afb7 100644 --- a/src/main/orcad/orcad-bundled-runtime.test.ts +++ b/src/main/orcad/orcad-bundled-runtime.test.ts @@ -2,7 +2,11 @@ import { EventEmitter } from 'node:events' import { join, resolve } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { handoffToBundledOrcad, resolveBundledOrcadSlot } from './orcad-bundled-runtime' -import { NODE_RUNTIME_ASSETS, NODE_RUNTIME_PIN } from '../../shared/node-runtime-pin' +import { + NODE_RUNTIME_ASSETS, + NODE_RUNTIME_COMPAT_ASSETS, + NODE_RUNTIME_PIN +} from '../../shared/node-runtime-pin' import { ORCAD_NODE_RUNTIME_MARKER_FILENAME, ORCAD_SERVER_TARGET_FILENAME, @@ -84,6 +88,26 @@ describe('bundled Orca runtime handoff', () => { expect(fixture.spawn).not.toHaveBeenCalled() }) + it('hands a glibc 2.17 compat slot to the compat runtime it names', () => { + const compatSha = NODE_RUNTIME_COMPAT_ASSETS['linux-x64-glibc217'].executableSha256 + fixture.read.mockImplementation((path) => + path.endsWith(ORCAD_SERVER_TARGET_FILENAME) ? 'linux-x64-glibc217\n' : `${compatSha}\n` + ) + expect(handoffToBundledOrcad()).toBe(true) + expect(fixture.spawn).toHaveBeenCalledWith( + expect.objectContaining({ + program: join('/slot', '..', 'runtimes', `node-${compatSha}`, 'bin', 'node') + }) + ) + }) + + it('refuses a compat slot that names the default runtime', () => { + fixture.read.mockImplementation((path) => + path.endsWith(ORCAD_SERVER_TARGET_FILENAME) ? 'linux-x64-glibc217\n' : `${SHA}\n` + ) + expect(() => handoffToBundledOrcad()).toThrow(`does not name Node ${NODE_RUNTIME_PIN.version}`) + }) + it('refuses a slot without its runtime reference', () => { fixture.exists.mockImplementation((path) => !path.endsWith(ORCAD_NODE_RUNTIME_MARKER_FILENAME)) expect(() => handoffToBundledOrcad()).toThrow('bundled Orca runtime reference is missing') diff --git a/src/main/orcad/orcad-bundled-runtime.ts b/src/main/orcad/orcad-bundled-runtime.ts index 230cc5f326f..497e8a1120f 100644 --- a/src/main/orcad/orcad-bundled-runtime.ts +++ b/src/main/orcad/orcad-bundled-runtime.ts @@ -9,20 +9,11 @@ import { ORCAD_VERSION_FILENAME, orcadNodeRuntimeRelativePath } from '../../shared/orcad-artifacts' -import { - NODE_RUNTIME_ASSETS, - NODE_RUNTIME_PIN, - SERVER_TARGETS, - type ServerTarget -} from '../../shared/node-runtime-pin' +import { NODE_RUNTIME_PIN, nodeRuntimeAsset } from '../../shared/node-runtime-pin' export class OrcadBundledRuntimeError extends Error {} export const ORCAD_BUNDLED_LAUNCHER_ENV = 'ORCA_BUNDLED_LAUNCHER_CHANNEL' -function isServerTarget(value: string): value is ServerTarget { - return SERVER_TARGETS.some((target) => target === value) -} - /** * The pinned Node a packaged slot in `directory` runs on, or null for an unpackaged entry. * Throws when the slot is packaged but its runtime reference is torn or foreign. @@ -42,13 +33,15 @@ export function resolveBundledOrcadRuntime(directory: string): string | null { throw new OrcadBundledRuntimeError('The bundled Orca runtime reference is missing') } const target = readFileSync(targetPath, 'utf8').trim() - if (!isServerTarget(target)) { + // A compat target (design D6 rung B) runs the same Node version built for an older glibc. + const asset = nodeRuntimeAsset(target) + if (!asset) { throw new OrcadBundledRuntimeError(`The bundled Orca runtime target is invalid: ${target}`) } const executableSha256 = readFileSync(markerPath, 'utf8').trim() // Why the pin and not only a digest shape: the marker becomes a path segment, and a slot // naming another runtime was not built by this code. - if (executableSha256 !== NODE_RUNTIME_ASSETS[target].executableSha256) { + if (executableSha256 !== asset.executableSha256) { throw new OrcadBundledRuntimeError( `The bundled Orca runtime reference does not name Node ${NODE_RUNTIME_PIN.version}` ) diff --git a/src/main/orcad/orcad-command-arguments.ts b/src/main/orcad/orcad-command-arguments.ts index f4fd748de61..8077b2cb898 100644 --- a/src/main/orcad/orcad-command-arguments.ts +++ b/src/main/orcad/orcad-command-arguments.ts @@ -21,6 +21,17 @@ export function parseArgs(argv: string[]): OrcadOptions { options.json = true } else if (arg === '--no-pairing') { options.noPairing = true + } else if (arg === '--mobile-pairing') { + options.mobilePairing = true + } else if (arg === '--recipe-json') { + options.recipeJson = true + } else if (arg === '--project-root') { + const value = argv[i + 1] + if (!value) { + throw new Error('--project-root expects a value') + } + options.projectRoot = value + i += 1 } else if (arg === '--bind') { const value = argv[i + 1] if (value === undefined) { @@ -39,5 +50,8 @@ export function parseArgs(argv: string[]): OrcadOptions { throw new Error(`Unknown argument: ${arg}`) } } + if (options.recipeJson && !options.projectRoot) { + throw new Error('--recipe-json requires --project-root') + } return options } diff --git a/src/main/orcad/orcad-completed-stop-receipt.ts b/src/main/orcad/orcad-completed-stop-receipt.ts new file mode 100644 index 00000000000..7f155df696b --- /dev/null +++ b/src/main/orcad/orcad-completed-stop-receipt.ts @@ -0,0 +1,110 @@ +/** Durable outcomes of a managed stop, in the data root beside the instance lock. */ +import { lstatSync } from 'node:fs' +import { dirname, join } from 'node:path' +import type { z } from 'zod' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { hasErrorCode } from '../daemon/daemon-process-inspection' +import { + ORCAD_STOP_RECEIPTS_DIRNAME, + OrcadCompletedStopReceiptSchema, + OrcadManagedStopRequestSchema, + OrcadDaemonRetirementRecordSchema, + type OrcadCompletedStopReceipt, + type OrcadDaemonRetirementRecord, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' + +const RECEIPT_MAX_BYTES = 64 * 1024 + +export type ReceiptKind = 'completed' | 'retirement' | 'decision' + +export function orcadStopReceiptPath(request: OrcadManagedStopRequest, kind: ReceiptKind): string { + const { transactionId, instance } = OrcadManagedStopRequestSchema.parse(request) + const suffix = kind === 'completed' ? '' : `.${kind}` + return join( + dirname(instance.lockPath), + ORCAD_STOP_RECEIPTS_DIRNAME, + `${transactionId}${suffix}.json` + ) +} + +/** `null` when absent; a receipt for a different request throws rather than reading as absent. */ +export function readOrcadStopReceipt( + request: OrcadManagedStopRequest, + kind: ReceiptKind, + schema: z.ZodType +): T | null { + const path = orcadStopReceiptPath(request, kind) + try { + if (!lstatSync(path).isFile()) { + throw new Error('orcad_stop_receipt_unverifiable') + } + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + return null + } + throw error + } + const receipt = schema.parse( + JSON.parse(readNodeFileSyncWithinLimit(path, RECEIPT_MAX_BYTES).buffer.toString('utf8')) + ) + const expected = OrcadManagedStopRequestSchema.parse(request) + if (JSON.stringify(receipt.request) !== JSON.stringify(expected)) { + throw new Error('orcad_stop_receipt_mismatch') + } + return receipt +} + +function writeReceipt(request: OrcadManagedStopRequest, kind: ReceiptKind, receipt: unknown): void { + if (!writeDurableSecureJsonFile(orcadStopReceiptPath(request, kind), receipt)) { + throw new Error('orcad_stop_receipt_permissions_unconfirmed') + } +} + +export function readOrcadCompletedStopReceipt( + request: OrcadManagedStopRequest +): OrcadCompletedStopReceipt | null { + return readOrcadStopReceipt(request, 'completed', OrcadCompletedStopReceiptSchema) +} + +/** + * Called only after exit is proven; rewriting an existing receipt re-runs its fsync. A retiring + * request whose orcad left no retirement record reports `unverifiable`, never `retired`. + */ +export function persistOrcadCompletedStopReceipt( + request: OrcadManagedStopRequest, + exitedAt: Date +): OrcadCompletedStopReceipt { + const parsed = OrcadManagedStopRequestSchema.parse(request) + const retirement = parsed.retireIdleDaemon + ? (readOrcadDaemonRetirementRecord(parsed)?.retirement ?? 'unverifiable') + : undefined + const receipt = readOrcadCompletedStopReceipt(parsed) ?? { + schemaVersion: 1 as const, + kind: 'orcad_managed_stop_completed' as const, + request: parsed, + exitedAt: exitedAt.toISOString(), + ...(retirement ? { retirement } : {}) + } + writeReceipt(parsed, 'completed', receipt) + return receipt +} + +export function readOrcadDaemonRetirementRecord( + request: OrcadManagedStopRequest +): OrcadDaemonRetirementRecord | null { + return readOrcadStopReceipt(request, 'retirement', OrcadDaemonRetirementRecordSchema) +} + +export function persistOrcadDaemonRetirementRecord( + request: OrcadManagedStopRequest, + outcome: Pick +): void { + writeReceipt(request, 'retirement', { + schemaVersion: 1, + kind: 'orcad_managed_stop_retirement', + request: OrcadManagedStopRequestSchema.parse(request), + ...outcome + }) +} diff --git a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts index 09cc44ed3b0..8e3e2d3be9a 100644 --- a/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts +++ b/src/main/orcad/orcad-cross-runtime-daemon-adoption.integration.test.ts @@ -28,8 +28,11 @@ import { orcadBunRuntimeFilename, orcadNodeRuntimeRelativePath } from '../../shared/orcad-artifacts' -import { removeTreeSync } from '../../shared/windows-transient-lock-removal' -import { readDaemonPidRecord } from '../daemon/daemon-endpoint-incarnation' +import { + killAndAwaitExit, + killProfileDaemons, + removeTestRoot +} from './orcad-daemon-teardown-fixture' import { PROTOCOL_VERSION } from '../daemon/types' import type { ServeReadiness } from '../server/serve-readiness' import { @@ -176,7 +179,8 @@ async function launch(slot: Slot, userDataDir: string): Promise fullVersion: slot.version, userDataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(userDataDir, '.orcad-activation-transaction') }) ) ).trim() @@ -256,13 +260,10 @@ async function backUpProfile(slot: Slot, runtime: string, userDataDir: string): expect(JSON.parse(result.stdout.trim().split('\n').at(-1) ?? '')).toEqual({ ok: true }) } -function killLaunched(): void { - for (const pid of launched) { - try { - process.kill(pid, 'SIGKILL') - } catch {} - } +async function killLaunched(): Promise { + const pids = [...launched] launched.clear() + await killAndAwaitExit(pids) } /** The protocol the Bun slot's own daemon reports, read from a throwaway launch. */ @@ -276,17 +277,9 @@ async function probeBunDaemonProtocol(): Promise { return daemon.protocolVersion! } finally { await stop(slot).catch(() => {}) - killLaunched() + await killLaunched() // Even after a failed launch: the daemon outlives orcad, and only its pid file names it. - const daemonDir = join(userDataDir, 'daemon') - for (const name of existsSync(daemonDir) ? readdirSync(daemonDir) : []) { - const pid = /^daemon-v\d+\.pid$/.test(name) - ? readDaemonPidRecord(join(daemonDir, name))?.pid - : undefined - if (pid && isAlive(pid)) { - process.kill(pid, 'SIGKILL') - } - } + await killProfileDaemons(userDataDir) } } @@ -362,8 +355,8 @@ worker.on('error', (error) => { console.error(error); process.exitCode = 1 }) afterEach(killLaunched) - afterAll(() => { - removeTreeSync(root) + afterAll(async () => { + await removeTestRoot(root) }) it.for([ @@ -459,8 +452,8 @@ worker.on('error', (error) => { console.error(error); process.exitCode = 1 }) await daemonClient('kill', userDataDir, sessionId, '') await vi.waitFor(() => expect(isAlive(created.pid)).toBe(false), { timeout: 10_000 }) } finally { - if (daemonPid && isAlive(daemonPid)) { - process.kill(daemonPid, 'SIGKILL') + if (daemonPid) { + await killAndAwaitExit([daemonPid]) } } } diff --git a/src/main/orcad/orcad-daemon-retirement.test.ts b/src/main/orcad/orcad-daemon-retirement.test.ts new file mode 100644 index 00000000000..3c952cf9b6a --- /dev/null +++ b/src/main/orcad/orcad-daemon-retirement.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../daemon/daemon-init', () => ({ + requestIdleDaemonRetirement: vi.fn(), + listLiveDaemonSessions: vi.fn(), + releaseDaemonRetirementFence: vi.fn() +})) + +import type { DaemonIdleRetirementResult } from '../daemon/daemon-pty-runtime-state' +import { retireOrcadDaemonIfIdle } from './orcad-daemon-retirement' + +function ports( + result: () => Promise, + liveSessions: number | null = null +) { + return { + request: vi.fn(result), + releaseFence: vi.fn(), + countLiveSessions: vi.fn(async () => liveSessions), + timeoutMs: 20 + } +} + +describe('best-effort daemon retirement', () => { + it('reports retired only when the daemon accepted, and keeps its fence', async () => { + const retiring = ports(async () => ({ state: 'retiring' })) + expect(await retireOrcadDaemonIfIdle(retiring)).toMatchObject({ retirement: 'retired' }) + expect(retiring.releaseFence).not.toHaveBeenCalled() + }) + + it('leaves a busy daemon running, reopens admission, and reports live', async () => { + const busy = ports(async () => ({ state: 'busy', liveSessions: 3 })) + expect(await retireOrcadDaemonIfIdle(busy)).toMatchObject({ + retirement: 'live', + liveSessions: 3 + }) + expect(busy.releaseFence).toHaveBeenCalledOnce() + }) + + it('counts sessions itself when the daemon did not say how many', async () => { + const busy = ports(async () => ({ state: 'busy', liveSessions: null }), 1) + expect(await retireOrcadDaemonIfIdle(busy)).toMatchObject({ + retirement: 'live', + liveSessions: 1 + }) + }) + + it.each([ + [ + 'an unsupported daemon', + async (): Promise => ({ state: 'unsupported' }) + ], + [ + 'an unverifiable census', + async (): Promise => ({ state: 'unverifiable' }) + ], + [ + 'lost contact', + async (): Promise => { + throw new Error('socket closed') + } + ], + ['no answer', () => new Promise(() => {})] + ])('never reads %s as idle, and reopens admission', async (_name, result) => { + const unanswered = ports(result) + expect(await retireOrcadDaemonIfIdle(unanswered)).toMatchObject({ + retirement: 'unverifiable', + liveSessions: null + }) + expect(unanswered.releaseFence).toHaveBeenCalledOnce() + }) + + it('reopens admission again once a timed-out attempt is refused late', async () => { + let answer: (result: DaemonIdleRetirementResult) => void = () => {} + const late = ports(() => new Promise((resolve) => (answer = resolve))) + await retireOrcadDaemonIfIdle(late) + expect(late.releaseFence).toHaveBeenCalledOnce() + answer({ state: 'busy', liveSessions: 1 }) + await vi.waitFor(() => expect(late.releaseFence).toHaveBeenCalledTimes(2)) + }) + + it('keeps the fence when a timed-out attempt turns out to retire the daemon', async () => { + let answer: (result: DaemonIdleRetirementResult) => void = () => {} + const late = ports(() => new Promise((resolve) => (answer = resolve))) + await retireOrcadDaemonIfIdle(late) + answer({ state: 'retiring' }) + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(late.releaseFence).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/orcad/orcad-daemon-retirement.ts b/src/main/orcad/orcad-daemon-retirement.ts new file mode 100644 index 00000000000..15d90d9e60f --- /dev/null +++ b/src/main/orcad/orcad-daemon-retirement.ts @@ -0,0 +1,97 @@ +/** + * Best-effort retirement of the terminal daemon when a managed stop asks for it. + * + * The daemon normally outlives orcad so terminals survive a restart (D7). Retirement happens + * only when the daemon itself proves it owns no live session across every generation; a busy + * or unanswering daemon stays up, and orcad's own stop never waits on that outcome. + */ +import { + listLiveDaemonSessions, + releaseDaemonRetirementFence, + requestIdleDaemonRetirement +} from '../daemon/daemon-init' +import type { OrcadDaemonRetirementVerdict } from '../../shared/orcad-stop-request' +import { ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS } from './orcad-stop-deadlines' + +export type OrcadDaemonRetirement = { + retirement: OrcadDaemonRetirementVerdict + liveSessions: number | null + reason: string | null +} + +/** Live sessions across every daemon generation, or `null` when any could not answer. */ +export async function countLiveOrcadDaemonSessions(): Promise { + try { + // The inventory lists live sessions only. + return (await listLiveDaemonSessions())?.length ?? null + } catch { + return null + } +} + +type RetirementPorts = { + request: typeof requestIdleDaemonRetirement + releaseFence: typeof releaseDaemonRetirementFence + countLiveSessions: typeof countLiveOrcadDaemonSessions + timeoutMs: number +} + +const DEFAULT_PORTS: RetirementPorts = { + request: requestIdleDaemonRetirement, + releaseFence: releaseDaemonRetirementFence, + countLiveSessions: countLiveOrcadDaemonSessions, + timeoutMs: ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS +} + +export async function retireOrcadDaemonIfIdle( + ports: Partial = {} +): Promise { + const { request, releaseFence, countLiveSessions, timeoutMs } = { ...DEFAULT_PORTS, ...ports } + const attempt = request().catch(() => ({ state: 'unverifiable' as const })) + const result = await withTimeout(attempt, timeoutMs, { state: 'timed-out' as const }) + if (result.state === 'timed-out') { + // The fence cannot reopen while the attempt is pending; reopen it once a late refusal lands. + void attempt.then((late) => late.state !== 'retiring' && releaseFence()) + } + if (result.state === 'retiring') { + return { retirement: 'retired', liveSessions: 0, reason: null } + } + // A daemon that stays must not be left refusing new terminals. + releaseFence() + const liveSessions = + result.state === 'busy' && result.liveSessions !== null + ? result.liveSessions + : await withTimeout(countLiveSessions(), timeoutMs, null) + if (liveSessions !== null && liveSessions > 0) { + return { + retirement: 'live', + liveSessions, + reason: + `${liveSessions} terminal ${liveSessions === 1 ? 'session is' : 'sessions are'} still ` + + 'live, so the daemon stays up and keeps them.' + } + } + return { + retirement: 'unverifiable', + liveSessions, + reason: + result.state === 'unsupported' + ? 'The terminal daemon predates idle retirement, so it was left running.' + : 'The host could not prove the daemon idle, so it was left running.' + } +} + +/** No answer within the bound is `fallback`; the caller treats it as unverifiable. */ +async function withTimeout(work: Promise, timeoutMs: number, fallback: F): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + work, + new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), timeoutMs) + }) + ]) + } finally { + clearTimeout(timer) + } +} diff --git a/src/main/orcad/orcad-daemon-session-client-fixture.ts b/src/main/orcad/orcad-daemon-session-client-fixture.ts new file mode 100644 index 00000000000..b9e058a091f --- /dev/null +++ b/src/main/orcad/orcad-daemon-session-client-fixture.ts @@ -0,0 +1,53 @@ +// Test fixture: a terminal client that talks to a profile's terminal daemon directly, the way a +// paired client's terminal does, so a test can prove a session outlives a serve-host switch. +import { build } from 'esbuild' + +export type DaemonSessionClientResult = { pid: number; isReattach: boolean; output: boolean } + +/** Bundles the client to `outfile`; run it with Node as ` `. */ +export async function buildDaemonSessionClient(outfile: string): Promise { + await build({ + stdin: { + contents: ` + import { DaemonPtyAdapter } from './src/main/daemon/daemon-pty-adapter' + import { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from './src/main/daemon/daemon-spawner' + import { setAppEnvironment } from './src/shared/app-environment' + const [op, runtimeDir, sessionId, marker, cwd] = process.argv.slice(2) + setAppEnvironment({ + getPath: () => cwd, getAppPath: () => cwd, getVersion: () => 'test', isPackaged: () => true, + onWillQuit() {}, exit: code => process.exit(code), getAppMetrics: () => [] + }) + const adapter = new DaemonPtyAdapter({ + socketPath: getDaemonSocketPath(runtimeDir), tokenPath: getDaemonTokenPath(runtimeDir), + pidPath: getDaemonPidPath(runtimeDir), profileScope: runtimeDir, runtimeDir + }) + let output = '' + adapter.onData(event => { if (event.id === sessionId) output += event.data }) + const deadline = setTimeout(() => { console.error('timed out; output: ' + output); process.exit(98) }, 20_000) + ;(async () => { + const win32 = process.platform === 'win32' + const spawned = await adapter.spawn(op === 'create' + ? { sessionId, cols: 80, rows: 24, cwd, shellOverride: win32 ? 'cmd.exe' : '/bin/sh' } + : { sessionId, cols: 80, rows: 24 }) + // Both shells echo what is typed, so the typed form must not already read as the marker. + const typed = win32 ? 'echo ORCA_^SERVE_' + marker : "printf 'ORCA_SERVE_%s\\\\n' " + marker + adapter.write(spawned.id, typed + "\\r") + while (!output.includes('ORCA_SERVE_' + marker)) await new Promise(r => setTimeout(r, 50)) + clearTimeout(deadline) + await adapter.disconnectOnly() + console.log(JSON.stringify({ pid: spawned.pid, isReattach: spawned.isReattach === true, output: true })) + process.exit(0) + })().catch(error => { console.error(error); process.exit(1) }) + `, + resolveDir: process.cwd(), + loader: 'ts' + }, + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node18', + external: ['electron', 'node-pty', '@parcel/watcher', '*.node'], + outfile, + logLevel: 'silent' + }) +} diff --git a/src/main/orcad/orcad-daemon-slot-pins.ts b/src/main/orcad/orcad-daemon-slot-pins.ts new file mode 100644 index 00000000000..f3afc10c1c9 --- /dev/null +++ b/src/main/orcad/orcad-daemon-slot-pins.ts @@ -0,0 +1,55 @@ +/** + * Slots a live terminal daemon still runs from. Local `orca serve` forks its daemon from its own + * slot, and that daemon outlives orcad, so the instance lock alone stops naming the slot. + */ +import { readdirSync, readFileSync } from 'node:fs' +import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { parseDaemonPidFile } from '../daemon/daemon-pid-file-parse' +import { hasErrorCode, isProcessAlive } from '../daemon/daemon-process-inspection' + +const DAEMON_PID_FILE = /^daemon-v\d+\.pid$/u + +/** Slot names under `cacheRoot` holding a live daemon's entry or runtime; null when unprovable. */ +export function liveDaemonOrcadSlots( + userDataPath: string, + cacheRoot: string, + isAlive: (pid: number) => boolean = isProcessAlive +): string[] | null { + const runtimeDir = join(userDataPath, 'daemon') + let names: string[] + try { + names = readdirSync(runtimeDir) + } catch (error) { + // No daemon directory means no daemon; anything else leaves the slots unprovable. + return hasErrorCode(error, 'ENOENT') ? [] : null + } + const slots: string[] = [] + for (const name of names.filter((candidate) => DAEMON_PID_FILE.test(candidate))) { + let record + try { + record = parseDaemonPidFile(readFileSync(join(runtimeDir, name), 'utf8')) + } catch { + return null + } + if (!record || !Number.isInteger(record.pid) || record.pid <= 0) { + // A record being written or torn names no process we could rule out. + return null + } + if (!isAlive(record.pid)) { + continue + } + for (const path of [record.entryPath, record.spawnerExecPath]) { + const slot = path ? slotContaining(cacheRoot, path) : null + if (slot) { + slots.push(slot) + } + } + } + return slots +} + +function slotContaining(cacheRoot: string, path: string): string | null { + const rel = relative(resolve(cacheRoot), resolve(path)) + const [target, slot] = rel.split(sep) + return rel && !isAbsolute(rel) && target !== '..' && target && slot ? slot : null +} diff --git a/src/main/orcad/orcad-daemon-supervision.test.ts b/src/main/orcad/orcad-daemon-supervision.test.ts index 19616c9d5ef..ba5cd989ad9 100644 --- a/src/main/orcad/orcad-daemon-supervision.test.ts +++ b/src/main/orcad/orcad-daemon-supervision.test.ts @@ -56,6 +56,20 @@ describe('startOrcadDaemon', () => { }) }) + it('gives a cold Windows daemon a longer startup budget', async () => { + await startOrcadDaemon('win32') + expect(initDaemonPtyProviderMock).toHaveBeenCalledWith(undefined, { + macosLoginSessionWatch: false, + startupTimeoutMs: 30_000 + }) + }) + + it('retries the daemon spawn once before falling back to in-process terminals', async () => { + initDaemonPtyProviderMock.mockRejectedValueOnce(new Error('Daemon startup timed out')) + await expect(startOrcadDaemon()).resolves.toEqual({ state: 'live', pid: 4242 }) + expect(initDaemonPtyProviderMock).toHaveBeenCalledTimes(2) + }) + it('reports degraded when fresh terminals would fall back to the local provider', async () => { daemonOwnsFreshPersistentPtysMock.mockReturnValue(false) const result = await startOrcadDaemon() @@ -71,6 +85,7 @@ describe('startOrcadDaemon', () => { state: 'unavailable', reason: 'node-pty is missing' }) + expect(initDaemonPtyProviderMock).toHaveBeenCalledTimes(2) }) }) diff --git a/src/main/orcad/orcad-daemon-supervision.ts b/src/main/orcad/orcad-daemon-supervision.ts index d35b03048d3..d0fa08cfbe9 100644 --- a/src/main/orcad/orcad-daemon-supervision.ts +++ b/src/main/orcad/orcad-daemon-supervision.ts @@ -18,6 +18,12 @@ import { readDaemonPidRecord } from '../daemon/daemon-init' +const WINDOWS_DAEMON_STARTUP_TIMEOUT_MS = 30_000 + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + export type OrcadDaemonStartup = | { state: 'live'; pid: number | null } | { state: 'degraded'; reason: string } @@ -31,13 +37,25 @@ export type OrcadDaemonStartup = * — `daemonOwnsFreshPersistentPtys()` is what the runtime reads for that, and it answers * false here without any extra bookkeeping. */ -export async function startOrcadDaemon(): Promise { +export async function startOrcadDaemon( + platform: NodeJS.Platform = process.platform +): Promise { + // Why no login-session watch: that retires the daemon when the spawning macOS GUI login + // session dies. An orcad daemon must survive its SSH session ending — that is the point. + const policy = { + macosLoginSessionWatch: false, + // A freshly uploaded node.exe plus conpty can take well over 10 s on its first, AV-scanned exec. + ...(platform === 'win32' ? { startupTimeoutMs: WINDOWS_DAEMON_STARTUP_TIMEOUT_MS } : {}) + } try { - // Why no login-session watch: that retires the daemon when the spawning macOS GUI login - // session dies. An orcad daemon must survive its SSH session ending — that is the point. - await initDaemonPtyProvider(undefined, { macosLoginSessionWatch: false }) + await initDaemonPtyProvider(undefined, policy).catch((error: unknown) => { + // One warm retry: activation refuses a daemonless candidate, so a cold-start miss would + // otherwise leave a first deploy serving nothing. + console.warn(`[orcad] The terminal daemon did not start (${errorMessage(error)}); retrying`) + return initDaemonPtyProvider(undefined, policy) + }) } catch (error) { - const reason = error instanceof Error ? error.message : String(error) + const reason = errorMessage(error) console.error( `[orcad] The terminal daemon did not start: ${reason}\n` + '[orcad] Terminals will run in-process and WILL NOT survive an orcad restart.' diff --git a/src/main/orcad/orcad-daemon-teardown-fixture.ts b/src/main/orcad/orcad-daemon-teardown-fixture.ts new file mode 100644 index 00000000000..41798ac2ac9 --- /dev/null +++ b/src/main/orcad/orcad-daemon-teardown-fixture.ts @@ -0,0 +1,92 @@ +/** + * Teardown for integration tests whose orcad started a real terminal daemon in a temp profile. + * + * The daemon outlives orcad by design and only its pid record names it. Killing it is not + * enough: until it has exited it can still remove its socket, pid and token files and append + * to its log, so a recursive delete racing it fails with ENOTEMPTY (seen on macOS). + */ +import { existsSync, readdirSync } from 'node:fs' +import { join } from 'node:path' +import { removeTreeSync } from '../../shared/windows-transient-lock-removal' +import { readDaemonPidRecord } from '../daemon/daemon-endpoint-incarnation' +import type { spawnProcess } from '../../shared/child-process/run-process' + +const EXIT_WAIT_MS = 10_000 +const POLL_MS = 50 +const REMOVE_ATTEMPTS = 8 +const REMOVE_RETRY_MS = 150 + +const sleep = (ms: number): Promise => new Promise((settle) => setTimeout(settle, ms)) + +function errorCode(error: unknown): string | undefined { + return typeof error === 'object' && + error !== null && + 'code' in error && + typeof error.code === 'string' + ? error.code + : undefined +} + +function hasExited(pid: number): boolean { + try { + process.kill(pid, 0) + return false + } catch (error) { + // EPERM still proves the process exists. + return errorCode(error) === 'ESRCH' + } +} + +/** SIGKILLs each pid and waits, bounded, until the OS no longer knows it. */ +export async function killAndAwaitExit(pids: Iterable): Promise { + const pending = [...pids] + for (const pid of pending) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } + const deadline = Date.now() + EXIT_WAIT_MS + while (pending.some((pid) => !hasExited(pid)) && Date.now() < deadline) { + await sleep(POLL_MS) + } +} + +/** Kills a spawned child and waits for its exit event, so nothing it spawned races teardown. */ +export async function killChildAndWait(child: ReturnType): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return + } + const exited = new Promise((settle) => child.once('exit', settle)) + child.kill('SIGKILL') + await exited +} + +/** Kills every terminal daemon `/daemon` names and waits until each has exited. */ +export async function killProfileDaemons(userData: string): Promise { + const daemonDir = join(userData, 'daemon') + const pids = (existsSync(daemonDir) ? readdirSync(daemonDir) : []).flatMap((name) => { + const pid = /^daemon-v\d+\.pid$/u.test(name) + ? readDaemonPidRecord(join(daemonDir, name))?.pid + : undefined + return pid ? [pid] : [] + }) + await killAndAwaitExit(pids) +} + +/** Removes a test root, retrying a late writer's ENOTEMPTY/EBUSY on every platform. */ +export async function removeTestRoot(root: string): Promise { + for (let attempt = 1; ; attempt += 1) { + try { + removeTreeSync(root) + return + } catch (error) { + if ( + attempt >= REMOVE_ATTEMPTS || + !['ENOTEMPTY', 'EBUSY', 'EPERM'].includes(errorCode(error) ?? '') + ) { + throw error + } + await sleep(REMOVE_RETRY_MS) + } + } +} diff --git a/src/main/orcad/orcad-data-root-privacy.ts b/src/main/orcad/orcad-data-root-privacy.ts new file mode 100644 index 00000000000..a3f99fc7a3c --- /dev/null +++ b/src/main/orcad/orcad-data-root-privacy.ts @@ -0,0 +1,99 @@ +import { chmodSync, statSync } from 'node:fs' +import process from 'node:process' +import { restrictWindowsPathSync } from '../../shared/secure-path-windows-acl' + +export type OrcadInstanceLockCode = + | 'orcad_data_root_unusable' + | 'orcad_data_root_wrong_owner' + | 'orcad_data_root_shared' + | 'orcad_instance_lock_held' + | 'orcad_instance_lock_foreign_identity' + | 'orcad_instance_lock_unreadable' + +export class OrcadInstanceLockError extends Error { + constructor( + readonly code: OrcadInstanceLockCode, + message: string + ) { + super(message) + this.name = 'OrcadInstanceLockError' + } +} + +export type OrcadDataRootPrivacyHooks = { + platform?: NodeJS.Platform + /** Windows: restrict the data root's ACL to this user; false when it could not be applied. */ + restrictWindowsDataRoot?: (dataRoot: string) => boolean +} + +/** + * Fail closed on a data root other identities can read or write. + * + * Why self-heal first and refuse second: orcad stores credentials unsealed (there is no OS + * keyring on this host), so a group- or world-accessible root is a real exposure — but if + * we own the directory, tightening it is strictly better than refusing to start. We refuse + * only when the permissions are not ours to fix. + */ +export function assertOrcadDataRootIsPrivate( + dataRoot: string, + hooks: OrcadDataRootPrivacyHooks +): void { + // Windows ACLs are not expressible as a POSIX mode, and `statSync().mode` there reports a + // synthesized one, so Windows restricts and verifies the ACL instead (icacls, no PowerShell). + if ((hooks.platform ?? process.platform) === 'win32') { + const restrict = + hooks.restrictWindowsDataRoot ?? ((path: string) => restrictWindowsPathSync(path, true)) + if (!restrict(dataRoot)) { + throw new OrcadInstanceLockError( + 'orcad_data_root_shared', + `Could not restrict the orcad data root ${dataRoot} to this user. orcad stores ` + + 'credentials there unsealed, so it refuses to start. Point ORCA_USER_DATA at a ' + + 'directory this account owns.' + ) + } + return + } + let stats + try { + stats = statSync(dataRoot) + } catch (error) { + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot stat the orcad data root ${dataRoot}: ${error instanceof Error ? error.message : String(error)}` + ) + } + const uid = process.getuid?.() + if (uid !== undefined && stats.uid !== uid) { + throw new OrcadInstanceLockError( + 'orcad_data_root_wrong_owner', + `The orcad data root ${dataRoot} is owned by uid ${stats.uid}, not by uid ${uid} running ` + + 'this process. Give orcad its own data root (ORCA_USER_DATA) or chown this one.' + ) + } + if ((stats.mode & 0o077) === 0) { + return + } + try { + chmodSync(dataRoot, 0o700) + } catch { + // Fall through to the re-stat, which produces the actionable message. + } + let mode: number + try { + mode = statSync(dataRoot).mode + } catch (error) { + throw new OrcadInstanceLockError( + 'orcad_data_root_unusable', + `Cannot stat the orcad data root ${dataRoot}: ${error instanceof Error ? error.message : String(error)}` + ) + } + if ((mode & 0o077) !== 0) { + throw new OrcadInstanceLockError( + 'orcad_data_root_shared', + `The orcad data root ${dataRoot} is accessible to other users (mode ` + + `${(mode & 0o777).toString(8)}) and could not be tightened. orcad stores credentials ` + + 'there unsealed, so it refuses to start. Run `chmod 700` on it, or point ORCA_USER_DATA ' + + 'at a private directory.' + ) + } +} diff --git a/src/main/orcad/orcad-entry.test.ts b/src/main/orcad/orcad-entry.test.ts index d77d252375b..8d7df876b7d 100644 --- a/src/main/orcad/orcad-entry.test.ts +++ b/src/main/orcad/orcad-entry.test.ts @@ -21,9 +21,10 @@ describe('orcad profile-state shutdown', () => { vi.spyOn(console, 'error').mockImplementation(() => {}) const stop = vi.fn(() => new Promise(() => {})) try { - installOrcadShutdownSignals(stop) + const shutdown = installOrcadShutdownSignals(stop) signal?.() signal?.() + expect(shutdown('idle')).toBe(false) expect(stop).toHaveBeenCalledOnce() expect(exit).not.toHaveBeenCalled() expect(() => vi.advanceTimersByTime(ORCAD_SHUTDOWN_DEADLINE_MS)).toThrow('shutdown deadline') @@ -34,6 +35,28 @@ describe('orcad profile-state shutdown', () => { } }) + it('retracts a clean-stop record only when the stop fails', async () => { + vi.spyOn(process, 'on').mockImplementation(() => process) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stub only records the code; nothing after process.exit runs in these paths. + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) + vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + const onFailed = vi.fn() + installOrcadShutdownSignals(async () => { + throw new Error('flush failed') + })('idle', onFailed) + await vi.waitFor(() => expect(exit).toHaveBeenCalled()) + expect(onFailed).toHaveBeenCalledOnce() + + const notFailed = vi.fn() + installOrcadShutdownSignals(async () => {})('idle', notFailed) + await vi.waitFor(() => expect(exit).toHaveBeenLastCalledWith(0)) + expect(notFailed).not.toHaveBeenCalled() + } finally { + vi.restoreAllMocks() + } + }) + it('flushes durably before closing the profile store', async () => { const events: string[] = [] const store = { diff --git a/src/main/orcad/orcad-entry.ts b/src/main/orcad/orcad-entry.ts index 522ef82d742..b89c813774f 100644 --- a/src/main/orcad/orcad-entry.ts +++ b/src/main/orcad/orcad-entry.ts @@ -22,6 +22,12 @@ import { startOrcadWithHost } from './orcad-lifecycle' import { parseArgs } from './orcad-command-arguments' +import type { OrcadRuntimeCleanup } from './orcad-runtime-lifetime' +import { installOrcadStopRequestListeners } from './orcad-stop-request-listener' +import { prepareOrcadManagedStop } from './orcad-managed-stop-admission' +import type { OrcadManagedStopContext } from '../../shared/orcad-stop-request' +import { beginOrcadIdleExit, bindOrcadIdleShutdown } from './orcad-managed-idle-exit-host' +import { orcadAutomationsKeepHostBusy, startOrcadAutomations } from './orcad-automations' import { changedAiVaultSearchSettings, type AiVaultSearchSettings @@ -37,13 +43,18 @@ function createNodeAppEnvironment(): AppEnvironment { // The main signal handler awaits runtime and browser teardown before process.exit. // Keep will-quit callbacks synchronous, but never let them pre-empt that async barrier. runOrcadQuitHandlers = (): void => { + const errors: unknown[] = [] for (const handler of quitHandlers.splice(0)) { try { handler() } catch (error) { - console.error('[orcad] shutdown handler failed:', error) + errors.push(error) } } + // Why throw: a quit handler that failed may leave a writer running, which keeps the lock. + if (errors.length > 0) { + throw new AggregateError(errors, 'orcad_quit_handlers_failed') + } } return { getPath: resolveOrcadPath, @@ -91,12 +102,19 @@ export type OrcadOptions = { json?: boolean noPairing?: boolean pairingAddress?: string + /** Desktop `orca serve` parity: a mobile-scoped offer with a terminal QR. */ + mobilePairing?: boolean + /** Desktop `orca serve` parity: print only the ephemeral-VM recipe line. */ + recipeJson?: boolean + projectRoot?: string /** Literal IP to bind. Defaults to loopback; see orcad-bind-address.ts. */ bind?: string } export type OrcadHandle = { readiness: ServeReadiness + /** What an instance-bound stop request must name to stop this process. */ + managedStop: OrcadManagedStopContext stop(): Promise } @@ -107,21 +125,26 @@ export type OrcadHandle = { */ export async function startOrcad(options: OrcadOptions = {}): Promise { installOrcadHostAdapters() - return startOrcadWithHost( + const { readiness, instance, stop } = await startOrcadWithHost( resolveUserDataPath(), (registerCleanup) => startOrcadRuntime(options, registerCleanup), () => { - runOrcadQuitHandlers() - // Last, after every quit handler, so the spans they end still reach the file. - closeOrcadObservability() - closeOrcadObservability = () => {} + try { + runOrcadQuitHandlers() + } finally { + // Last, after every quit handler (even a failing one), so their spans still reach the file. + closeOrcadObservability() + closeOrcadObservability = () => {} + } } ) + const version = process.env.ORCA_VERSION ?? '0.0.0-orcad' + return { readiness, managedStop: { version, runtimeId: readiness.runtimeId, instance }, stop } } async function startOrcadRuntime( options: OrcadOptions, - registerCleanup: (cleanup: () => Promise) => void + registerCleanup: (cleanup: OrcadRuntimeCleanup) => void ): Promise> { const { OrcaRuntimeService } = await import('../runtime/orca-runtime') const { OrcaRuntimeRpcServer } = await import('../runtime/runtime-rpc') @@ -130,8 +153,9 @@ async function startOrcadRuntime( const { getAppEnvironment } = await import('../../shared/app-environment') const { installOrcadObservability } = await import('./orcad-observability') closeOrcadObservability = installOrcadObservability() - const { resolveAdvertisedPairingEndpoint } = await import('../runtime/pairing-endpoint') const { ServeReadinessPublisher } = await import('../server/serve-readiness') + const { assertServeProjectRoot } = await import('../server/serve-pairing-output') + const { buildOrcadServeReadiness } = await import('./orcad-serve-readiness') const { createOrcadProfileStateStartup } = await import('./orcad-profile-state-startup') const { startOrcadDaemon, stopOrcadDaemon } = await import('./orcad-daemon-supervision') const { daemonOwnsFreshPersistentPtys } = await import('../daemon/daemon-init') @@ -145,54 +169,40 @@ async function startOrcadRuntime( const { AgentStatusObservedPaneIdentities, AgentStatusObservedPaneIdentityCapture } = await import('../runtime/agent-status-observed-pane-identity') - let rpc: InstanceType | null = null + const { disposeWatcherProcessAndWait } = await import('../ipc/parcel-watcher-process') + let profileStoreForShutdown: | { flushFinalOrThrowAsync(): Promise; freezeWritesAsync(): Promise } | undefined let uninstallHookStatusRepublish = (): void => {} let uninstallObservedStatusIdentity = (): void => {} let removeStatusHookSettingsListener = (): void => {} + // Cleanups run in reverse: RPC, then recovery and watchers, then the final flush, then daemon. + registerCleanup(() => agentHookServer.stop()) + registerCleanup(() => uninstallHookStatusRepublish()) + registerCleanup(() => uninstallObservedStatusIdentity()) + registerCleanup(() => removeStatusHookSettingsListener()) + // Why disconnect and not shut down: the daemon must outlive this process, or an orcad + // restart goes back to killing every running terminal. + registerCleanup(() => stopOrcadDaemon()) registerCleanup(async () => { - try { - await rpc?.stop() - } finally { - try { - // Stop accepting RPC writes before the final persistence barrier. A SQLite-backed - // orcad has no JSON mirror to absorb a debounced write after SIGTERM. - if (profileStoreForShutdown) { - await flushOrcadProfileStoreForShutdown(profileStoreForShutdown) - } - } finally { - try { - // Why disconnect and not shut down: the daemon must outlive this process, or an - // orcad restart goes back to killing every running terminal. - await stopOrcadDaemon() - } finally { - removeStatusHookSettingsListener() - uninstallObservedStatusIdentity() - uninstallHookStatusRepublish() - agentHookServer.stop() - } - } + // A SQLite-backed orcad has no JSON mirror to absorb a debounced write after SIGTERM. + if (profileStoreForShutdown) { + await flushOrcadProfileStoreForShutdown(profileStoreForShutdown) } }) + // Watcher children outlive a disposal that does not wait for them. + registerCleanup(() => disposeWatcherProcessAndWait()) const { DesktopPushService } = await import('../runtime/push/desktop-push-service') const { resolvePushGatewayOrigin } = await import('../runtime/push/push-gateway-origin') const runtimeUserDataPath = getAppEnvironment().getPath('userData') + const idleExitStartup = beginOrcadIdleExit(runtimeUserDataPath) const { store: profileStore, authority: profileStateAuthority } = await createOrcadProfileStateStartup(runtimeUserDataPath) const observedPaneIdentities = new AgentStatusObservedPaneIdentities() const observedStatusCapture = new AgentStatusObservedPaneIdentityCapture(observedPaneIdentities) - // Why a real Store: without one every persistence-backed RPC throws `runtime_unavailable` - // and the read paths that use `this.store?.x ?? []` quietly answer "empty" instead — - // a server that pairs and lists nothing looks healthy and is not. - // Why: orcad IS the runtime authority — loading as 'desktop' would classify its - // own runtime-scheduled automations as ambiguous mirrors and orphan them. profileStoreForShutdown = profileStore - // Why: every SSH connect consults this sidecar. Left unbound it reports nothing trusted, - // which is safe but silently discards accept records on every launch. - uninstallObservedStatusIdentity = agentHookServer.subscribeEnrichedStatus((enriched) => observedStatusCapture.observe(enriched) ) @@ -318,11 +328,17 @@ async function startOrcadRuntime( await runtime.refreshRestoredOrchestrationAuthority() await runtime.reconcileLegacyWorkerTerminals() + // A retry armed during recovery would otherwise write after the final profile flush. + registerCleanup(() => runtime.stopLegacyWorkerTerminalRecovery()) + // Recovery binds terminal and dispatch identities; only now can startup observations be fenced. observedStatusCapture.attach(runtime) + // Why before the RPC server binds: like `--serve`, the first client must find a ready graph. + const { publishHeadlessRuntimeGraph } = await import('../runtime/headless-runtime-graph') + publishHeadlessRuntimeGraph(runtime) const bindHost = resolveOrcadBindHost(options.bind) - rpc = new OrcaRuntimeRpcServer({ + const rpc = new OrcaRuntimeRpcServer({ runtime, userDataPath: runtimeUserDataPath, enableWebSocket: true, @@ -333,7 +349,10 @@ async function startOrcadRuntime( pinnedBindHost: bindHost, ...(options.port !== undefined ? { wsPort: options.port, preferPinnedWsPort: true } : {}) }) + // Stops first: no RPC may write while the rest of the runtime is torn down. + registerCleanup(() => rpc.stop()) await rpc.start() + startOrcadAutomations(runtime, profileStore, registerCleanup) const pushService = DesktopPushService.create({ runtime, runtimeRpc: rpc, @@ -343,50 +362,32 @@ async function startOrcadRuntime( getAppEnvironment().onWillQuit(() => pushService?.stop()) console.error(`[orcad] ${describeOrcadBindExposure(bindHost)}`) - const boundEndpoint = rpc.getWebSocketEndpoint() - const advertised = boundEndpoint - ? resolveAdvertisedPairingEndpoint(boundEndpoint, options.pairingAddress) - : null - const offer = options.noPairing - ? ({ - available: false, - reason: 'disabled_by_operator', - guidance: 'Restart without --no-pairing to create a client pairing offer.' - } as const) - : rpc.createPairingOffer({ - address: options.pairingAddress, - name: `CLI ${new Date().toLocaleDateString()}`, - scope: 'runtime' - }) - - const readiness: ServeReadiness = { + const readiness = await buildOrcadServeReadiness({ + options, runtimeId: runtime.getRuntimeId(), - boundEndpoint, - advertisedEndpoint: advertised?.ok ? advertised.endpoint : null, - // Why 'settled': the WSL CLI reconciliation barrier is a desktop-launch concern. - // orcad never runs it, so there is no pending repair a client could race. - managedWslCliReconciliation: 'settled', - pairing: offer.available - ? { - available: true, - url: offer.pairingUrl, - endpoint: offer.endpoint, - deviceId: offer.deviceId, - webClientUrl: offer.webClientUrl, - scope: 'runtime', - qr: null - } - : offer, - // Why in the readiness payload: this is the one message a supervisor and a deploy - // transaction both read, and a green orcad with a dead daemon is exactly the - // looks-healthy-but-useless state they must not activate. - health: await collectOrcadHealth(getAppEnvironment().getVersion(), profileStateAuthority) - } - - await new ServeReadinessPublisher().publish(readiness, { - mode: options.json ? 'json' : 'human' + rpc, + collectHealth: () => + collectOrcadHealth( + getAppEnvironment().getVersion(), + profileStateAuthority, + idleExitStartup.previousIdleStop + ) }) + await new ServeReadinessPublisher().publish( + readiness, + options.recipeJson && options.projectRoot + ? { mode: 'recipe-json', projectRoot: assertServeProjectRoot(options.projectRoot) } + : { mode: options.json ? 'json' : 'human' } + ) + + await idleExitStartup.start({ + rpc, + agentStates: () => agentHookServer.getStatusSnapshot(), + hasStagedMigration: () => profileStore.hasStagedOrcadMigrationCatalog(), + automationsBusy: () => orcadAutomationsKeepHostBusy(profileStore), + registerCleanup + }) return { readiness } } @@ -410,6 +411,13 @@ export { ORCAD_SHUTDOWN_DEADLINE_MS } from './orcad-lifecycle' export async function main(argv: string[] = process.argv.slice(2)): Promise { const startup = startOrcad(parseArgs(argv)) - installOrcadShutdownSignals(async () => (await startup).stop()) - await startup + const requestShutdown = installOrcadShutdownSignals(async () => (await startup).stop()) + const handle = await startup + // Why after startup: a managed request must name the runtime and instance this run became. + installOrcadStopRequestListeners(() => requestShutdown('stop request'), { + installRoot: resolveOrcadInstallRoot(), + managedStop: handle.managedStop, + beforeManagedStop: prepareOrcadManagedStop + }) + bindOrcadIdleShutdown(requestShutdown) } diff --git a/src/main/orcad/orcad-health.test.ts b/src/main/orcad/orcad-health.test.ts index 70e02b37e87..4f75c7c9025 100644 --- a/src/main/orcad/orcad-health.test.ts +++ b/src/main/orcad/orcad-health.test.ts @@ -11,13 +11,21 @@ const { readDaemonPidRecordMock, daemonOwnsFreshPersistentPtysMock } = vi.hoisted(() => ({ - checkDaemonHealthMock: vi.fn<() => Promise>(), + checkDaemonHealthMock: vi.fn<(socketPath: string, tokenPath: string) => Promise>(), getDaemonEndpointFactsMock: vi.fn<() => unknown>(), readDaemonPidRecordMock: vi.fn<() => ParsedDaemonPid | null>(), daemonOwnsFreshPersistentPtysMock: vi.fn<() => boolean>() })) -vi.mock('../daemon/daemon-health', () => ({ checkDaemonHealth: checkDaemonHealthMock })) +// The real coverage fallback is per platform; a daemon may also report its own coverage. +const reportedCoverage = vi.hoisted(() => ({ value: new Array<'pty-spawn' | 'handshake'>() })) +vi.mock('../daemon/daemon-health', () => ({ + checkDaemonHealthWithCoverage: async (socketPath: string, tokenPath: string) => ({ + verdict: await checkDaemonHealthMock(socketPath, tokenPath), + coverage: + reportedCoverage.value.shift() ?? (process.platform === 'win32' ? 'handshake' : 'pty-spawn') + }) +})) vi.mock('../daemon/daemon-init', () => ({ getDaemonEndpointFacts: getDaemonEndpointFactsMock, readDaemonPidRecord: readDaemonPidRecordMock, @@ -123,6 +131,12 @@ describe('collectTerminalDaemonHealth', () => { // green verdict there must not be reported as a PTY round trip. expect(health.selfTest.coverage).toBe('handshake') }) + + it('reports the coverage the daemon says its probe achieved', async () => { + reportedCoverage.value.push('handshake') + const health = await collectTerminalDaemonHealth() + expect(health.selfTest).toMatchObject({ ok: true, coverage: 'handshake' }) + }) }) describe('collectOrcadHealth', () => { diff --git a/src/main/orcad/orcad-health.ts b/src/main/orcad/orcad-health.ts index 0c3a3525171..c5ab26d28c3 100644 --- a/src/main/orcad/orcad-health.ts +++ b/src/main/orcad/orcad-health.ts @@ -11,13 +11,16 @@ import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import process from 'node:process' -import { checkDaemonHealth, type DaemonHealth } from '../daemon/daemon-health' +import { checkDaemonHealthWithCoverage, type DaemonHealth } from '../daemon/daemon-health' +import { ptySpawnHealthPlatformCoverage } from '../daemon/daemon-health-identity' import { daemonOwnsFreshPersistentPtys, getDaemonEndpointFacts, readDaemonPidRecord } from '../daemon/daemon-init' import type { OrcadProfileStateAuthoritySelection } from './orcad-profile-state-telemetry' +import { ORCAD_STOP_REQUESTS_CAPABILITY } from '../../shared/orcad-stop-request' +import type { OrcadIdleStopRecord } from '../../shared/orcad-idle-exit' /** * How much a green self-test actually proves. @@ -67,6 +70,16 @@ export type OrcadHealth = { terminalDaemon: TerminalDaemonHealth /** The low-cardinality profile-state authority selected during startup, when available. */ profileStateAuthority?: OrcadProfileStateAuthoritySelection + /** + * Present when this build consumes stop-request files and answers the managed-stop commands. + * Absent on older builds, which a client must keep stopping with SIGTERM. + */ + stopRequests?: typeof ORCAD_STOP_REQUESTS_CAPABILITY + /** + * Managed launches only: how the previous run ended if it stopped for idleness, else null + * (a crash, a signal, or a first start). Absent on user-started and older builds. + */ + previousIdleStop?: OrcadIdleStopRecord | null } /** @@ -100,14 +113,20 @@ export async function runTerminalDaemonSelfTest( now: () => number = () => Date.now() ): Promise { const startedAt = now() - // Why: `checkPtySpawnHealth` returns immediately on win32 without spawning anything, so a - // green verdict there covers the handshake only. Say so instead of overclaiming. - const coverage: PtySelfTestCoverage = process.platform === 'win32' ? 'handshake' : 'pty-spawn' const facts = getDaemonEndpointFacts() if (!facts) { - return { ok: false, coverage, verdict: 'no-daemon', durationMs: now() - startedAt } + return { + ok: false, + coverage: ptySpawnHealthPlatformCoverage(), + verdict: 'no-daemon', + durationMs: now() - startedAt + } } - const verdict = await checkDaemonHealth(facts.socketPath, facts.tokenPath) + // The daemon reports what its probe actually did; an older daemon falls back by platform. + const { verdict, coverage } = await checkDaemonHealthWithCoverage( + facts.socketPath, + facts.tokenPath + ) return { ok: verdict === 'healthy', coverage, verdict, durationMs: now() - startedAt } } @@ -151,7 +170,8 @@ export async function collectTerminalDaemonHealth(): Promise { return { buildHash: computeOrcadBuildHash(), @@ -162,6 +182,8 @@ export async function collectOrcadHealth( arch: process.arch, pid: process.pid, terminalDaemon: await collectTerminalDaemonHealth(), - ...(profileStateAuthority ? { profileStateAuthority } : {}) + ...(profileStateAuthority ? { profileStateAuthority } : {}), + stopRequests: ORCAD_STOP_REQUESTS_CAPABILITY, + ...(previousIdleStop !== undefined ? { previousIdleStop } : {}) } } diff --git a/src/main/orcad/orcad-idle-exit-monitor.test.ts b/src/main/orcad/orcad-idle-exit-monitor.test.ts new file mode 100644 index 00000000000..fbe31074a52 --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from 'vitest' +import { + OrcadIdleExitMonitor, + resolveOrcadIdlePollMs, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +function harness(options: { timeoutMs?: number; lastClientActivityAt?: number } = {}) { + let now = 1_000 + let lastActivity = options.lastClientActivityAt ?? 0 + const verdicts: Record = { clients: 'idle', terminals: 'idle' } + const probes: OrcadIdleProbe[] = Object.keys(verdicts).map((name) => ({ + name, + read: () => { + const verdict = verdicts[name] + if (verdict instanceof Error) { + throw verdict + } + return verdict + } + })) + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: options.timeoutMs ?? 100, + probes, + lastClientActivityAt: () => lastActivity, + onIdle, + now: () => now, + log: () => {} + }) + return { + monitor, + onIdle, + verdicts, + advance: (ms: number) => (now += ms), + touch: () => (lastActivity = now) + } +} + +describe('OrcadIdleExitMonitor', () => { + it('fires once every probe has stayed idle for the whole quiet period', async () => { + const h = harness() + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + expect(h.onIdle).toHaveBeenCalledWith({ quietSince: 1_000, stoppedAt: 1_100, timeoutMs: 100 }) + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).toHaveBeenCalledTimes(1) + }) + + it('restarts the quiet period whenever any probe is busy', async () => { + const h = harness() + await h.monitor.check() + h.advance(90) + h.verdicts.terminals = 'busy' + expect(await h.monitor.check()).toBe(false) + h.verdicts.terminals = 'idle' + h.advance(10) + expect(await h.monitor.check()).toBe(false) + h.advance(99) + expect(await h.monitor.check()).toBe(false) + h.advance(1) + expect(await h.monitor.check()).toBe(true) + }) + + it.each([ + ['unverifiable', 'unverifiable' as const], + ['throwing', new Error('daemon did not answer')] + ])('treats a %s probe as busy, never as idle', async (_label, verdict) => { + const h = harness() + h.verdicts.terminals = verdict + for (let i = 0; i < 5; i += 1) { + expect(await h.monitor.check()).toBe(false) + h.advance(100) + } + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('counts a request that came and went between checks as activity', async () => { + const h = harness() + await h.monitor.check() + h.advance(80) + h.touch() + h.advance(20) + expect(await h.monitor.check()).toBe(false) + h.advance(80) + expect(await h.monitor.check()).toBe(true) + }) + + it('does not fire after it was stopped', async () => { + const h = harness() + await h.monitor.check() + h.monitor.stop() + h.advance(1_000) + expect(await h.monitor.check()).toBe(false) + expect(h.onIdle).not.toHaveBeenCalled() + }) + + it('polls often enough for a short test timeout and at most once a minute', () => { + expect(resolveOrcadIdlePollMs(15 * 60_000)).toBe(60_000) + expect(resolveOrcadIdlePollMs(2_000)).toBe(400) + expect(resolveOrcadIdlePollMs(10)).toBe(250) + }) + + it('stops itself on a timer once idle', async () => { + vi.useFakeTimers() + try { + const onIdle = vi.fn() + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: 1_000, + probes: [{ name: 'clients', read: () => 'idle' }], + lastClientActivityAt: () => 0, + onIdle, + log: () => {} + }) + monitor.start() + await vi.advanceTimersByTimeAsync(1_600) + expect(onIdle).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(5_000) + expect(onIdle).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/orcad/orcad-idle-exit-monitor.ts b/src/main/orcad/orcad-idle-exit-monitor.ts new file mode 100644 index 00000000000..58a053adffc --- /dev/null +++ b/src/main/orcad/orcad-idle-exit-monitor.ts @@ -0,0 +1,116 @@ +/** + * Decides when a managed orcad has been unused long enough to stop. + * + * Every probe must answer `idle` on the same check, continuously for the whole quiet period. + * A probe that throws or cannot answer counts as busy: silence is never evidence of idleness. + */ + +export type OrcadIdleVerdict = 'idle' | 'busy' | 'unverifiable' + +export type OrcadIdleProbe = { + name: string + read: () => OrcadIdleVerdict | Promise +} + +export type OrcadIdleExitEvidence = { quietSince: number; stoppedAt: number; timeoutMs: number } + +export type OrcadIdleExitMonitorOptions = { + timeoutMs: number + probes: readonly OrcadIdleProbe[] + /** Any client request restarts the quiet period, even one that came and went between checks. */ + lastClientActivityAt: () => number + onIdle: (evidence: OrcadIdleExitEvidence) => void + now?: () => number + pollMs?: number + log?: (line: string) => void +} + +export function resolveOrcadIdlePollMs(timeoutMs: number): number { + return Math.min(60_000, Math.max(250, Math.floor(timeoutMs / 5))) +} + +export class OrcadIdleExitMonitor { + private timer: ReturnType | null = null + private quietSince: number | null = null + private blocker: string | null = null + private stopped = false + private readonly now: () => number + private readonly pollMs: number + private readonly log: (line: string) => void + + constructor(private readonly options: OrcadIdleExitMonitorOptions) { + this.now = options.now ?? Date.now + this.pollMs = options.pollMs ?? resolveOrcadIdlePollMs(options.timeoutMs) + this.log = options.log ?? ((line) => console.error(line)) + } + + start(): void { + this.schedule() + } + + stop(): void { + this.stopped = true + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + } + + /** One check; resolves true once the quiet period elapsed and `onIdle` fired. */ + async check(): Promise { + const blocker = await this.findBlocker() + if (this.stopped) { + return false + } + const now = this.now() + if (blocker) { + if (this.quietSince !== null || this.blocker !== blocker) { + this.log(`[orcad] idle exit waiting: ${blocker}`) + } + this.quietSince = null + this.blocker = blocker + return false + } + if (this.quietSince === null) { + this.quietSince = now + this.blocker = null + this.log(`[orcad] idle; stopping after ${this.options.timeoutMs}ms unless a client returns`) + } + const quietSince = Math.max(this.quietSince, this.options.lastClientActivityAt()) + if (now - quietSince < this.options.timeoutMs) { + return false + } + this.stop() + this.options.onIdle({ quietSince, stoppedAt: now, timeoutMs: this.options.timeoutMs }) + return true + } + + private async findBlocker(): Promise { + for (const probe of this.options.probes) { + let verdict: OrcadIdleVerdict + try { + verdict = await probe.read() + } catch { + verdict = 'unverifiable' + } + if (verdict !== 'idle') { + return `${probe.name} ${verdict}` + } + } + return null + } + + private schedule(): void { + if (this.stopped) { + return + } + this.timer = setTimeout(() => { + this.timer = null + void this.check() + .catch((error: unknown) => this.log(`[orcad] idle check failed: ${String(error)}`)) + .finally(() => this.schedule()) + }, this.pollMs) + // Never the reason the process stays alive. + this.timer.unref?.() + } +} diff --git a/src/main/orcad/orcad-idle-stop-record.test.ts b/src/main/orcad/orcad-idle-stop-record.test.ts new file mode 100644 index 00000000000..ac742420fcd --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.test.ts @@ -0,0 +1,84 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + consumeOrcadIdleStopRecord, + orcadIdleStopRecordPath, + writeOrcadIdleStopRecord +} from './orcad-idle-stop-record' +import { createIdleStopRecordOwnership } from './orcad-managed-idle-exit-host' + +let root: string + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-idle-stop-')) +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +describe('orcad idle-stop record', () => { + it('lets the next start tell an idle stop apart from a crash, exactly once', () => { + writeOrcadIdleStopRecord( + root, + { + quietSince: Date.parse('2026-10-03T10:00:00Z'), + stoppedAt: Date.parse('2026-10-03T10:15:00Z'), + timeoutMs: 900_000 + }, + '1.2.3' + ) + + expect(consumeOrcadIdleStopRecord(root)).toMatchObject({ + kind: 'orcad_idle_stop', + pid: process.pid, + version: '1.2.3', + quietSince: '2026-10-03T10:00:00.000Z', + stoppedAt: '2026-10-03T10:15:00.000Z', + idleTimeoutMs: 900_000 + }) + // A crash after this start must not inherit the earlier idle stop. + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('reads a start with no record as "not an idle stop"', () => { + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + }) + + it('drops a corrupt record instead of trusting it', () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + writeFileSync(orcadIdleStopRecordPath(root), '{"kind":"orcad_idle_stop"') + + expect(consumeOrcadIdleStopRecord(root)).toBeNull() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + }) + + it('drops the record when a signal stop already owned the shutdown', () => { + const evidence = { quietSince: 1, stoppedAt: 2, timeoutMs: 3 } + writeOrcadIdleStopRecord(root, evidence, '1.0.0') + createIdleStopRecordOwnership(root).requestShutdown(() => false) + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + + writeOrcadIdleStopRecord(root, evidence, '1.0.0') + const owned = createIdleStopRecordOwnership(root) + let onFailed: (() => void) | undefined + owned.requestShutdown((_reason, failed) => { + onFailed = failed + return true + }) + owned.onShutdown() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(true) + onFailed?.() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + }) + + it('drops the record when a signal stop finishes before the idle stop asks to shut down', () => { + writeOrcadIdleStopRecord(root, { quietSince: 1, stoppedAt: 2, timeoutMs: 3 }, '1.0.0') + // The signal's stop runs every cleanup, then exits before the idle request ever runs. + createIdleStopRecordOwnership(root).onShutdown() + expect(existsSync(orcadIdleStopRecordPath(root))).toBe(false) + }) +}) diff --git a/src/main/orcad/orcad-idle-stop-record.ts b/src/main/orcad/orcad-idle-stop-record.ts new file mode 100644 index 00000000000..11672fe4e73 --- /dev/null +++ b/src/main/orcad/orcad-idle-stop-record.ts @@ -0,0 +1,62 @@ +/** + * The record that tells a clean idle stop apart from a crash. Written just before an idle + * stop and discarded if that stop fails; the next start reports it once and removes it, so a + * later crash never inherits it. + */ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { + ORCAD_IDLE_STOP_RECORD_FILENAME, + OrcadIdleStopRecordSchema, + type OrcadIdleStopRecord +} from '../../shared/orcad-idle-exit' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' +import { hasErrorCode } from '../daemon/daemon-process-inspection' + +const RECORD_MAX_BYTES = 16 * 1024 + +export function orcadIdleStopRecordPath(userDataPath: string): string { + return join(userDataPath, ORCAD_IDLE_STOP_RECORD_FILENAME) +} + +export function writeOrcadIdleStopRecord( + userDataPath: string, + evidence: OrcadIdleExitEvidence, + version: string +): void { + const record: OrcadIdleStopRecord = { + schemaVersion: 1, + kind: 'orcad_idle_stop', + pid: process.pid, + version, + quietSince: new Date(evidence.quietSince).toISOString(), + stoppedAt: new Date(evidence.stoppedAt).toISOString(), + idleTimeoutMs: evidence.timeoutMs + } + if (!writeDurableSecureJsonFile(orcadIdleStopRecordPath(userDataPath), record)) { + throw new Error('orcad_idle_stop_record_permissions_unconfirmed') + } +} + +/** The previous run's idle stop, or null when it ended any other way (or never ran). */ +export function consumeOrcadIdleStopRecord(userDataPath: string): OrcadIdleStopRecord | null { + const path = orcadIdleStopRecordPath(userDataPath) + let record: OrcadIdleStopRecord | null = null + try { + const raw = readNodeFileSyncWithinLimit(path, RECORD_MAX_BYTES).buffer.toString('utf8') + record = OrcadIdleStopRecordSchema.parse(JSON.parse(raw)) + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + return null + } + console.error('[orcad] ignoring an unreadable idle-stop record:', error) + } + rmSync(path, { force: true }) + return record +} + +export function discardOrcadIdleStopRecord(userDataPath: string): void { + rmSync(orcadIdleStopRecordPath(userDataPath), { force: true }) +} diff --git a/src/main/orcad/orcad-instance-lock.test.ts b/src/main/orcad/orcad-instance-lock.test.ts index 45f7543c515..36dc3700f23 100644 --- a/src/main/orcad/orcad-instance-lock.test.ts +++ b/src/main/orcad/orcad-instance-lock.test.ts @@ -2,9 +2,12 @@ import { chmodSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, + renameSync, rmSync, statSync, + utimesSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' @@ -14,7 +17,8 @@ import { acquireOrcadInstanceLock, ORCAD_LOCK_FILE_NAME, OrcadInstanceLockError, - type OrcadInstanceLockHooks + type OrcadInstanceLockHooks, + type OrcadLockRecord } from './orcad-instance-lock' const roots: string[] = [] @@ -37,6 +41,18 @@ function hooks(overrides: OrcadInstanceLockHooks = {}): OrcadInstanceLockHooks { } } +function persistedRecord(overrides: Partial = {}): OrcadLockRecord { + return { + pid: 424242, + startedAtMs: 1, + identity: 'uid-1000', + version: '1.0.0-test', + acquiredAt: '2026-01-01T00:00:00.000Z', + nonce: 'stale', + ...overrides + } +} + afterEach(() => { for (const root of roots.splice(0)) { rmSync(root, { recursive: true, force: true }) @@ -68,20 +84,14 @@ describe('acquireOrcadInstanceLock', () => { it('reclaims the record of a holder that is gone', () => { const root = makeRoot() - writeFileSync( - join(root, ORCAD_LOCK_FILE_NAME), - JSON.stringify({ pid: 424242, identity: 'uid-1000', startedAtMs: 1, nonce: 'stale' }) - ) + writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord())) const lock = acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => false })) expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid) }) it('treats a live pid whose start time does not match as a recycled pid, not a holder', () => { const root = makeRoot() - writeFileSync( - join(root, ORCAD_LOCK_FILE_NAME), - JSON.stringify({ pid: 424242, identity: 'uid-1000', startedAtMs: 1, nonce: 'stale' }) - ) + writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), JSON.stringify(persistedRecord())) const lock = acquireOrcadInstanceLock( root, hooks({ processIsAlive: () => true, startTimeMatches: () => false }) @@ -89,11 +99,37 @@ describe('acquireOrcadInstanceLock', () => { expect(JSON.parse(readFileSync(lock.path, 'utf8')).pid).toBe(process.pid) }) + it('does not displace a successor published between stale inspection and reclaim', () => { + const root = makeRoot() + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + const successor = persistedRecord({ pid: 777, startedAtMs: 2, nonce: 'successor' }) + writeFileSync(lockPath, JSON.stringify(persistedRecord())) + let raced = false + const lockHooks = hooks({ + processIsAlive: () => { + if (!raced) { + raced = true + // Simulate a contender replacing the stale record and publishing its own lock + // after this process inspected liveness but before it renames the entry. + const displacedPath = `${lockPath}.displaced` + renameSync(lockPath, displacedPath) + writeFileSync(lockPath, JSON.stringify(successor), { flag: 'wx', mode: 0o600 }) + } + return false + } + }) + + expect(() => acquireOrcadInstanceLock(root, lockHooks)).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_held' }) + ) + expect(JSON.parse(readFileSync(lockPath, 'utf8')).nonce).toBe('successor') + }) + it('never reclaims a lock held by a different identity, even a dead one', () => { const root = makeRoot() writeFileSync( join(root, ORCAD_LOCK_FILE_NAME), - JSON.stringify({ pid: 424242, identity: 'uid-2000', startedAtMs: 1, nonce: 'other' }) + JSON.stringify(persistedRecord({ identity: 'uid-2000', nonce: 'other' })) ) expect(() => acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => false }))).toThrow( expect.objectContaining({ code: 'orcad_instance_lock_foreign_identity' }) @@ -106,12 +142,62 @@ describe('acquireOrcadInstanceLock', () => { // A successor reclaimed the root while this process was wedged. writeFileSync( lock.path, - JSON.stringify({ pid: 777, identity: 'uid-1000', startedAtMs: 2, nonce: 'successor' }) + JSON.stringify(persistedRecord({ pid: 777, startedAtMs: 2, nonce: 'successor' })) ) lock.release() expect(JSON.parse(readFileSync(lock.path, 'utf8')).nonce).toBe('successor') }) + const garbledLocks = [ + ['nothing (a torn write)', ''], + ['invalid JSON', '{'], + ['an incomplete record', JSON.stringify({ pid: 424242, identity: 'uid-1000' })], + ['an invalid pid', JSON.stringify(persistedRecord({ pid: -1 }))], + ['an invalid start time', JSON.stringify({ ...persistedRecord(), startedAtMs: 'yesterday' })] + ] + + it.each(garbledLocks)('leaves a lock still being written alone: %s', (_label, contents) => { + const root = makeRoot() + writeFileSync(join(root, ORCAD_LOCK_FILE_NAME), contents) + + expect(() => acquireOrcadInstanceLock(root, hooks())).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_held' }) + ) + expect(readFileSync(join(root, ORCAD_LOCK_FILE_NAME), 'utf8')).toBe(contents) + }) + + it.each(garbledLocks)('reclaims an abandoned lock containing %s', (_label, contents) => { + const root = makeRoot() + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + writeFileSync(lockPath, contents) + const longAgo = new Date(Date.now() - 60_000) + utimesSync(lockPath, longAgo, longAgo) + + const lock = acquireOrcadInstanceLock(root, hooks()) + expect(JSON.parse(readFileSync(lockPath, 'utf8')).nonce).toBe(lock.record.nonce) + expect(readdirSync(root).sort()).toEqual([ORCAD_LOCK_FILE_NAME]) + }) + + it('fails closed when the existing lock is not a regular file', () => { + const root = makeRoot() + mkdirSync(join(root, ORCAD_LOCK_FILE_NAME)) + + expect(() => acquireOrcadInstanceLock(root, hooks())).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_unreadable' }) + ) + }) + + it('fails closed without reading an oversized lock into memory', () => { + const root = makeRoot() + const lockPath = join(root, ORCAD_LOCK_FILE_NAME) + writeFileSync(lockPath, 'x'.repeat(64 * 1024 + 1)) + + expect(() => acquireOrcadInstanceLock(root, hooks())).toThrow( + expect.objectContaining({ code: 'orcad_instance_lock_unreadable' }) + ) + expect(statSync(lockPath).size).toBe(64 * 1024 + 1) + }) + it.runIf(process.platform !== 'win32')( 'tightens a group/world-accessible data root rather than refusing when it can', () => { @@ -122,17 +208,21 @@ describe('acquireOrcadInstanceLock', () => { } ) - it.runIf(process.platform !== 'win32')('refuses a data root owned by another uid', () => { - const root = makeRoot() - // /tmp itself is root-owned and sticky on every supported platform, so it stands in for - // "a data root this process does not own" without needing privileges to create one. - expect(() => acquireOrcadInstanceLock('/tmp', hooks())).toThrow( - expect.objectContaining({ code: 'orcad_data_root_wrong_owner' }) - ) - // And the private root this test made is still acceptable, so the refusal is about - // ownership rather than a blanket rejection. - expect(acquireOrcadInstanceLock(root, hooks()).record.identity).toBe('uid-1000') - }) + // Not as root: root owns /tmp, so "a root this process does not own" has no stand-in there. + it.runIf(process.platform !== 'win32' && process.getuid?.() !== 0)( + 'refuses a data root owned by another uid', + () => { + const root = makeRoot() + // /tmp itself is root-owned and sticky on every supported platform, so it stands in for + // "a data root this process does not own" without needing privileges to create one. + expect(() => acquireOrcadInstanceLock('/tmp', hooks())).toThrow( + expect.objectContaining({ code: 'orcad_data_root_wrong_owner' }) + ) + // And the private root this test made is still acceptable, so the refusal is about + // ownership rather than a blanket rejection. + expect(acquireOrcadInstanceLock(root, hooks()).record.identity).toBe('uid-1000') + } + ) it('leaves the terminal daemon alone: the lock covers only the runtime role', () => { const root = makeRoot() @@ -148,4 +238,95 @@ describe('acquireOrcadInstanceLock', () => { const next = acquireOrcadInstanceLock(root, hooks()) expect(next.record.pid).toBe(process.pid) }) + + it('restricts a Windows data root by ACL, and refuses when the ACL cannot be applied', () => { + const root = makeRoot() + const restricted: string[] = [] + const windows = (applied: boolean) => + hooks({ + platform: 'win32', + restrictWindowsDataRoot: (path) => { + restricted.push(path) + return applied + } + }) + expect(() => acquireOrcadInstanceLock(root, windows(false))).toThrow( + expect.objectContaining({ code: 'orcad_data_root_shared' }) + ) + // Refused before any record was published. + expect(() => readFileSync(join(root, ORCAD_LOCK_FILE_NAME))).toThrow() + expect(acquireOrcadInstanceLock(root, windows(true)).record.pid).toBe(process.pid) + expect(restricted).toEqual([root, root]) + }) + + it.runIf(process.platform === 'win32')( + 'records a real creation time and leaves a really restricted data root on Windows', + () => { + const root = makeRoot() + const lock = acquireOrcadInstanceLock(root, { identity: () => 'uid-1000' }) + expect(lock.record.startedAtMs).toEqual(expect.any(Number)) + expect( + Math.abs(lock.record.startedAtMs! - (Date.now() - process.uptime() * 1000)) + ).toBeLessThan(5_000) + lock.release() + } + ) + + it('makes the desktop app and orcad refuse each other on one profile', () => { + const root = makeRoot() + const desktop = acquireOrcadInstanceLock(root, hooks({ role: 'desktop' })) + expect(JSON.parse(readFileSync(desktop.path, 'utf8')).role).toBe('desktop') + expect(() => acquireOrcadInstanceLock(root, hooks({ processIsAlive: () => true }))).toThrow( + expect.objectContaining({ + code: 'orcad_instance_lock_held', + message: expect.stringContaining('The Orca desktop app') + }) + ) + desktop.release() + const orcad = acquireOrcadInstanceLock(root, hooks()) + expect(() => + acquireOrcadInstanceLock(root, hooks({ role: 'desktop', processIsAlive: () => true })) + ).toThrow( + expect.objectContaining({ + code: 'orcad_instance_lock_held', + message: expect.stringContaining('Another orcad') + }) + ) + orcad.release() + }) + + it('lets a desktop reclaim a crashed desktop record whose PID was reused', () => { + const root = makeRoot() + writeFileSync( + join(root, ORCAD_LOCK_FILE_NAME), + JSON.stringify(persistedRecord({ role: 'desktop', startedAtMs: null })) + ) + // A null start time cannot disprove the reused PID; Electron's own lock already does. + const lock = acquireOrcadInstanceLock( + root, + hooks({ role: 'desktop', processIsAlive: () => true }) + ) + expect(JSON.parse(readFileSync(lock.path, 'utf8')).nonce).toBe(lock.record.nonce) + lock.release() + }) + + it.runIf(process.platform !== 'win32')( + "leaves the desktop profile's permissions as they were", + () => { + const root = makeRoot() + chmodSync(root, 0o755) + const restricted: string[] = [] + acquireOrcadInstanceLock(root, hooks({ role: 'desktop' })).release() + acquireOrcadInstanceLock( + root, + hooks({ + role: 'desktop', + platform: 'win32', + restrictWindowsDataRoot: (path) => restricted.push(path) > 0 + }) + ).release() + expect(restricted).toEqual([]) + expect(statSync(root).mode & 0o777).toBe(0o755) + } + ) }) diff --git a/src/main/orcad/orcad-instance-lock.ts b/src/main/orcad/orcad-instance-lock.ts index 052140bcd2e..72fdb8263b7 100644 --- a/src/main/orcad/orcad-instance-lock.ts +++ b/src/main/orcad/orcad-instance-lock.ts @@ -13,49 +13,49 @@ * it says nothing about the daemon, which is what makes a non-destructive restart possible. */ import { randomUUID } from 'node:crypto' -import { - chmodSync, - mkdirSync, - readFileSync, - renameSync, - statSync, - unlinkSync, - writeFileSync -} from 'node:fs' +import { linkSync, mkdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs' import { userInfo } from 'node:os' import { join } from 'node:path' import process from 'node:process' -import { getProcessStartedAtMs, startTimeMatches } from '../daemon/daemon-process-start-time' +import { z } from 'zod' +import { + orcadProcessStartTimeMatches, + readOrcadProcessStartedAtMs +} from './orcad-process-start-time' +import { + assertOrcadDataRootIsPrivate, + OrcadInstanceLockError, + type OrcadDataRootPrivacyHooks +} from './orcad-data-root-privacy' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { hasErrorCode, isProcessAlive } from '../daemon/daemon-process-inspection' export const ORCAD_LOCK_FILE_NAME = 'orcad.lock' +const MAX_ORCAD_LOCK_BYTES = 64 * 1024 +// A writer finishes in milliseconds; a garbled record older than this has no live writer. +const GARBLED_LOCK_GRACE_MS = 10_000 -export type OrcadInstanceLockCode = - | 'orcad_data_root_unusable' - | 'orcad_data_root_wrong_owner' - | 'orcad_data_root_shared' - | 'orcad_instance_lock_held' - | 'orcad_instance_lock_foreign_identity' +export { OrcadInstanceLockError, type OrcadInstanceLockCode } from './orcad-data-root-privacy' -export class OrcadInstanceLockError extends Error { - constructor( - readonly code: OrcadInstanceLockCode, - message: string - ) { - super(message) - this.name = 'OrcadInstanceLockError' - } -} - -export type OrcadLockRecord = { - pid: number +const OrcadLockRecordSchema = z.object({ + pid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), /** Null where the platform cannot read it; PID alone is then the (weaker) fence. */ - startedAtMs: number | null + startedAtMs: z.number().finite().nonnegative().nullable(), /** POSIX uid, or the Windows username. Compared as an opaque string. */ - identity: string - version: string - acquiredAt: string + identity: z.string().min(1).max(1_024), + version: z.string().min(1).max(255), + acquiredAt: z.iso.datetime({ offset: true }), /** Distinguishes our record from a replacement written after we lost the race. */ - nonce: string + nonce: z.string().min(1).max(255), + /** Absent in records orcad wrote before the desktop app shared this lock. */ + role: z.enum(['orcad', 'desktop']).optional() +}) + +export type OrcadLockRecord = z.infer + +/** `null` when absent, unreadable, oversized or malformed; callers must not read that as free. */ +export function readOrcadInstanceLockRecord(path: string): OrcadLockRecord | null { + return parseOrcadInstanceLockRecord(readBoundedLockFile(path) ?? '') } export type OrcadInstanceLock = { @@ -64,7 +64,7 @@ export type OrcadInstanceLock = { release(): void } -export type OrcadInstanceLockHooks = { +export type OrcadInstanceLockHooks = OrcadDataRootPrivacyHooks & { identity?: () => string version?: () => string now?: () => Date @@ -72,6 +72,11 @@ export type OrcadInstanceLockHooks = { processIsAlive?: (pid: number) => boolean startedAtMs?: (pid: number) => number | null startTimeMatches?: (pid: number, expected: number | null) => boolean + /** + * Who takes the profile. The desktop app takes it too, so the two refuse each other; it must + * hold Electron's single-instance lock first, which is what lets it reclaim a desktop record. + */ + role?: 'orcad' | 'desktop' } function defaultIdentity(): string { @@ -82,101 +87,16 @@ function defaultIdentity(): string { : String(process.getuid?.() ?? 'unknown') } -function defaultProcessIsAlive(pid: number): boolean { +/** `null` for anything that is not a complete record; never a reason to treat a lock as free. */ +export function parseOrcadInstanceLockRecord(content: string): OrcadLockRecord | null { try { - process.kill(pid, 0) - return true - } catch (error) { - return isErrorCode(error, 'EPERM') - } -} - -function isErrorCode(error: unknown, code: string): boolean { - return typeof error === 'object' && error !== null && 'code' in error && error.code === code -} - -function parseLockRecord(content: string): OrcadLockRecord | null { - try { - const parsed: unknown = JSON.parse(content) - if (!parsed || typeof parsed !== 'object') { - return null - } - const record = parsed as Partial - if (typeof record.pid !== 'number' || typeof record.identity !== 'string') { - return null - } - return { - pid: record.pid, - startedAtMs: typeof record.startedAtMs === 'number' ? record.startedAtMs : null, - identity: record.identity, - version: typeof record.version === 'string' ? record.version : 'unknown', - acquiredAt: typeof record.acquiredAt === 'string' ? record.acquiredAt : '', - nonce: typeof record.nonce === 'string' ? record.nonce : '' - } + const result = OrcadLockRecordSchema.safeParse(JSON.parse(content)) + return result.success ? result.data : null } catch { return null } } -/** - * Fail closed on a data root other identities can read or write. - * - * Why self-heal first and refuse second: orcad stores credentials unsealed (there is no OS - * keyring on this host), so a group- or world-accessible root is a real exposure — but if - * we own the directory, tightening it is strictly better than refusing to start. We refuse - * only when the permissions are not ours to fix. - */ -function assertDataRootIsPrivate(dataRoot: string): void { - // Windows ACLs are not expressible as a POSIX mode, and `statSync().mode` there reports a - // synthesized one. Checking it would refuse correct deployments and pass wrong ones. - if (process.platform === 'win32') { - return - } - let stats - try { - stats = statSync(dataRoot) - } catch (error) { - throw new OrcadInstanceLockError( - 'orcad_data_root_unusable', - `Cannot stat the orcad data root ${dataRoot}: ${(error as Error).message}` - ) - } - const uid = process.getuid?.() - if (uid !== undefined && stats.uid !== uid) { - throw new OrcadInstanceLockError( - 'orcad_data_root_wrong_owner', - `The orcad data root ${dataRoot} is owned by uid ${stats.uid}, not by uid ${uid} running ` + - 'this process. Give orcad its own data root (ORCA_USER_DATA) or chown this one.' - ) - } - if ((stats.mode & 0o077) === 0) { - return - } - try { - chmodSync(dataRoot, 0o700) - } catch { - // Fall through to the re-stat, which produces the actionable message. - } - let mode: number - try { - mode = statSync(dataRoot).mode - } catch (error) { - throw new OrcadInstanceLockError( - 'orcad_data_root_unusable', - `Cannot stat the orcad data root ${dataRoot}: ${(error as Error).message}` - ) - } - if ((mode & 0o077) !== 0) { - throw new OrcadInstanceLockError( - 'orcad_data_root_shared', - `The orcad data root ${dataRoot} is accessible to other users (mode ` + - `${(mode & 0o777).toString(8)}) and could not be tightened. orcad stores credentials ` + - 'there unsealed, so it refuses to start. Run `chmod 700` on it, or point ORCA_USER_DATA ' + - 'at a private directory.' - ) - } -} - /** * Take the lock, or throw an `OrcadInstanceLockError` naming why. * @@ -188,19 +108,22 @@ export function acquireOrcadInstanceLock( hooks: OrcadInstanceLockHooks = {} ): OrcadInstanceLock { const identity = (hooks.identity ?? defaultIdentity)() - const isAlive = hooks.processIsAlive ?? defaultProcessIsAlive - const readStartedAt = hooks.startedAtMs ?? getProcessStartedAtMs - const matchesStartTime = hooks.startTimeMatches ?? startTimeMatches + const isAlive = hooks.processIsAlive ?? isProcessAlive + const readStartedAt = hooks.startedAtMs ?? readOrcadProcessStartedAtMs + const matchesStartTime = hooks.startTimeMatches ?? orcadProcessStartTimeMatches try { mkdirSync(dataRoot, { recursive: true, mode: 0o700 }) } catch (error) { throw new OrcadInstanceLockError( 'orcad_data_root_unusable', - `Cannot create the orcad data root ${dataRoot}: ${(error as Error).message}` + `Cannot create the orcad data root ${dataRoot}: ${error instanceof Error ? error.message : String(error)}` ) } - assertDataRootIsPrivate(dataRoot) + // The desktop's profile keeps the permissions it was created with; orcad tightens its own. + if ((hooks.role ?? 'orcad') === 'orcad') { + assertOrcadDataRootIsPrivate(dataRoot, hooks) + } const lockPath = join(dataRoot, ORCAD_LOCK_FILE_NAME) const record: OrcadLockRecord = { @@ -209,22 +132,28 @@ export function acquireOrcadInstanceLock( identity, version: (hooks.version ?? (() => process.env.ORCA_VERSION ?? 'unknown'))(), acquiredAt: (hooks.now ?? (() => new Date()))().toISOString(), - nonce: randomUUID() + nonce: randomUUID(), + role: hooks.role ?? 'orcad' } const serialized = JSON.stringify(record) + // Staged then hard-linked: the canonical path only ever holds a complete record, never a torn one. const publish = (): boolean => { + const staged = `${lockPath}.staged-${process.pid}-${randomUUID()}` try { - writeFileSync(lockPath, serialized, { flag: 'wx', mode: 0o600 }) + writeFileSync(staged, serialized, { flag: 'wx', mode: 0o600 }) + linkSync(staged, lockPath) return true } catch (error) { - if (isErrorCode(error, 'EEXIST')) { + if (hasErrorCode(error, 'EEXIST')) { return false } throw new OrcadInstanceLockError( 'orcad_data_root_unusable', - `Cannot write the orcad instance lock ${lockPath}: ${(error as Error).message}` + `Cannot write the orcad instance lock ${lockPath}: ${error instanceof Error ? error.message : String(error)}` ) + } finally { + unlinkQuietly(staged) } } @@ -232,8 +161,29 @@ export function acquireOrcadInstanceLock( return makeLock(lockPath, record) } - const existing = parseLockRecord(safeRead(lockPath) ?? '') - if (existing && existing.identity !== identity) { + const existingContents = readBoundedLockFile(lockPath) + if (existingContents === null) { + // Why fail closed: a record we cannot read proves nothing about its holder having exited. + throw new OrcadInstanceLockError( + 'orcad_instance_lock_unreadable', + `The orcad instance lock at ${lockPath} is unreadable or larger than ` + + `${MAX_ORCAD_LOCK_BYTES} bytes. Refusing to reclaim it without proof that its holder ` + + 'has exited. Stop orcad and remove the stale lock manually.' + ) + } + const existing = parseOrcadInstanceLockRecord(existingContents) + if (!existing) { + // Only a torn write (an older build, or a crash mid-write) leaves this; once it has aged + // past any live writer, no owner can be behind it. + if (garbledLockIsAbandoned(lockPath)) { + return reclaimAndPublish(lockPath, dataRoot, existingContents, publish, record) + } + throw new OrcadInstanceLockError( + 'orcad_instance_lock_held', + `The orcad instance lock at ${lockPath} is still being written by another process. Retry.` + ) + } + if (existing.identity !== identity) { throw new OrcadInstanceLockError( 'orcad_instance_lock_foreign_identity', `The orcad data root ${dataRoot} is locked by identity ${existing.identity} (pid ` + @@ -241,24 +191,36 @@ export function acquireOrcadInstanceLock( 'root corrupts it. Give each its own ORCA_USER_DATA.' ) } - if (existing && isAlive(existing.pid) && matchesStartTime(existing.pid, existing.startedAtMs)) { + // The desktop takes this lock only after Electron's single-instance lock, which already proves + // no other desktop runs; a desktop record is then stale even when its PID was reused. + const staleDesktopRecord = hooks.role === 'desktop' && existing.role === 'desktop' + if ( + !staleDesktopRecord && + isAlive(existing.pid) && + matchesStartTime(existing.pid, existing.startedAtMs) + ) { throw new OrcadInstanceLockError( 'orcad_instance_lock_held', - `Another orcad (pid ${existing.pid}, started ${existing.acquiredAt || 'unknown'}) already ` + - `owns the data root ${dataRoot}. Stop it before starting another, or use a different ` + - 'ORCA_USER_DATA.' - ) - } - if (!existing) { - console.warn( - `[orcad] The instance lock at ${lockPath} is unreadable; reclaiming it. If another orcad ` + - 'is running on this data root, stop it now.' + `${describeLockHolder(existing)} (pid ${existing.pid}, started ` + + `${existing.acquiredAt || 'unknown'}) already owns the data root ${dataRoot}. Stop it ` + + 'before starting another, or use a different ORCA_USER_DATA.' ) } + return reclaimAndPublish(lockPath, dataRoot, existingContents, publish, record) +} - // Why rename-and-then-publish rather than unlink-and-write: rename claims one exact - // directory entry, so a replacement written between our read and our write stays at the - // canonical path and wins — we never delete a record we did not inspect. +/** + * Why rename-and-then-publish rather than unlink-and-write: rename claims one exact directory + * entry, so a replacement written between our read and our write stays at the canonical path + * and wins — we never delete a record we did not inspect. + */ +function reclaimAndPublish( + lockPath: string, + dataRoot: string, + inspectedContents: string, + publish: () => boolean, + record: OrcadLockRecord +): OrcadInstanceLock { const claimPath = `${lockPath}.stale-${process.pid}-${randomUUID()}` try { renameSync(lockPath, claimPath) @@ -269,29 +231,74 @@ export function acquireOrcadInstanceLock( 'holding it. Retry, or stop the other orcad.' ) } - if (!publish()) { + // A contender may have replaced the entry after the liveness check; never displace its record. + const claimedContents = readBoundedLockFile(claimPath) + if (claimedContents !== inspectedContents) { + restoreDisplacedLock(claimPath, lockPath, claimedContents) + throw new OrcadInstanceLockError( + 'orcad_instance_lock_held', + `The orcad instance lock at ${lockPath} changed while reclaiming a stale record.` + ) + } + const published = publish() + // A uniquely named claim is inert either way. + unlinkQuietly(claimPath) + if (!published) { // Someone else claimed it first. Their record is authoritative; ours is not. - try { - unlinkSync(claimPath) - } catch { - // A uniquely named claim is inert. - } throw new OrcadInstanceLockError( 'orcad_instance_lock_held', `Another orcad took the data root ${dataRoot} while this one was reclaiming a stale lock.` ) } - try { - unlinkSync(claimPath) - } catch { - // The canonical record is authoritative; the claim is inert. - } return makeLock(lockPath, record) } -function safeRead(path: string): string | null { +function garbledLockIsAbandoned(lockPath: string): boolean { try { - return readFileSync(path, 'utf8') + return Date.now() - statSync(lockPath).mtimeMs > GARBLED_LOCK_GRACE_MS + } catch { + return false + } +} + +function unlinkQuietly(path: string): void { + try { + unlinkSync(path) + } catch { + // Already gone, or inert under its unique name. + } +} + +function describeLockHolder(record: OrcadLockRecord): string { + return record.role === 'desktop' ? 'The Orca desktop app' : 'Another orcad' +} + +/** No-clobber restore: a third contender's newer record at the canonical path stays authoritative. */ +function restoreDisplacedLock( + claimPath: string, + lockPath: string, + claimedContents: string | null +): void { + try { + linkSync(claimPath, lockPath) + unlinkSync(claimPath) + } catch { + if (claimedContents === null) { + return + } + try { + writeFileSync(lockPath, claimedContents, { flag: 'wx', mode: 0o600 }) + unlinkSync(claimPath) + } catch { + // A newer contender won, or restoration is unavailable; fail closed. + } + } +} + +function readBoundedLockFile(path: string): string | null { + try { + const { buffer, stats } = readNodeFileSyncWithinLimit(path, MAX_ORCAD_LOCK_BYTES) + return stats.isFile() ? buffer.toString('utf8') : null } catch { return null } @@ -310,7 +317,7 @@ function makeLock(lockPath: string, record: OrcadLockRecord): OrcadInstanceLock // Why re-read before unlinking: a reclaim by a later orcad (after, say, a SIGKILL that // this process somehow survived enough to run handlers) leaves a record that is not // ours. Deleting it would unlock a live runtime. - const current = parseLockRecord(safeRead(lockPath) ?? '') + const current = parseOrcadInstanceLockRecord(readBoundedLockFile(lockPath) ?? '') if (!current || current.nonce !== record.nonce) { return } diff --git a/src/main/orcad/orcad-launch-contract.test.ts b/src/main/orcad/orcad-launch-contract.test.ts index 26f2f28cbed..f8f7b2c85fa 100644 --- a/src/main/orcad/orcad-launch-contract.test.ts +++ b/src/main/orcad/orcad-launch-contract.test.ts @@ -26,6 +26,25 @@ describe('parseArgs', () => { }) }) + it('takes the desktop serve flags orca serve forwards', () => { + expect( + parseArgs([ + '--mobile-pairing', + '--recipe-json', + '--project-root', + '/work/app', + '--no-pairing' + ]) + ).toEqual({ + mobilePairing: true, + recipeJson: true, + projectRoot: '/work/app', + noPairing: true + }) + expect(() => parseArgs(['--recipe-json'])).toThrow('--recipe-json requires --project-root') + expect(() => parseArgs(['--project-root'])).toThrow('--project-root expects a value') + }) + it('rejects --bind with no value rather than silently binding the default', () => { expect(() => parseArgs(['--bind'])).toThrow('--bind expects a value') expect(() => parseArgs(['--bind', '--json'])).not.toThrow() diff --git a/src/main/orcad/orcad-lifecycle-host.test.ts b/src/main/orcad/orcad-lifecycle-host.test.ts new file mode 100644 index 00000000000..58004faafa5 --- /dev/null +++ b/src/main/orcad/orcad-lifecycle-host.test.ts @@ -0,0 +1,71 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./orcad-browser-startup', () => ({ + startOrcadBrowserProvider: () => ({ ready: Promise.resolve(), stop: async () => {} }) +})) + +import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock' +import { startOrcadWithHost } from './orcad-lifecycle' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function dataRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'orcad-lifecycle-host-')) + roots.push(root) + return root +} + +describe('startOrcadWithHost', () => { + it('names the instance a managed stop must address, and releases its lock on a clean stop', async () => { + const root = dataRoot() + const handle = await startOrcadWithHost( + root, + async () => ({}), + () => {} + ) + expect(readOrcadInstanceLockRecord(handle.instance.lockPath)).toMatchObject({ + pid: handle.instance.pid, + nonce: handle.instance.nonce + }) + await handle.stop() + expect(existsSync(join(root, ORCAD_LOCK_FILE_NAME))).toBe(false) + }) + + it('keeps the instance lock when a runtime writer could not be stopped', async () => { + const root = dataRoot() + const failure = new Error('profile writer still running') + const handle = await startOrcadWithHost( + root, + async (registerCleanup) => { + registerCleanup(() => { + throw failure + }) + return {} + }, + () => {} + ) + await expect(handle.stop()).rejects.toBe(failure) + expect(existsSync(join(root, ORCAD_LOCK_FILE_NAME))).toBe(true) + }) + + it('keeps the instance lock when a quit handler fails', async () => { + const root = dataRoot() + const handle = await startOrcadWithHost( + root, + async () => ({}), + () => { + throw new AggregateError([new Error('handler')], 'orcad_quit_handlers_failed') + } + ) + await expect(handle.stop()).rejects.toThrow('orcad_quit_handlers_failed') + expect(existsSync(join(root, ORCAD_LOCK_FILE_NAME))).toBe(true) + }) +}) diff --git a/src/main/orcad/orcad-lifecycle.ts b/src/main/orcad/orcad-lifecycle.ts index 6121838e2d4..3bd3ad96889 100644 --- a/src/main/orcad/orcad-lifecycle.ts +++ b/src/main/orcad/orcad-lifecycle.ts @@ -1,8 +1,11 @@ import { setRuntimeBrowserCommandsFactory } from '../runtime/runtime-browser-commands-factory' -import { resolveOrcadBrowserProvider } from './orcad-browser-provider' +import { startOrcadBrowserProvider } from './orcad-browser-startup' +import { OrcadRuntimeLifetime, type OrcadRuntimeCleanup } from './orcad-runtime-lifetime' +import type { OrcadManagedStopInstance } from '../../shared/orcad-stop-request' import { acquireOrcadInstanceLock } from './orcad-instance-lock' import { ORCAD_BUNDLED_LAUNCHER_ENV } from './orcad-bundled-runtime' import { resolveOrcadExitCode } from './orcad-exit-code' +import { ORCAD_SHUTDOWN_DEADLINE_MS } from './orcad-stop-deadlines' import { acquireProfileStateRuntimeAdmission, type ProfileStateRuntimeAdmission @@ -19,29 +22,39 @@ function createIdempotentOrcadCleanup(cleanup: () => Promise): () => Promi } } -export const ORCAD_SHUTDOWN_DEADLINE_MS = 15_000 +export { ORCAD_SHUTDOWN_DEADLINE_MS } -/** A launcher and its child can both receive the same process-group or service stop signal. */ +/** True when this call began the stop; false when another source already owns it. */ +export type OrcadShutdownTrigger = (reason: string, onFailed?: () => void) => boolean + +/** + * A launcher and its child can both receive the same process-group or service stop signal. + * Returns the trigger stop-request listeners share, so every source runs one bounded stop. + * `onFailed` runs before a failed or overdue stop exits, so a caller can retract a clean record. + */ export function installOrcadShutdownSignals( stop: () => Promise, deadlineMs = ORCAD_SHUTDOWN_DEADLINE_MS -): void { +): OrcadShutdownTrigger { let stopping = false - const shutdown = (signal: string): void => { + const shutdown: OrcadShutdownTrigger = (signal, onFailed) => { if (stopping) { - return + return false } stopping = true setTimeout(() => { console.error(`orcad: shutdown after ${signal} exceeded ${deadlineMs}ms — exiting`) + onFailed?.() process.exit(1) }, deadlineMs) stop() .then(() => process.exit(0)) .catch((error) => { console.error(`orcad: shutdown after ${signal} failed:`, error) + onFailed?.() process.exit(resolveOrcadExitCode(error)) }) + return true } process.on('SIGINT', () => shutdown('SIGINT')) process.on('SIGTERM', () => shutdown('SIGTERM')) @@ -55,26 +68,26 @@ export function installOrcadShutdownSignals( shutdown('launcher disconnect') } } + return shutdown } export async function startOrcadWithLifecycle( - start: (registerRuntimeCleanup: (cleanup: () => Promise) => void) => Promise, + start: (registerRuntimeCleanup: (cleanup: OrcadRuntimeCleanup) => void) => Promise, cleanupHost: (runtimeCleanupSucceeded: boolean) => Promise ): Promise }> { - let cleanupRuntime = async (): Promise => {} + // Runtime resources stop in reverse registration order before any host resource. + const runtime = new OrcadRuntimeLifetime() const cleanup = createIdempotentOrcadCleanup(async () => { let runtimeCleanupSucceeded = false try { - await cleanupRuntime() + await runtime.stop() runtimeCleanupSucceeded = true } finally { await cleanupHost(runtimeCleanupSucceeded) } }) try { - const handle = await start((nextCleanup) => { - cleanupRuntime = nextCleanup - }) + const handle = await start((nextCleanup) => runtime.add(nextCleanup)) return { ...handle, stop: cleanup } } catch (error) { try { @@ -87,41 +100,40 @@ export async function startOrcadWithLifecycle( } } -/** Keep profile admission until every runtime writer has stopped. */ +/** Keep profile admission and the instance lock until every runtime writer has stopped. */ export async function startOrcadWithHost( userDataPath: string, - start: (registerCleanup: (cleanup: () => Promise) => void) => Promise, + start: (registerCleanup: (cleanup: OrcadRuntimeCleanup) => void) => Promise, runQuitHandlers: () => void -): Promise }> { +): Promise; instance: OrcadManagedStopInstance }> { const instanceLock = acquireOrcadInstanceLock(userDataPath) + const { pid, startedAtMs, nonce } = instanceLock.record + const instance = { pid, startedAtMs, nonce, lockPath: instanceLock.path } let admission: ProfileStateRuntimeAdmission | undefined - let browserProvider: Awaited> | undefined + let browserProvider: ReturnType | undefined return startOrcadWithLifecycle( async (registerCleanup) => { admission = acquireProfileStateRuntimeAdmission(userDataPath) - browserProvider = await resolveOrcadBrowserProvider({ userDataPath }) - const provider = browserProvider - setRuntimeBrowserCommandsFactory(provider?.factory ?? null, { - headless: provider !== null, - ...(provider ? { isAvailable: () => provider.isAvailable() } : {}) - }) - return start(registerCleanup) + // Why not awaited: a desktop sidecar's authorization UI must not hold RPC readiness hostage. + browserProvider = startOrcadBrowserProvider({ userDataPath }) + return { ...(await start(registerCleanup)), instance } }, async (runtimeCleanupSucceeded) => { - try { - await browserProvider?.stop() - } finally { - setRuntimeBrowserCommandsFactory(null) - runQuitHandlers() - try { - // Failed teardown excludes recovery until the process actually exits. - if (runtimeCleanupSucceeded) { - admission?.release() - } - } finally { + // Failed teardown keeps both fences until the process actually exits. + const host = new OrcadRuntimeLifetime(() => { + if (runtimeCleanupSucceeded) { instanceLock.release() } - } + }) + host.add(({ failed }) => { + if (runtimeCleanupSucceeded && !failed) { + admission?.release() + } + }) + host.add(() => runQuitHandlers()) + host.add(() => setRuntimeBrowserCommandsFactory(null)) + host.add(() => browserProvider?.stop()) + await host.stop() } ) } diff --git a/src/main/orcad/orcad-local-serve-selection-entry.ts b/src/main/orcad/orcad-local-serve-selection-entry.ts new file mode 100644 index 00000000000..016ac730159 --- /dev/null +++ b/src/main/orcad/orcad-local-serve-selection-entry.ts @@ -0,0 +1,49 @@ +/** + * `orca serve`'s app-side question, run by the CLI with the app's own executable under + * ELECTRON_RUN_AS_NODE: prepare this machine's orcad slot if it can serve, and print one + * `ORCA_SERVE_RUNTIME` line saying which host to run. Diagnostics go to stderr. + */ +import { join } from 'node:path' +import process from 'node:process' +import { + formatServeRuntimeSelection, + ORCAD_LOCAL_SERVE_SELECTION_FLAGS as FLAGS +} from '../../shared/orcad-local-serve-selection' +import { selectServeRuntime } from './orcad-local-serve-selection' +import { pruneDesktopOrcadArtifactCache } from './orcad-artifact-cache-retention' + +function flagValue(argv: readonly string[], flag: string): string | null { + const index = argv.indexOf(flag) + return index === -1 ? null : (argv[index + 1] ?? null) +} + +async function run(argv: readonly string[]): Promise { + const userDataPath = flagValue(argv, FLAGS.userData) + const appRoot = flagValue(argv, FLAGS.appRoot) + if (!userDataPath || !appRoot) { + throw new Error(`${FLAGS.userData} and ${FLAGS.appRoot} are required`) + } + const selection = await selectServeRuntime({ + env: process.env, + platform: process.platform, + userDataPath, + templateDirs: [ + ...(process.resourcesPath ? [join(process.resourcesPath, 'orcad-template')] : []), + join(appRoot, 'out', 'orcad-template') + ] + }) + // orcad serve never starts the desktop's startup pass, so the slot cache is bounded here too. + await pruneDesktopOrcadArtifactCache( + userDataPath, + selection.kind === 'orcad' ? [selection.version] : [] + ).catch(() => []) + process.stdout.write(`${formatServeRuntimeSelection(selection)}\n`, () => process.exit(0)) +} + +run(process.argv.slice(2)).catch((error: unknown) => { + // Any failure here is a reason to serve on Electron, never to fail `orca serve`. + const reason = `orcad selection failed: ${error instanceof Error ? error.message : String(error)}` + process.stdout.write(`${formatServeRuntimeSelection({ kind: 'electron', reason })}\n`, () => + process.exit(0) + ) +}) diff --git a/src/main/orcad/orcad-local-serve-selection.test.ts b/src/main/orcad/orcad-local-serve-selection.test.ts new file mode 100644 index 00000000000..708c7454050 --- /dev/null +++ b/src/main/orcad/orcad-local-serve-selection.test.ts @@ -0,0 +1,124 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { NODE_RUNTIME_ASSETS } from '../../shared/node-runtime-pin' +import { + ORCAD_NODE_RUNTIME_MARKER_FILENAME, + ORCAD_SERVER_TARGET_FILENAME, + ORCAD_VERSION_FILENAME, + orcadNodeRuntimeRelativePath +} from '../../shared/orcad-artifacts' +import { formatOrcadNativePreflightReport } from '../../shared/orcad-native-preflight-report' +import { SERVE_RUNTIME_ENV } from '../../shared/orcad-local-serve-selection' +import { selectServeRuntime, type ServeRuntimeSelectionInput } from './orcad-local-serve-selection' + +const TARGET = 'linux-x64-glibc' +const SHA = NODE_RUNTIME_ASSETS[TARGET].executableSha256 +let root = '' + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-serve-runtime-')) +}) + +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +/** A materialized slot as the template would leave it: no runtime of its own yet. */ +function slotFixture(): string { + const slot = join(root, 'orcad-artifacts', TARGET, '0.1.0+abc') + mkdirSync(slot, { recursive: true }) + writeFileSync(join(slot, ORCAD_SERVER_TARGET_FILENAME), `${TARGET}\n`) + writeFileSync(join(slot, ORCAD_NODE_RUNTIME_MARKER_FILENAME), `${SHA}\n`) + writeFileSync(join(slot, ORCAD_VERSION_FILENAME), '0.1.0+abc\n') + writeFileSync(join(slot, 'orcad.js'), '') + return slot +} + +function input(overrides: Partial = {}): ServeRuntimeSelectionInput { + const template = join(root, 'orcad-template') + mkdirSync(template, { recursive: true }) + const cachedNode = join(root, 'cached-node') + writeFileSync(cachedNode, '#!/bin/sh\n') + return { + env: {}, + platform: 'linux', + userDataPath: root, + templateDirs: [join(root, 'missing-template'), template], + hostTarget: () => TARGET, + materializeSlot: vi.fn(async () => slotFixture()), + materializeRuntime: vi.fn(async () => cachedNode), + nativePreflight: async () => `${formatOrcadNativePreflightReport('ok', null)}\n`, + ...overrides + } +} + +describe('orca serve runtime selection', () => { + it('stays on Electron, silently, when Electron is asked for', async () => { + const options = input({ env: { [SERVE_RUNTIME_ENV]: 'electron' } }) + expect(await selectServeRuntime(options)).toEqual({ kind: 'electron', reason: null }) + expect(options.materializeSlot).not.toHaveBeenCalled() + }) + + it('serves on orcad by default and when orcad is asked for by name, Windows included', async () => { + for (const env of [{}, { [SERVE_RUNTIME_ENV]: 'orcad' }]) { + for (const platform of ['linux', 'win32'] as const) { + expect(await selectServeRuntime(input({ env, platform }))).toMatchObject({ kind: 'orcad' }) + } + } + }) + + it('runs the local slot on its pinned Node, linked into userData beside it', async () => { + const options = input() + const selection = await selectServeRuntime(options) + const slot = join(root, 'orcad-artifacts', TARGET, '0.1.0+abc') + const runtime = join(slot, ...orcadNodeRuntimeRelativePath(TARGET, SHA)) + expect(selection).toEqual({ + kind: 'orcad', + runtime, + entry: join(slot, 'orcad.js'), + version: '0.1.0+abc' + }) + expect(existsSync(runtime)).toBe(true) + expect(runtime.startsWith(join(root, 'orcad-artifacts'))).toBe(true) + expect(options.materializeSlot).toHaveBeenCalledWith(TARGET, { + templateDir: join(root, 'orcad-template'), + cacheRoot: join(root, 'orcad-artifacts') + }) + }) + + it.each([ + [ + 'an unknown runtime name', + { env: { [SERVE_RUNTIME_ENV]: 'bun' } }, + 'ORCA_SERVE_RUNTIME=bun is neither orcad nor electron' + ], + ['an unsupported host', { hostTarget: () => 'linux-riscv64-glibc' }, 'no orcad build exists'], + ['an install without the template', { templateDirs: [] }, 'carries no orcad template'], + [ + 'an offline first run', + { + materializeRuntime: vi.fn(async (): Promise => { + throw new Error('fetch failed') + }) + }, + 'could not be prepared: fetch failed' + ], + [ + 'a host whose node-pty cannot load', + { + nativePreflight: async () => formatOrcadNativePreflightReport('blocked', 'load_crashed') + }, + 'cannot run terminals here (blocked: load_crashed)' + ], + [ + 'a silent preflight', + { nativePreflight: async () => '' }, + 'did not answer its native preflight' + ] + ])('falls back to Electron on %s and says why', async (_name, overrides, reason) => { + const selection = await selectServeRuntime(input(overrides)) + expect(selection).toEqual({ kind: 'electron', reason: expect.stringContaining(reason) }) + }) +}) diff --git a/src/main/orcad/orcad-local-serve-selection.ts b/src/main/orcad/orcad-local-serve-selection.ts new file mode 100644 index 00000000000..e9fa6230984 --- /dev/null +++ b/src/main/orcad/orcad-local-serve-selection.ts @@ -0,0 +1,145 @@ +/** + * Which host runs `orca serve`: orcad on this machine's packaged slot, or Electron `--serve`. + * App-side on purpose: the CLI runs it through `orcad-local-serve-selection-entry.ts`, so the + * CLI bundle never carries the materializers. + * + * orcad by default; `ORCA_SERVE_RUNTIME=electron` opts out. Anything that stops orcad from + * serving this machine (no slot for the target, no pinned Node, a native module it cannot load, + * a host or packaging path it does not cover yet) falls back to Electron with one stderr line + * saying why. Everything this writes stays inside the desktop's userData (design D7): the slot + * under `orcad-artifacts/`, assembled from the template inside the app bundle. + */ +import { chmodSync, copyFileSync, existsSync, linkSync, mkdirSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { runProcess } from '../../shared/child-process/run-process' +import { NODE_RUNTIME_ASSETS, type ServerTarget } from '../../shared/node-runtime-pin' +import { ORCAD_VERSION_FILENAME, orcadNodeRuntimeRelativePath } from '../../shared/orcad-artifacts' +import { + ORCAD_NATIVE_PREFLIGHT_FLAG, + parseOrcadNativePreflightReport +} from '../../shared/orcad-native-preflight-report' +import { + SERVE_RUNTIME_ELECTRON, + SERVE_RUNTIME_ENV, + type ServeRuntimeSelection +} from '../../shared/orcad-local-serve-selection' +import { resolveBundledOrcadRuntime } from './orcad-bundled-runtime' +import { detectNativeHostAbi, nativeSlotName } from './native-host-abi' +import { materializeOrcadArtifact } from '../ssh/orcad-artifact-materializer' +import { materializeCachedNodeRuntime } from '../ssh/pinned-runtime-materializer' + +const NATIVE_PREFLIGHT_TIMEOUT_MS = 30_000 + +export type ServeRuntimeSelectionInput = { + env: NodeJS.ProcessEnv + platform: NodeJS.Platform + userDataPath: string + templateDirs: readonly string[] + hostTarget?: () => string + materializeSlot?: typeof materializeOrcadArtifact + materializeRuntime?: typeof materializeCachedNodeRuntime + nativePreflight?: (runtime: string, entry: string) => Promise +} + +function isServerTarget(value: string): value is ServerTarget { + return Object.keys(NODE_RUNTIME_ASSETS).includes(value) +} + +function electron(reason: string): ServeRuntimeSelection { + return { kind: 'electron', reason } +} + +export async function selectServeRuntime( + input: ServeRuntimeSelectionInput +): Promise { + const requested = input.env[SERVE_RUNTIME_ENV] + if (requested === SERVE_RUNTIME_ELECTRON) { + return { kind: 'electron', reason: null } + } + if (requested && requested !== 'orcad') { + return electron(`${SERVE_RUNTIME_ENV}=${requested} is neither orcad nor electron`) + } + const target = (input.hostTarget ?? (() => nativeSlotName(detectNativeHostAbi())))() + if (!isServerTarget(target)) { + return electron(`no orcad build exists for this host (${target})`) + } + const templateDir = input.templateDirs.find((candidate) => existsSync(candidate)) + if (!templateDir) { + return electron('this Orca install carries no orcad template') + } + const cacheRoot = join(input.userDataPath, 'orcad-artifacts') + let slotDir: string + try { + slotDir = await (input.materializeSlot ?? materializeOrcadArtifact)(target, { + templateDir, + cacheRoot + }) + await placeSlotRuntime(slotDir, target, cacheRoot, input.materializeRuntime) + } catch (error) { + return electron(`the orcad slot for ${target} could not be prepared: ${errorText(error)}`) + } + let runtime: string | null + try { + runtime = resolveBundledOrcadRuntime(slotDir) + } catch (error) { + return electron(`the orcad slot is incomplete: ${errorText(error)}`) + } + if (!runtime) { + return electron('the orcad slot names no pinned runtime') + } + const entry = join(slotDir, 'orcad.js') + const report = parseOrcadNativePreflightReport( + await (input.nativePreflight ?? runNativePreflight)(runtime, entry).catch(() => '') + ) + if (!report) { + return electron('orcad did not answer its native preflight') + } + if (report.status === 'blocked' || report.status === 'degraded') { + return electron( + `orcad cannot run terminals here (${report.status}: ${report.reason ?? 'unknown'})` + ) + } + return { + kind: 'orcad', + runtime, + entry, + version: readFileSync(join(slotDir, ORCAD_VERSION_FILENAME), 'utf8').trim() + } +} + +/** The slot references its runtime beside it; the cached pinned Node is linked into place. */ +async function placeSlotRuntime( + slotDir: string, + target: ServerTarget, + cacheRoot: string, + materializeRuntime: typeof materializeCachedNodeRuntime = materializeCachedNodeRuntime +): Promise { + const destination = join( + slotDir, + ...orcadNodeRuntimeRelativePath(target, NODE_RUNTIME_ASSETS[target].executableSha256) + ) + if (existsSync(destination)) { + return + } + const cached = await materializeRuntime(target, cacheRoot, { fetcher: fetch }) + mkdirSync(dirname(destination), { recursive: true }) + try { + linkSync(cached, destination) + } catch { + copyFileSync(cached, destination) + chmodSync(destination, 0o755) + } +} + +async function runNativePreflight(runtime: string, entry: string): Promise { + const result = await runProcess({ + program: runtime, + args: [entry, ORCAD_NATIVE_PREFLIGHT_FLAG], + timeoutMs: NATIVE_PREFLIGHT_TIMEOUT_MS + }) + return result.stdout +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/main/orcad/orcad-managed-idle-exit-host.ts b/src/main/orcad/orcad-managed-idle-exit-host.ts new file mode 100644 index 00000000000..c243ee16fb6 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit-host.ts @@ -0,0 +1,140 @@ +/** Binds managed idle exit to this orcad's RPC server, PTY provider and terminal daemon. */ +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import type { OrcadIdleExitEvidence } from './orcad-idle-exit-monitor' +import { + activationFenceExists, + installOrcadManagedIdleExit, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitConfig +} from './orcad-managed-idle-exit' +import { + consumeOrcadIdleStopRecord, + discardOrcadIdleStopRecord, + writeOrcadIdleStopRecord +} from './orcad-idle-stop-record' +import type { OrcadShutdownTrigger } from './orcad-lifecycle' +import type { OrcadIdleStopRecord } from '../../shared/orcad-idle-exit' + +let requestIdleShutdown: OrcadShutdownTrigger | null = null + +/** main binds its shutdown once signal handling exists; the quiet period outlasts that gap. */ +export function bindOrcadIdleShutdown(request: OrcadShutdownTrigger): void { + requestIdleShutdown = request +} + +type OrcadIdleExitRuntimePorts = { + rpc: { readClientActivity(): RuntimeRpcClientActivity } + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + automationsBusy: () => boolean + /** Shutdown stops the monitor first, so a signal stop is never recorded as an idle one. */ + registerCleanup: (cleanup: () => void) => void +} + +/** + * Runs under the instance lock at startup: reads the previous run's idle-stop record before + * anything this run does could be mistaken for it. Inert for an orcad no client launched. + */ +export function beginOrcadIdleExit(userDataPath: string): { + previousIdleStop: OrcadIdleStopRecord | null | undefined + start: (ports: OrcadIdleExitRuntimePorts) => Promise +} { + const config = resolveOrcadManagedIdleExit(process.env) + if (!config) { + return { previousIdleStop: undefined, start: async () => {} } + } + const previousIdleStop = consumeOrcadIdleStopRecord(userDataPath) + if (previousIdleStop) { + console.error(`[orcad] the previous run stopped idle at ${previousIdleStop.stoppedAt}`) + } + const version = process.env.ORCA_VERSION ?? '0.0.0-orcad' + return { + previousIdleStop, + start: (ports) => startOrcadManagedIdleExit({ ...ports, config, userDataPath, version }) + } +} + +async function startOrcadManagedIdleExit( + input: OrcadIdleExitRuntimePorts & { + config: OrcadManagedIdleExitConfig + userDataPath: string + version: string + } +): Promise { + const { getLocalPtyProvider } = await import('../ipc/pty') + const { getDaemonEndpointFacts } = await import('../daemon/daemon-init') + const { countLiveOrcadDaemonSessions, retireOrcadDaemonIfIdle } = + await import('./orcad-daemon-retirement') + const idleRecord = createIdleStopRecordOwnership(input.userDataPath) + const dispose = installOrcadManagedIdleExit({ + config: input.config, + ports: { + readClientActivity: () => input.rpc.readClientActivity(), + listTerminals: () => getLocalPtyProvider().listProcesses(), + countDaemonSessions: countLiveOrcadDaemonSessions, + hasDaemon: () => getDaemonEndpointFacts() !== null, + agentStates: input.agentStates, + hasStagedMigration: input.hasStagedMigration, + automationsBusy: input.automationsBusy, + activationFenceExists + }, + stop: (evidence) => { + void stopForIdle(input, evidence, retireOrcadDaemonIfIdle).finally(() => + idleRecord.requestShutdown(requestIdleShutdown) + ) + } + }) + input.registerCleanup(() => { + dispose() + idleRecord.onShutdown() + }) +} + +/** + * A stop that fails or overruns is not clean, and one another source (a signal, a stop request) + * took over is not idle; the next start must report neither as an idle stop. `onShutdown` runs + * in every stop's cleanup, so a takeover that finishes before the idle request is handled too. + */ +export function createIdleStopRecordOwnership(userDataPath: string): { + requestShutdown(request: OrcadShutdownTrigger | null): void + onShutdown(): void +} { + let idleOwnsStop = false + const discard = (): void => discardOrcadIdleStopRecord(userDataPath) + return { + requestShutdown: (request) => { + // Set first: the stop this starts may run its cleanup before the trigger returns. + idleOwnsStop = true + if (request?.('idle', discard) !== true) { + idleOwnsStop = false + discard() + } + }, + onShutdown: () => { + if (!idleOwnsStop) { + discard() + } + } + } +} + +async function stopForIdle( + input: { userDataPath: string; version: string }, + evidence: OrcadIdleExitEvidence, + retireDaemon: () => Promise<{ retirement: string; reason: string | null }> +): Promise { + console.error(`[orcad] stopping: no client, terminal or job for ${evidence.timeoutMs}ms`) + try { + writeOrcadIdleStopRecord(input.userDataPath, evidence, input.version) + } catch (error) { + console.error('[orcad] could not record the idle stop:', error) + } + // The daemon leaves only if it proves itself empty; a busy one stays up with its terminals. + const outcome = await retireDaemon().catch((error: unknown) => ({ + retirement: 'unverifiable', + reason: String(error) + })) + console.error( + `[orcad] terminal daemon on idle stop: ${outcome.retirement}${outcome.reason ? ` (${outcome.reason})` : ''}` + ) +} diff --git a/src/main/orcad/orcad-managed-idle-exit.test.ts b/src/main/orcad/orcad-managed-idle-exit.test.ts new file mode 100644 index 00000000000..d14532b2ff0 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.test.ts @@ -0,0 +1,139 @@ +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + activationFenceExists, + createOrcadIdleProbes, + resolveOrcadManagedIdleExit, + type OrcadManagedIdleExitPorts +} from './orcad-managed-idle-exit' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +const config = { timeoutMs: 1_000, activationRoot: '/home/u/.orca-remote/.fence' } + +function idlePorts(): OrcadManagedIdleExitPorts { + return { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 0, lastRequestAt: 0 }), + listTerminals: async () => [], + countDaemonSessions: async () => 0, + hasDaemon: () => true, + agentStates: () => [{ state: 'done' }], + hasStagedMigration: () => false, + automationsBusy: () => false, + activationFenceExists: async () => false + } +} + +async function verdicts(ports: OrcadManagedIdleExitPorts): Promise> { + const entries = await Promise.all( + createOrcadIdleProbes(config, ports).map(async (probe) => [probe.name, await probe.read()]) + ) + return Object.fromEntries(entries) +} + +describe('resolveOrcadManagedIdleExit', () => { + it('stays off for a server the user started or paired', () => { + expect(resolveOrcadManagedIdleExit({})).toBeNull() + expect(resolveOrcadManagedIdleExit({ [ORCAD_E2E_IDLE_TIMEOUT_ENV]: '50' })).toBeNull() + }) + + it('matches the relay quiet period for a managed launch', () => { + expect(resolveOrcadManagedIdleExit({ [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f' })).toEqual({ + timeoutMs: ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot: '/f' + }) + expect(ORCAD_IDLE_EXIT_TIMEOUT_MS).toBe(15 * 60_000) + }) + + it.each([ + ['3000', 3_000], + ['0', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['-1', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['1.5', ORCAD_IDLE_EXIT_TIMEOUT_MS], + ['abc', ORCAD_IDLE_EXIT_TIMEOUT_MS], + [String(2 * 60 * 60_000), ORCAD_IDLE_EXIT_TIMEOUT_MS] + ])('accepts only a bounded test timeout (%s)', (raw, expected) => { + expect( + resolveOrcadManagedIdleExit({ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV]: '/f', + [ORCAD_E2E_IDLE_TIMEOUT_ENV]: raw + })?.timeoutMs + ).toBe(expected) + }) +}) + +describe('createOrcadIdleProbes', () => { + it('reads an unused host as idle on every probe', async () => { + expect(await verdicts(idlePorts())).toEqual({ + clients: 'idle', + terminals: 'idle', + agents: 'idle', + migration: 'idle', + automations: 'idle', + activation: 'idle' + }) + }) + + it.each<[string, Partial, string]>([ + [ + 'an open client socket', + { + readClientActivity: () => ({ openConnections: 1, requestsInFlight: 0, lastRequestAt: 0 }) + }, + 'clients' + ], + [ + 'a request still running', + { + readClientActivity: () => ({ openConnections: 0, requestsInFlight: 1, lastRequestAt: 0 }) + }, + 'clients' + ], + ['an in-process terminal', { listTerminals: async () => [{ id: 'pty-1' }] }, 'terminals'], + ['a live daemon session', { countDaemonSessions: async () => 2 }, 'terminals'], + ['a working agent', { agentStates: () => [{ state: 'working' }] }, 'agents'], + ['a staged migration', { hasStagedMigration: () => true }, 'migration'], + ['an enabled or running automation', { automationsBusy: () => true }, 'automations'], + ['a held activation fence', { activationFenceExists: async () => true }, 'activation'] + ])('reads %s as busy', async (_label, override, probe) => { + expect((await verdicts({ ...idlePorts(), ...override }))[probe]).toBe('busy') + }) + + it('treats a daemon that did not answer as unverifiable, not as no terminals', async () => { + const result = await verdicts({ ...idlePorts(), countDaemonSessions: async () => null }) + expect(result.terminals).toBe('unverifiable') + }) + + it('needs only the provider census when this orcad runs without a daemon', async () => { + const result = await verdicts({ + ...idlePorts(), + hasDaemon: () => false, + countDaemonSessions: async () => null + }) + expect(result.terminals).toBe('idle') + }) +}) + +describe('activationFenceExists', () => { + let root: string | null = null + afterEach(() => { + if (root) { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('follows the lock a client holds during an update, not a root an aborted acquire left', async () => { + root = mkdtempSync(join(tmpdir(), 'orcad-fence-')) + const fence = join(root, '.orcad-activation-transaction') + expect(await activationFenceExists(fence)).toBe(false) + mkdirSync(fence) + expect(await activationFenceExists(fence)).toBe(false) + mkdirSync(join(fence, '.install-lock')) + expect(await activationFenceExists(fence)).toBe(true) + }) +}) diff --git a/src/main/orcad/orcad-managed-idle-exit.ts b/src/main/orcad/orcad-managed-idle-exit.ts new file mode 100644 index 00000000000..74229ef2a65 --- /dev/null +++ b/src/main/orcad/orcad-managed-idle-exit.ts @@ -0,0 +1,123 @@ +/** + * What a managed orcad counts as "in use" before an idle stop, and the stop itself. + * + * The stop is the ordinary graceful shutdown, which disconnects from the terminal daemon and + * never shuts it down, so no terminal can be killed by it. Terminals are still a blocker: a + * host with live terminals keeps its server so a returning client finds it serving. + */ +import { stat } from 'node:fs/promises' +import { join } from 'node:path' +import { RELAY_INSTALL_LOCK_NAME } from '../../shared/relay-install-lock-name' +import { hasErrorCode } from '../daemon/daemon-process-inspection' +import type { RuntimeRpcClientActivity } from '../runtime/runtime-rpc/runtime-rpc-shutdown' +import { + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + ORCAD_IDLE_EXIT_TIMEOUT_MS, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' +import { + OrcadIdleExitMonitor, + type OrcadIdleExitEvidence, + type OrcadIdleProbe, + type OrcadIdleVerdict +} from './orcad-idle-exit-monitor' + +export type OrcadManagedIdleExitConfig = { timeoutMs: number; activationRoot: string } + +/** Null for any orcad a client did not launch: a user-started or paired server never idles out. */ +export function resolveOrcadManagedIdleExit( + env: NodeJS.ProcessEnv +): OrcadManagedIdleExitConfig | null { + const activationRoot = env[ORCAD_MANAGED_ACTIVATION_ROOT_ENV] + if (!activationRoot) { + return null + } + return { + timeoutMs: readOrcadE2EIdleTimeoutMs(env) ?? ORCAD_IDLE_EXIT_TIMEOUT_MS, + activationRoot + } +} + +export type OrcadManagedIdleExitPorts = { + readClientActivity: () => RuntimeRpcClientActivity + /** Every terminal the PTY provider knows, daemon-owned or in-process. */ + listTerminals: () => Promise + /** Live daemon sessions across generations; null when a daemon did not answer. */ + countDaemonSessions: () => Promise + hasDaemon: () => boolean + agentStates: () => readonly { state: string }[] + hasStagedMigration: () => boolean + /** An enabled schedule or an unsettled run; nothing would fire either once the host exits. */ + automationsBusy: () => boolean + /** Exists while a client holds the host's activation fence (update, rollback, decommission). */ + activationFenceExists: (root: string) => Promise +} + +export function createOrcadIdleProbes( + config: OrcadManagedIdleExitConfig, + ports: OrcadManagedIdleExitPorts +): OrcadIdleProbe[] { + const verdict = (busy: boolean): OrcadIdleVerdict => (busy ? 'busy' : 'idle') + return [ + { + name: 'clients', + read: () => { + const activity = ports.readClientActivity() + return verdict(activity.openConnections > 0 || activity.requestsInFlight > 0) + } + }, + { + name: 'terminals', + read: async () => { + if ((await ports.listTerminals()).length > 0) { + return 'busy' + } + // Why read the daemon too: it outlives this process and may hold sessions no provider lists. + if (!ports.hasDaemon()) { + return 'idle' + } + const live = await ports.countDaemonSessions() + return live === null ? 'unverifiable' : verdict(live > 0) + } + }, + { + name: 'agents', + read: () => verdict(ports.agentStates().some((entry) => entry.state === 'working')) + }, + { name: 'migration', read: () => verdict(ports.hasStagedMigration()) }, + { name: 'automations', read: () => verdict(ports.automationsBusy()) }, + { + name: 'activation', + read: async () => verdict(await ports.activationFenceExists(config.activationRoot)) + } + ] +} + +/** The client holds the fence only while the lock exists; a bare root is an interrupted acquire. */ +export async function activationFenceExists(root: string): Promise { + try { + await stat(join(root, RELAY_INSTALL_LOCK_NAME)) + return true + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + return false + } + throw error + } +} + +export function installOrcadManagedIdleExit(input: { + config: OrcadManagedIdleExitConfig + ports: OrcadManagedIdleExitPorts + /** Records the clean stop, then runs the same graceful shutdown as SIGTERM. */ + stop: (evidence: OrcadIdleExitEvidence) => void +}): () => void { + const monitor = new OrcadIdleExitMonitor({ + timeoutMs: input.config.timeoutMs, + probes: createOrcadIdleProbes(input.config, input.ports), + lastClientActivityAt: () => input.ports.readClientActivity().lastRequestAt, + onIdle: input.stop + }) + monitor.start() + return () => monitor.stop() +} diff --git a/src/main/orcad/orcad-managed-stop-admission.ts b/src/main/orcad/orcad-managed-stop-admission.ts new file mode 100644 index 00000000000..af4731ec582 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-admission.ts @@ -0,0 +1,34 @@ +import type { OrcadManagedStopRequest } from '../../shared/orcad-stop-request' +import { persistOrcadDaemonRetirementRecord } from './orcad-completed-stop-receipt' +import type { OrcadDaemonRetirement } from './orcad-daemon-retirement' +import { stopOrcadAutomationScheduler } from './orcad-automations' + +// Lazy like the rest of orcad's daemon graph: the entry module must not load it at import time. +async function retireLazily(): Promise { + const { retireOrcadDaemonIfIdle } = await import('./orcad-daemon-retirement') + return retireOrcadDaemonIfIdle() +} + +/** Runs before orcad stops for a managed request; it can record an outcome but never veto. */ +export async function prepareOrcadManagedStop( + request: OrcadManagedStopRequest, + retire: () => Promise = retireLazily, + stopAutomations: () => void = stopOrcadAutomationScheduler +): Promise { + // First, so no dispatch races the census below or writes a run the stop then discards. + stopAutomations() + if (!request.retireIdleDaemon) { + return + } + const outcome = await retire().catch((error: unknown): OrcadDaemonRetirement => ({ + retirement: 'unverifiable', + liveSessions: null, + reason: `Retirement failed: ${error instanceof Error ? error.message : String(error)}` + })) + try { + persistOrcadDaemonRetirementRecord(request, outcome) + } catch (error) { + // The completion command reads a missing record as `unverifiable`. + console.error('[orcad] could not record daemon retirement:', error) + } +} diff --git a/src/main/orcad/orcad-managed-stop-cancel-race.test.ts b/src/main/orcad/orcad-managed-stop-cancel-race.test.ts new file mode 100644 index 00000000000..38b23f24f4c --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-cancel-race.test.ts @@ -0,0 +1,48 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import type * as StopDecision from './orcad-managed-stop-decision' + +const decisions = vi.hoisted(() => ({ read: vi.fn<() => string | null>() })) +vi.mock('./orcad-managed-stop-decision', async (importOriginal) => ({ + ...(await importOriginal()), + readOrcadManagedStopDecision: decisions.read +})) + +import { acquireOrcadInstanceLock } from './orcad-instance-lock' +import { completeOrcadManagedStop } from './orcad-managed-stop-completion' +import { orcadManagedStopRequestPath } from './orcad-managed-stop-request' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +it('withdraws its own request when a cancel wins between the decision check and the write', async () => { + const root = mkdtempSync(join(tmpdir(), 'orcad-stop-race-')) + roots.push(root) + const lock = acquireOrcadInstanceLock(root, { identity: () => 'uid-1000', startedAtMs: () => 5 }) + const { pid, startedAtMs, nonce } = lock.record + const request = { + schemaVersion: 1 as const, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + version: '1.0.0', + runtimeId: 'runtime-1', + instance: { pid, startedAtMs, nonce, lockPath: lock.path } + } + // The cancel lands after the first read, when there is no request file yet for it to remove. + decisions.read.mockReturnValueOnce(null).mockReturnValue('canceled') + + const verdict = await completeOrcadManagedStop(request, { + probeProcess: () => 'alive', + startedAtMs: () => 5, + sleep: async () => {}, + attempts: 1 + }) + + expect(verdict).toBe('live') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(false) +}) diff --git a/src/main/orcad/orcad-managed-stop-cancellation.test.ts b/src/main/orcad/orcad-managed-stop-cancellation.test.ts new file mode 100644 index 00000000000..d4c3c229108 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-cancellation.test.ts @@ -0,0 +1,124 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + ORCAD_CANCEL_MANAGED_STOP_FLAG, + OrcadManagedStopCancellationSchema, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { acquireOrcadInstanceLock } from './orcad-instance-lock' +import { orcadManagedStopRequestPath } from './orcad-managed-stop-request' +import { cancelOrcadManagedStop } from './orcad-managed-stop-cancellation' +import { + claimOrcadManagedStopDecision, + readOrcadManagedStopDecision +} from './orcad-managed-stop-decision' +import { completeOrcadManagedStop } from './orcad-managed-stop-completion' +import { runOrcadManagedStopCancelCommand } from './orcad-managed-stop-command' +import { installOrcadStopRequestListeners } from './orcad-stop-request-listener' + +const roots: string[] = [] +afterEach(() => { + vi.restoreAllMocks() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function running(): OrcadManagedStopRequest { + const root = mkdtempSync(join(tmpdir(), 'orcad-stop-cancel-')) + roots.push(root) + const lock = acquireOrcadInstanceLock(root, { identity: () => 'uid-1000', startedAtMs: () => 5 }) + const { pid, startedAtMs, nonce } = lock.record + return { + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + version: '1.0.0', + runtimeId: 'runtime-1', + instance: { pid, startedAtMs, nonce, lockPath: lock.path } + } +} + +describe('cancelling a managed stop', () => { + it('lets exactly one side decide a transaction', () => { + const request = running() + expect(claimOrcadManagedStopDecision(request, 'canceled')).toBe('canceled') + expect(claimOrcadManagedStopDecision(request, 'dispatched')).toBe('canceled') + expect(readOrcadManagedStopDecision(request)).toBe('canceled') + }) + + it('withdraws a request orcad has not acted on, and removes its file', () => { + const request = running() + writeFileSync(orcadManagedStopRequestPath(request.instance), JSON.stringify(request)) + expect(cancelOrcadManagedStop(request)).toBe('canceled') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(false) + }) + + it('reports dispatched once orcad acted first, and leaves its request in place', () => { + const request = running() + writeFileSync(orcadManagedStopRequestPath(request.instance), JSON.stringify(request)) + expect(claimOrcadManagedStopDecision(request, 'dispatched')).toBe('dispatched') + expect(cancelOrcadManagedStop(request)).toBe('dispatched') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(true) + }) + + it('never removes another transaction pending for the same instance', () => { + const request = running() + const other = { ...request, transactionId: '5a7e1f0c-3b2d-4e6f-9a8b-7c6d5e4f3a2b' } + writeFileSync(orcadManagedStopRequestPath(request.instance), JSON.stringify(other)) + expect(cancelOrcadManagedStop(request)).toBe('canceled') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(true) + }) + + it('does not reissue a cancelled transaction while orcad keeps running', async () => { + const request = running() + cancelOrcadManagedStop(request) + expect( + await completeOrcadManagedStop(request, { + probeProcess: () => 'alive', + startedAtMs: () => 5, + sleep: async () => {} + }) + ).toBe('live') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(false) + }) + + it('keeps orcad running when its listener meets a cancelled request', async () => { + const request = running() + cancelOrcadManagedStop(request) + writeFileSync(orcadManagedStopRequestPath(request.instance), JSON.stringify(request)) + const onRequest = vi.fn() + const report = vi.spyOn(console, 'error').mockImplementation(() => {}) + const root = mkdtempSync(join(tmpdir(), 'orcad-stop-cancel-slot-')) + roots.push(root) + const listener = installOrcadStopRequestListeners(onRequest, { + installRoot: root, + managedStop: { + version: request.version, + runtimeId: request.runtimeId, + instance: request.instance + }, + pollIntervalMs: 10 + }) + await vi.waitFor(() => expect(report).toHaveBeenCalled()) + listener.close() + expect(onRequest).not.toHaveBeenCalled() + expect(String(report.mock.calls[0]?.[1])).toContain('orcad_managed_stop_canceled') + // Left in place, it would make every later transaction's completion answer unverifiable. + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(false) + }) + + it('prints one cancellation line through the command', () => { + const request = running() + const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const cancellation = runOrcadManagedStopCancelCommand([ + ORCAD_CANCEL_MANAGED_STOP_FLAG, + JSON.stringify(request) + ]) + expect(cancellation.outcome).toBe('canceled') + expect( + OrcadManagedStopCancellationSchema.parse(JSON.parse(String(write.mock.calls[0]?.[0]))) + ).toEqual(cancellation) + }) +}) diff --git a/src/main/orcad/orcad-managed-stop-cancellation.ts b/src/main/orcad/orcad-managed-stop-cancellation.ts new file mode 100644 index 00000000000..49c37985e6e --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-cancellation.ts @@ -0,0 +1,34 @@ +/** Withdrawing a managed stop request that the running orcad has not acted on yet. */ +import { rmSync } from 'node:fs' +import type { + OrcadManagedStopCancellation, + OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { claimOrcadManagedStopDecision } from './orcad-managed-stop-decision' +import { + orcadManagedStopRequestPath, + readOrcadManagedStopRequest +} from './orcad-managed-stop-request' + +/** `dispatched` means orcad already began stopping; only its completion can say how it ended. */ +export function cancelOrcadManagedStop( + request: OrcadManagedStopRequest +): OrcadManagedStopCancellation['outcome'] { + const outcome = claimOrcadManagedStopDecision(request, 'canceled') + if (outcome === 'canceled') { + withdrawOrcadManagedStopRequest(request) + } + return outcome +} + +/** Removes the request file only while it still carries this transaction. */ +export function withdrawOrcadManagedStopRequest(request: OrcadManagedStopRequest): void { + const path = orcadManagedStopRequestPath(request.instance) + try { + if (readOrcadManagedStopRequest(path).transactionId === request.transactionId) { + rmSync(path, { force: true }) + } + } catch { + // Absent or another transaction's request: the standing decision already fences this one. + } +} diff --git a/src/main/orcad/orcad-managed-stop-command.ts b/src/main/orcad/orcad-managed-stop-command.ts new file mode 100644 index 00000000000..af11040ed88 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-command.ts @@ -0,0 +1,100 @@ +/** + * `orcad --complete-managed-stop `: one stop, one JSON verdict line on stdout. + * `orcad --cancel-managed-stop `: one cancellation, one JSON outcome line. + * Either takes `--request-file ` in place of the JSON. + * + * Exit 0 means a result was printed — read it, the exit code does not carry it. 64 is a + * malformed invocation; 1 is a failure before any result, which is never evidence of exit. + */ +import { + ORCAD_CANCEL_MANAGED_STOP_FLAG, + ORCAD_COMPLETE_MANAGED_STOP_FLAG, + ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG, + OrcadManagedStopRequestSchema, + type OrcadManagedStopCancellation, + type OrcadManagedStopCompletion, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { cancelOrcadManagedStop } from './orcad-managed-stop-cancellation' +import { readOrcadCompletedStopReceipt } from './orcad-completed-stop-receipt' +import { readOrcadManagedStopRequest } from './orcad-managed-stop-request' +import { ZodError } from 'zod' +import { + completeOrcadManagedStop, + type OrcadManagedStopCompletionOptions +} from './orcad-managed-stop-completion' + +export async function runOrcadManagedStopCommand( + argv: readonly string[], + options: OrcadManagedStopCompletionOptions = {} +): Promise { + const request = parseRequestArgument(argv, ORCAD_COMPLETE_MANAGED_STOP_FLAG) + const verdict = await completeOrcadManagedStop(request, options) + const retirement = + verdict === 'exited' && request.retireIdleDaemon + ? readOrcadCompletedStopReceipt(request)?.retirement + : undefined + const completion: OrcadManagedStopCompletion = { + ...request, + kind: 'orcad_managed_stop_completion', + verdict, + receiptPersisted: verdict === 'exited', + ...(retirement ? { retirement } : {}) + } + process.stdout.write(`${JSON.stringify(completion)}\n`) + return completion +} + +export function runOrcadManagedStopCancelCommand( + argv: readonly string[] +): OrcadManagedStopCancellation { + const request = parseRequestArgument(argv, ORCAD_CANCEL_MANAGED_STOP_FLAG) + const cancellation: OrcadManagedStopCancellation = { + ...request, + kind: 'orcad_managed_stop_cancellation', + outcome: cancelOrcadManagedStop(request) + } + process.stdout.write(`${JSON.stringify(cancellation)}\n`) + return cancellation +} + +function parseRequestArgument(argv: readonly string[], flag: string): OrcadManagedStopRequest { + if (argv[0] !== flag) { + throw new Error('orcad_managed_stop_invalid_arguments') + } + if (argv[1] === ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG) { + if (argv.length !== 3 || !argv[2]) { + throw new Error('orcad_managed_stop_invalid_arguments') + } + return readOrcadManagedStopRequest(argv[2]) + } + if (argv.length !== 2 || !argv[1]) { + throw new Error('orcad_managed_stop_invalid_arguments') + } + return OrcadManagedStopRequestSchema.parse(JSON.parse(argv[1])) +} + +export const ORCAD_MANAGED_STOP_EXIT_VERDICT = 0 +export const ORCAD_MANAGED_STOP_EXIT_FAILED = 1 +export const ORCAD_MANAGED_STOP_EXIT_USAGE = 64 + +export async function runOrcadManagedStopCommandAndExit(argv: readonly string[]): Promise { + let code = ORCAD_MANAGED_STOP_EXIT_VERDICT + try { + if (argv[0] === ORCAD_CANCEL_MANAGED_STOP_FLAG) { + runOrcadManagedStopCancelCommand(argv) + } else { + await runOrcadManagedStopCommand(argv) + } + } catch (error) { + console.error('orcad: managed stop failed:', error) + code = + error instanceof ZodError || + error instanceof SyntaxError || + (error instanceof Error && error.message === 'orcad_managed_stop_invalid_arguments') + ? ORCAD_MANAGED_STOP_EXIT_USAGE + : ORCAD_MANAGED_STOP_EXIT_FAILED + } + // Why exit after the write drains: the caller reads stdout to EOF. + process.stdout.write('', () => process.exit(code)) +} diff --git a/src/main/orcad/orcad-managed-stop-completion.ts b/src/main/orcad/orcad-managed-stop-completion.ts new file mode 100644 index 00000000000..7f3e8015056 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-completion.ts @@ -0,0 +1,136 @@ +/** + * Asking one orcad instance to stop and proving that it did. + * + * Runs as a short-lived command on the execution host. It never signals: it writes the + * instance-bound request the running orcad watches for, then observes until the instance is + * gone. `exited` needs proof — no process with that PID, or a PID whose start time shows it now + * belongs to another process. Anything the host cannot answer is `unverifiable`, never exit. + */ +import { setTimeout as delay } from 'node:timers/promises' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { + OrcadManagedStopRequestSchema, + type OrcadManagedStopInstance, + type OrcadManagedStopRequest, + type OrcadManagedStopVerdict +} from '../../shared/orcad-stop-request' +import { + START_TIME_TOLERANCE_MS, + startTimesWithinTolerance +} from '../daemon/daemon-process-start-time' +import { readOrcadProcessStartedAtMs } from './orcad-process-start-time' +import { hasErrorCode, inspectProcessSignal } from '../daemon/daemon-process-inspection' +import { persistOrcadCompletedStopReceipt } from './orcad-completed-stop-receipt' +import { readOrcadManagedStopDecision } from './orcad-managed-stop-decision' +import { withdrawOrcadManagedStopRequest } from './orcad-managed-stop-cancellation' +import { + orcadInstanceLockNames, + orcadManagedStopRequestPath, + readOrcadManagedStopRequest +} from './orcad-managed-stop-request' +import { + ORCAD_STOP_COMPLETION_POLL_ATTEMPTS, + ORCAD_STOP_COMPLETION_POLL_MS +} from './orcad-stop-deadlines' + +export type OrcadProcessProbe = (pid: number) => 'alive' | 'missing' | 'unverifiable' + +export type OrcadManagedStopCompletionOptions = { + probeProcess?: OrcadProcessProbe + startedAtMs?: (pid: number) => number | null + sleep?: () => Promise + attempts?: number + now?: () => Date +} + +function defaultProbe(pid: number): ReturnType { + const signal = inspectProcessSignal(pid) + // EPERM proves some process holds the PID; it cannot prove ours exited. + return signal === 'occupied' ? 'alive' : signal === 'missing' ? 'missing' : 'unverifiable' +} + +function observeInstance( + instance: OrcadManagedStopInstance, + options: OrcadManagedStopCompletionOptions +): OrcadManagedStopVerdict { + const probe = (options.probeProcess ?? defaultProbe)(instance.pid) + if (probe !== 'alive') { + return probe === 'missing' ? 'exited' : 'unverifiable' + } + const actual = (options.startedAtMs ?? readOrcadProcessStartedAtMs)(instance.pid) + // A reused PID is proof of exit only when both start times are known and disagree; on Windows + // without the addon both stay null, so a live PID is never read as exited. + if ( + actual !== null && + instance.startedAtMs !== null && + !startTimesWithinTolerance(actual, instance.startedAtMs, START_TIME_TOLERANCE_MS) + ) { + return 'exited' + } + return 'live' +} + +/** Writes the request only while the lock still names this instance, then waits for exit. */ +export async function completeOrcadManagedStop( + input: OrcadManagedStopRequest, + options: OrcadManagedStopCompletionOptions = {} +): Promise { + const request = OrcadManagedStopRequestSchema.parse(input) + const attempts = options.attempts ?? ORCAD_STOP_COMPLETION_POLL_ATTEMPTS + if (!Number.isSafeInteger(attempts) || attempts < 1 || attempts > 240) { + throw new Error('orcad_managed_stop_invalid_attempts') + } + const completed = (): 'exited' => { + persistOrcadCompletedStopReceipt(request, (options.now ?? (() => new Date()))()) + return 'exited' + } + let verdict = observeInstance(request.instance, options) + if (verdict !== 'live') { + return verdict === 'exited' ? completed() : verdict + } + if (!lockStillNamesInstance(request.instance)) { + // The process lives but no longer owns the lock it published: it is not ours to address. + return 'unverifiable' + } + if (readOrcadManagedStopDecision(request) === 'canceled') { + // A cancelled transaction is never reissued; its orcad keeps running. + return 'live' + } + const requestPath = orcadManagedStopRequestPath(request.instance) + if (!existingRequestMatches(requestPath, request)) { + return 'unverifiable' + } + if (!writeDurableSecureJsonFile(requestPath, request)) { + throw new Error('orcad_managed_stop_request_permissions_unconfirmed') + } + // A cancel that won between the check above and this write found no file to remove. + if (readOrcadManagedStopDecision(request) === 'canceled') { + withdrawOrcadManagedStopRequest(request) + return 'live' + } + for (let attempt = 0; attempt < attempts; attempt++) { + await (options.sleep ?? (() => delay(ORCAD_STOP_COMPLETION_POLL_MS)))() + verdict = observeInstance(request.instance, options) + if (verdict !== 'live') { + return verdict === 'exited' ? completed() : verdict + } + } + return 'live' +} + +function lockStillNamesInstance(instance: OrcadManagedStopInstance): boolean { + try { + return orcadInstanceLockNames(instance) + } catch { + return false + } +} + +/** A request already present must be this one; another transaction's request is not ours. */ +function existingRequestMatches(path: string, request: OrcadManagedStopRequest): boolean { + try { + return JSON.stringify(readOrcadManagedStopRequest(path)) === JSON.stringify(request) + } catch (error) { + return hasErrorCode(error, 'ENOENT') + } +} diff --git a/src/main/orcad/orcad-managed-stop-decision.ts b/src/main/orcad/orcad-managed-stop-decision.ts new file mode 100644 index 00000000000..a5b61e3d2eb --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-decision.ts @@ -0,0 +1,61 @@ +/** + * Arbitrating a managed request between the orcad acting on it and a client cancelling it. + * + * Both sides race to create one decision file per transaction with a hard link, which fails if + * the file exists. Exactly one wins, and the loser reads the winner's decision, so a cancel can + * never report success for a stop that orcad already began. + */ +import { linkSync, rmSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' +import { + OrcadManagedStopDecisionSchema, + OrcadManagedStopRequestSchema, + type OrcadManagedStopDecision, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { orcadStopReceiptPath, readOrcadStopReceipt } from './orcad-completed-stop-receipt' +import { hasErrorCode } from '../daemon/daemon-process-inspection' + +export type OrcadManagedStopDecisionValue = OrcadManagedStopDecision['decision'] + +export function readOrcadManagedStopDecision( + request: OrcadManagedStopRequest +): OrcadManagedStopDecisionValue | null { + return readOrcadStopReceipt(request, 'decision', OrcadManagedStopDecisionSchema)?.decision ?? null +} + +/** Returns the decision that stands: ours if we created it first, otherwise the other side's. */ +export function claimOrcadManagedStopDecision( + input: OrcadManagedStopRequest, + decision: OrcadManagedStopDecisionValue +): OrcadManagedStopDecisionValue { + const request = OrcadManagedStopRequestSchema.parse(input) + const path = orcadStopReceiptPath(request, 'decision') + const staged = `${path}.${process.pid}.${randomUUID()}.staged` + const record: OrcadManagedStopDecision = { + schemaVersion: 1, + kind: 'orcad_managed_stop_decision', + request, + decision + } + if (!writeDurableSecureJsonFile(staged, record)) { + rmSync(staged, { force: true }) + throw new Error('orcad_managed_stop_decision_permissions_unconfirmed') + } + try { + linkSync(staged, path) + return decision + } catch (error) { + if (!hasErrorCode(error, 'EEXIST')) { + throw error + } + const standing = readOrcadManagedStopDecision(request) + if (!standing) { + throw new Error('orcad_managed_stop_decision_unverifiable') + } + return standing + } finally { + rmSync(staged, { force: true }) + } +} diff --git a/src/main/orcad/orcad-managed-stop-request.ts b/src/main/orcad/orcad-managed-stop-request.ts new file mode 100644 index 00000000000..f7211dd7d7c --- /dev/null +++ b/src/main/orcad/orcad-managed-stop-request.ts @@ -0,0 +1,67 @@ +/** Validating an instance-bound stop request before the running orcad acts on it. */ +import { createHash } from 'node:crypto' +import { lstatSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { + ORCAD_MANAGED_STOP_REQUEST_MAX_BYTES, + ORCAD_MANAGED_STOP_REQUEST_PREFIX, + OrcadManagedStopRequestSchema, + type OrcadManagedStopContext, + type OrcadManagedStopInstance, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { readOrcadInstanceLockRecord } from './orcad-instance-lock' + +/** Keyed by the lock nonce, so a request for a previous instance is never this one's. */ +export function orcadManagedStopRequestPath(instance: OrcadManagedStopInstance): string { + const digest = createHash('sha256').update(instance.nonce).digest('hex') + return join(dirname(instance.lockPath), `${ORCAD_MANAGED_STOP_REQUEST_PREFIX}.${digest}`) +} + +export function readOrcadManagedStopRequest(path: string): OrcadManagedStopRequest { + if (!lstatSync(path).isFile()) { + throw new Error('orcad_managed_stop_request_not_regular_file') + } + const { buffer } = readNodeFileSyncWithinLimit(path, ORCAD_MANAGED_STOP_REQUEST_MAX_BYTES) + return OrcadManagedStopRequestSchema.parse(JSON.parse(buffer.toString('utf8'))) +} + +export function sameOrcadManagedStopInstance( + left: OrcadManagedStopInstance, + right: Pick +): boolean { + return ( + left.pid === right.pid && left.startedAtMs === right.startedAtMs && left.nonce === right.nonce + ) +} + +/** Whether the instance lock still names exactly this instance. */ +export function orcadInstanceLockNames(instance: OrcadManagedStopInstance): boolean { + if (!lstatSync(instance.lockPath).isFile()) { + return false + } + const record = readOrcadInstanceLockRecord(instance.lockPath) + return record !== null && sameOrcadManagedStopInstance(instance, record) +} + +/** Throws unless the request names this running instance and its lock is still ours. */ +export function validateOrcadManagedStopRequest( + context: OrcadManagedStopContext, + requestPath: string +): OrcadManagedStopRequest { + const request = readOrcadManagedStopRequest(requestPath) + if ( + request.version !== context.version || + request.runtimeId !== context.runtimeId || + // Resolved: a Windows client names the lock with `/`, orcad's own path.join with `\`. + resolve(request.instance.lockPath) !== resolve(context.instance.lockPath) || + !sameOrcadManagedStopInstance(request.instance, context.instance) + ) { + throw new Error('orcad_managed_stop_request_identity_mismatch') + } + if (!orcadInstanceLockNames(context.instance)) { + throw new Error('orcad_managed_stop_instance_lock_changed') + } + return request +} diff --git a/src/main/orcad/orcad-managed-stop.test.ts b/src/main/orcad/orcad-managed-stop.test.ts new file mode 100644 index 00000000000..0683a3e48b7 --- /dev/null +++ b/src/main/orcad/orcad-managed-stop.test.ts @@ -0,0 +1,331 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { basename, dirname, join, sep } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + OrcadManagedStopCompletionSchema, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { acquireOrcadInstanceLock, type OrcadInstanceLock } from './orcad-instance-lock' +import { + orcadManagedStopRequestPath, + validateOrcadManagedStopRequest +} from './orcad-managed-stop-request' +import { + completeOrcadManagedStop, + type OrcadManagedStopCompletionOptions +} from './orcad-managed-stop-completion' +import { + orcadStopReceiptPath, + readOrcadCompletedStopReceipt, + readOrcadDaemonRetirementRecord +} from './orcad-completed-stop-receipt' +import { prepareOrcadManagedStop } from './orcad-managed-stop-admission' +import { runOrcadManagedStopCommand } from './orcad-managed-stop-command' +import { + ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS, + ORCAD_SHUTDOWN_DEADLINE_MS, + ORCAD_STOP_COMPLETION_POLL_MS +} from './orcad-stop-deadlines' + +const roots: string[] = [] +afterEach(() => { + vi.restoreAllMocks() + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function running(): { lock: OrcadInstanceLock; request: OrcadManagedStopRequest } { + const root = mkdtempSync(join(tmpdir(), 'orcad-managed-stop-')) + roots.push(root) + const lock = acquireOrcadInstanceLock(root, { + identity: () => 'uid-1000', + version: () => '1.0.0', + startedAtMs: () => 5_000 + }) + const { pid, startedAtMs, nonce } = lock.record + return { + lock, + request: { + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + version: '1.0.0', + runtimeId: 'runtime-1', + instance: { pid, startedAtMs, nonce, lockPath: lock.path } + } + } +} + +const context = ({ version, runtimeId, instance }: OrcadManagedStopRequest) => ({ + version, + runtimeId, + instance +}) + +function options( + overrides: Partial = {} +): OrcadManagedStopCompletionOptions { + return { + probeProcess: () => 'alive', + startedAtMs: () => 5_000, + sleep: async () => {}, + attempts: 3, + now: () => new Date('2026-10-01T00:00:00.000Z'), + ...overrides + } +} + +describe('managed stop requests', () => { + it('keys the request file to the instance, so a previous instance never matches', () => { + const { request } = running() + const other = { ...request.instance, nonce: 'other' } + expect(orcadManagedStopRequestPath(request.instance)).not.toBe( + orcadManagedStopRequestPath(other) + ) + }) + + it('accepts a request a newer client wrote with a field this build does not know', () => { + const { request } = running() + const path = orcadManagedStopRequestPath(request.instance) + writeFileSync( + path, + JSON.stringify({ ...request, futureOption: true, instance: { ...request.instance, x: 1 } }) + ) + expect(validateOrcadManagedStopRequest(context(request), path)).toEqual(request) + }) + + it('accepts only a request naming this runtime, version and instance', () => { + const { request } = running() + const path = orcadManagedStopRequestPath(request.instance) + writeFileSync(path, JSON.stringify(request)) + expect(validateOrcadManagedStopRequest(context(request), path)).toEqual(request) + for (const mismatch of [ + { ...request, version: '0.9.0' }, + { ...request, runtimeId: 'runtime-2' }, + { ...request, instance: { ...request.instance, pid: request.instance.pid + 1 } }, + { + ...request, + instance: { ...request.instance, lockPath: `${request.instance.lockPath}.old` } + } + ]) { + writeFileSync(path, JSON.stringify(mismatch)) + expect(() => validateOrcadManagedStopRequest(context(request), path)).toThrow( + 'identity_mismatch' + ) + } + }) + + it('accepts the same lock path spelled differently, as a Windows client sends it', () => { + const { request } = running() + const path = orcadManagedStopRequestPath(request.instance) + const { lockPath } = request.instance + // `/` separators and a `.` segment: the same file, not the same string. + const respelled = `${dirname(lockPath)}/./${basename(lockPath)}`.replaceAll(sep, '/') + expect(respelled).not.toBe(lockPath) + const sent = { ...request, instance: { ...request.instance, lockPath: respelled } } + writeFileSync(path, JSON.stringify(sent)) + expect(validateOrcadManagedStopRequest(context(request), path)).toEqual(sent) + }) + + it('refuses a request once the instance lock names another holder', () => { + const { lock, request } = running() + const path = orcadManagedStopRequestPath(request.instance) + writeFileSync(path, JSON.stringify(request)) + writeFileSync(lock.path, JSON.stringify({ ...lock.record, nonce: 'successor' })) + expect(() => validateOrcadManagedStopRequest(context(request), path)).toThrow( + 'instance_lock_changed' + ) + }) +}) + +describe('completing a managed stop', () => { + it('writes the request, waits for proven exit, then persists a receipt', async () => { + const { request } = running() + const probes = ['alive', 'alive', 'missing'] as const + let index = 0 + const verdict = await completeOrcadManagedStop( + request, + options({ probeProcess: () => probes[Math.min(index++, probes.length - 1)] }) + ) + expect(verdict).toBe('exited') + expect(JSON.parse(readFileSync(orcadManagedStopRequestPath(request.instance), 'utf8'))).toEqual( + request + ) + expect(readOrcadCompletedStopReceipt(request)).toMatchObject({ + kind: 'orcad_managed_stop_completed', + exitedAt: '2026-10-01T00:00:00.000Z' + }) + }) + + it('treats a reused PID with a different start time as proof of exit', async () => { + const { request } = running() + expect(await completeOrcadManagedStop(request, options({ startedAtMs: () => 99_000 }))).toBe( + 'exited' + ) + }) + + it.each([ + ['an unanswerable probe', { probeProcess: () => 'unverifiable' as const }], + ['a reused PID with no readable start time', { startedAtMs: () => null, attempts: 1 }] + ])('never reports exit for %s', async (_name, overrides) => { + const { request } = running() + const verdict = await completeOrcadManagedStop(request, options(overrides)) + expect(verdict).not.toBe('exited') + expect(existsSync(orcadStopReceiptPath(request, 'completed'))).toBe(false) + }) + + it('reports a still-running instance as live after its attempts', async () => { + const { request } = running() + expect(await completeOrcadManagedStop(request, options())).toBe('live') + }) + + it('waits out daemon retirement plus the shutdown deadline before answering live', async () => { + const { request } = running() + let polls = 0 + const exitAfterMs = 2 * ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS + ORCAD_SHUTDOWN_DEADLINE_MS + const verdict = await completeOrcadManagedStop(request, { + ...options({ attempts: undefined }), + sleep: async () => void polls++, + probeProcess: () => + polls * ORCAD_STOP_COMPLETION_POLL_MS >= exitAfterMs ? 'missing' : 'alive' + }) + expect(verdict).toBe('exited') + }) + + it('does not address a live process that no longer holds the lock it published', async () => { + const { lock, request } = running() + writeFileSync(lock.path, JSON.stringify({ ...lock.record, nonce: 'successor' })) + expect(await completeOrcadManagedStop(request, options())).toBe('unverifiable') + expect(existsSync(orcadManagedStopRequestPath(request.instance))).toBe(false) + }) + + it('does not overwrite another transaction pending for the same instance', async () => { + const { request } = running() + const pending = { ...request, transactionId: '5a7e1f0c-3b2d-4e6f-9a8b-7c6d5e4f3a2b' } + writeFileSync(orcadManagedStopRequestPath(request.instance), JSON.stringify(pending)) + expect(await completeOrcadManagedStop(request, options())).toBe('unverifiable') + }) + + it('prints one completion line through the command', async () => { + const { request } = running() + const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const completion = await runOrcadManagedStopCommand( + ['--complete-managed-stop', JSON.stringify(request)], + options({ probeProcess: () => 'missing' }) + ) + expect(completion).toMatchObject({ verdict: 'exited', receiptPersisted: true }) + expect( + OrcadManagedStopCompletionSchema.parse(JSON.parse(String(write.mock.calls[0]?.[0]))) + ).toEqual(completion) + await expect(runOrcadManagedStopCommand(['--complete-managed-stop'])).rejects.toThrow( + 'invalid_arguments' + ) + }) + + it('reads the request from a staged file instead of argv', async () => { + const { lock, request } = running() + const staged = join(lock.path, '..', 'staged-request.json') + writeFileSync(staged, JSON.stringify(request)) + vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const completion = await runOrcadManagedStopCommand( + ['--complete-managed-stop', '--request-file', staged], + options({ probeProcess: () => 'missing' }) + ) + expect(completion).toMatchObject({ + transactionId: request.transactionId, + verdict: 'exited' + }) + await expect( + runOrcadManagedStopCommand(['--complete-managed-stop', '--request-file']) + ).rejects.toThrow('invalid_arguments') + }) +}) + +describe('managed stops that retire the daemon', () => { + const outcome = (retirement: 'retired' | 'live' | 'unverifiable', liveSessions: number | null) => + vi.fn(async () => ({ retirement, liveSessions, reason: null })) + + it('does not touch the daemon when retirement was not asked for', async () => { + const { request } = running() + const retire = outcome('retired', 0) + await prepareOrcadManagedStop(request, retire) + expect(retire).not.toHaveBeenCalled() + expect(readOrcadDaemonRetirementRecord(request)).toBeNull() + }) + + it.each([ + ['retired', 0], + ['live', 2], + ['unverifiable', null] + ] as const)( + 'still completes the stop and records retirement %s on the receipt', + async (retirement, liveSessions) => { + const { request } = running() + const retireRequest = { ...request, retireIdleDaemon: true as const } + await prepareOrcadManagedStop(retireRequest, outcome(retirement, liveSessions)) + expect(readOrcadDaemonRetirementRecord(retireRequest)).toMatchObject({ + retirement, + liveSessions + }) + expect( + await completeOrcadManagedStop(retireRequest, options({ probeProcess: () => 'missing' })) + ).toBe('exited') + expect(readOrcadCompletedStopReceipt(retireRequest)).toMatchObject({ retirement }) + } + ) + + it('stops the automation scheduler before the daemon census, with or without retirement', async () => { + const { request } = running() + const order: string[] = [] + const retire = vi.fn(async () => { + order.push('retire') + return { retirement: 'retired' as const, liveSessions: 0, reason: null } + }) + const stopAutomations = vi.fn(() => void order.push('automations')) + await prepareOrcadManagedStop({ ...request, retireIdleDaemon: true }, retire, stopAutomations) + expect(order).toEqual(['automations', 'retire']) + + await prepareOrcadManagedStop(request, retire, stopAutomations) + expect(stopAutomations).toHaveBeenCalledTimes(2) + }) + + it('records unverifiable when the retirement attempt itself failed', async () => { + const { request } = running() + const retireRequest = { ...request, retireIdleDaemon: true as const } + await prepareOrcadManagedStop(retireRequest, async () => { + throw new Error('daemon socket closed') + }) + expect(readOrcadDaemonRetirementRecord(retireRequest)).toMatchObject({ + retirement: 'unverifiable' + }) + }) + + it('reports unverifiable when orcad exited without recording retirement', async () => { + const { request } = running() + const retireRequest = { ...request, retireIdleDaemon: true as const } + await completeOrcadManagedStop(retireRequest, options({ probeProcess: () => 'missing' })) + expect(readOrcadCompletedStopReceipt(retireRequest)).toMatchObject({ + retirement: 'unverifiable' + }) + }) + + it('omits retirement from the receipt of a plain managed stop', async () => { + const { request } = running() + await completeOrcadManagedStop(request, options({ probeProcess: () => 'missing' })) + expect(readOrcadCompletedStopReceipt(request)).not.toHaveProperty('retirement') + }) + + it('reports the recorded retirement in the completion line, so a client need not read files', async () => { + const { request } = running() + const retireRequest = { ...request, retireIdleDaemon: true as const } + await prepareOrcadManagedStop(retireRequest, outcome('live', 1)) + vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + expect( + await runOrcadManagedStopCommand( + ['--complete-managed-stop', JSON.stringify(retireRequest)], + options({ probeProcess: () => 'missing' }) + ) + ).toMatchObject({ verdict: 'exited', retirement: 'live' }) + }) +}) diff --git a/src/main/orcad/orcad-migration-manifest-digest.ts b/src/main/orcad/orcad-migration-manifest-digest.ts new file mode 100644 index 00000000000..c70b42527c8 --- /dev/null +++ b/src/main/orcad/orcad-migration-manifest-digest.ts @@ -0,0 +1,26 @@ +import { createHash } from 'node:crypto' +import { + orcadMigrationManifestHashInput, + serializeOrcadMigrationValue, + type OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' + +export function computeOrcadMigrationManifestSha256( + manifest: Omit +): string { + return createHash('sha256').update(orcadMigrationManifestHashInput(manifest)).digest('hex') +} + +/** Pass the manifest as received: a newer peer's unknown fields are part of what it signed. */ +export function assertOrcadMigrationManifestDigest(manifest: unknown): void { + if (typeof manifest !== 'object' || manifest === null || Array.isArray(manifest)) { + throw new Error('orcad_migration_manifest_invalid') + } + const unsigned = Object.fromEntries( + Object.entries(manifest).filter(([key]) => key !== 'manifestSha256') + ) + const digest = createHash('sha256').update(serializeOrcadMigrationValue(unsigned)).digest('hex') + if (!('manifestSha256' in manifest) || digest !== manifest.manifestSha256) { + throw new Error('orcad_migration_manifest_digest_mismatch') + } +} diff --git a/src/main/orcad/orcad-node-slot-fixture.ts b/src/main/orcad/orcad-node-slot-fixture.ts index 007383ab735..73fa3c9c944 100644 --- a/src/main/orcad/orcad-node-slot-fixture.ts +++ b/src/main/orcad/orcad-node-slot-fixture.ts @@ -1,5 +1,5 @@ // Test fixture: a packaged Node slot whose runtime is the real pinned Node, laid out as shipped. -import { existsSync } from 'node:fs' +import { copyFileSync, existsSync, linkSync, mkdirSync, readdirSync } from 'node:fs' import { copyFile, link, mkdir, writeFile } from 'node:fs/promises' import { createRequire } from 'node:module' import type * as NodePty from 'node-pty' @@ -73,6 +73,40 @@ export async function writeNodeSlotFixture( return { slotDir, runtime } } +/** Hard-links the built `out/orcad` slot under `/orcad-test` with its pinned runtime beside it. */ +export function installPackagedOrcadSlotForTests( + root: string, + pinnedNode: string +): { slotDir: string; runtime: string } { + const linkOrCopy = (from: string, to: string): void => { + try { + linkSync(from, to) + } catch { + copyFileSync(from, to) + } + } + const installTree = (source: string, destination: string): void => { + mkdirSync(destination, { recursive: true }) + for (const entry of readdirSync(source, { withFileTypes: true })) { + if (entry.isDirectory()) { + installTree(join(source, entry.name), join(destination, entry.name)) + } else { + linkOrCopy(join(source, entry.name), join(destination, entry.name)) + } + } + } + const slotDir = join(root, 'orcad-test') + installTree(resolve('out/orcad'), slotDir) + const target = hostServerTarget() + const runtime = resolve( + slotDir, + ...orcadNodeRuntimeRelativePath(target, NODE_RUNTIME_ASSETS[target].executableSha256) + ) + mkdirSync(join(runtime, '..'), { recursive: true }) + linkOrCopy(pinnedNode, runtime) + return { slotDir, runtime } +} + /** Comma-separated input groups a lane must have; set by run-node-server-tests.mjs. */ export const ORCA_REQUIRED_TEST_INPUTS_ENV = 'ORCA_REQUIRED_TEST_INPUTS' export type RequiredTestInputGroup = 'artifact' | 'cross-runtime' diff --git a/src/main/orcad/orcad-observability.test.ts b/src/main/orcad/orcad-observability.test.ts index 03b49342523..82fc8d1b997 100644 --- a/src/main/orcad/orcad-observability.test.ts +++ b/src/main/orcad/orcad-observability.test.ts @@ -118,5 +118,8 @@ it('orcad installs the trace file before its runtime and closes it after every q expect(install).toBeLessThan(entry.indexOf('new OrcaRuntimeService(')) const quit = entry.indexOf(' runOrcadQuitHandlers()\n') expect(quit).toBeGreaterThan(-1) - expect(entry.indexOf(' closeOrcadObservability()\n', quit)).toBeGreaterThan(quit) + const close = entry.indexOf(' closeOrcadObservability()\n', quit) + expect(close).toBeGreaterThan(quit) + // A failing quit handler must not skip the close. + expect(entry.slice(quit, close)).toContain('} finally {') }) diff --git a/src/main/orcad/orcad-process-start-time.test.ts b/src/main/orcad/orcad-process-start-time.test.ts new file mode 100644 index 00000000000..50f6df6ff09 --- /dev/null +++ b/src/main/orcad/orcad-process-start-time.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as DaemonProcessStartTime from '../daemon/daemon-process-start-time' + +const { readWindowsProcessCreationTime, getProcessStartedAtMs } = vi.hoisted(() => ({ + readWindowsProcessCreationTime: vi.fn<(pid: number) => number | null>(), + getProcessStartedAtMs: vi.fn<(pid: number) => number | null>() +})) +vi.mock('../windows/windows-process-table', () => ({ readWindowsProcessCreationTime })) +vi.mock('../daemon/daemon-process-start-time', async (importOriginal) => ({ + ...(await importOriginal()), + getProcessStartedAtMs +})) + +import { + orcadProcessStartTimeMatches, + readOrcadProcessStartedAtMs +} from './orcad-process-start-time' + +const platform = process.platform +function onPlatform(value: NodeJS.Platform): void { + Object.defineProperty(process, 'platform', { value, configurable: true }) +} + +afterEach(() => { + onPlatform(platform) + vi.resetAllMocks() +}) + +describe('orcad process start time', () => { + it('reads the addon creation time on Windows, never the POSIX probe', () => { + onPlatform('win32') + readWindowsProcessCreationTime.mockReturnValue(1_700_000_000_123) + expect(readOrcadProcessStartedAtMs(42)).toBe(1_700_000_000_123) + expect(readWindowsProcessCreationTime).toHaveBeenCalledWith(42) + expect(getProcessStartedAtMs).not.toHaveBeenCalled() + }) + + it('keeps the POSIX probe elsewhere', () => { + onPlatform('linux') + getProcessStartedAtMs.mockReturnValue(5_000) + expect(readOrcadProcessStartedAtMs(42)).toBe(5_000) + expect(readWindowsProcessCreationTime).not.toHaveBeenCalled() + }) + + it('tells a reused Windows PID apart, and fails open when the addon cannot answer', () => { + onPlatform('win32') + readWindowsProcessCreationTime.mockReturnValue(9_000_000) + expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(false) + expect(orcadProcessStartTimeMatches(42, 9_000_100)).toBe(true) + readWindowsProcessCreationTime.mockReturnValue(null) + expect(orcadProcessStartTimeMatches(42, 1_000)).toBe(true) + }) +}) diff --git a/src/main/orcad/orcad-process-start-time.ts b/src/main/orcad/orcad-process-start-time.ts new file mode 100644 index 00000000000..9952d8485bb --- /dev/null +++ b/src/main/orcad/orcad-process-start-time.ts @@ -0,0 +1,29 @@ +/** + * A process's start time as orcad's identity records carry it (instance lock, managed stop). + * + * Windows reuses PIDs aggressively and `getProcessStartedAtMs` answers null there, so a PID + * alone would be the whole identity. The process-tree addon orcad's Windows slot stages reads + * the kernel creation time for one PID without a table snapshot or a PowerShell spawn. + * Null still means "cannot say": callers must never read it as proof of exit. + */ +import { + getProcessStartedAtMs, + START_TIME_TOLERANCE_MS, + startTimesWithinTolerance +} from '../daemon/daemon-process-start-time' +import { readWindowsProcessCreationTime } from '../windows/windows-process-table' + +export function readOrcadProcessStartedAtMs(pid: number): number | null { + return process.platform === 'win32' + ? readWindowsProcessCreationTime(pid) + : getProcessStartedAtMs(pid) +} + +/** Fails open on an unknown time on either side: an unprovable mismatch keeps the holder. */ +export function orcadProcessStartTimeMatches(pid: number, expected: number | null): boolean { + return startTimesWithinTolerance( + readOrcadProcessStartedAtMs(pid), + expected, + START_TIME_TOLERANCE_MS + ) +} diff --git a/src/main/orcad/orcad-profile-state-startup.ts b/src/main/orcad/orcad-profile-state-startup.ts index dba09365993..2665f1c1850 100644 --- a/src/main/orcad/orcad-profile-state-startup.ts +++ b/src/main/orcad/orcad-profile-state-startup.ts @@ -2,6 +2,7 @@ import { createProfileStateStoreForStartup } from '../persistence/profile-state/ import type { ProfileStateStoreFactoryResult } from '../persistence/profile-state/profile-state-store-factory' import { ensureActiveOrcaProfile, initOrcaProfilePaths } from '../orca-profiles/profile-index-store' import { initSshHostKeyStoreFile } from '../ssh/ssh-host-key-store' +import { initOrcadHeldFenceTokenFile } from '../ssh/orcad-held-fence-tokens' import { emitOrcadProfileStateAuthoritySelected } from './orcad-profile-state-telemetry' export type OrcadProfileStateProfile = { @@ -27,6 +28,9 @@ export async function createOrcadProfileStateStartup( ): Promise { initOrcaProfilePaths() const profile = ensureActiveOrcaProfile(userDataPath) + // Why a real Store: without one, persistence-backed RPCs throw and `store?.x ?? []` reads answer + // "empty", so a server that pairs and lists nothing looks healthy. As the runtime authority, + // orcad must not load as 'desktop', which would orphan its own scheduled automations. const result = await createProfileStateStoreForStartup({ dataFile: profile.dataFile, databaseFile: profile.stateDatabaseFile, @@ -43,6 +47,7 @@ export async function createOrcadProfileStateStartup( } try { initSshHostKeyStoreFile(profile.dataFile) + initOrcadHeldFenceTokenFile(profile.dataFile) emitOrcadProfileStateAuthoritySelected(authority) return { store: result.store, authority } } catch (error) { diff --git a/src/main/orcad/orcad-push-startup.test.ts b/src/main/orcad/orcad-push-startup.test.ts index 41bba43d4cf..025dc666773 100644 --- a/src/main/orcad/orcad-push-startup.test.ts +++ b/src/main/orcad/orcad-push-startup.test.ts @@ -15,6 +15,7 @@ const state = vi.hoisted(() => ({ controller: null as RuntimeMobileNotificationController | null, registry: null as DeviceRegistry | null, rpcStarted: false, + profileStartupErrors: new Array(), onSettingsChanged: vi.fn(), removeSettingsListener: vi.fn(), startDaemon: vi.fn(async () => {}), @@ -28,12 +29,24 @@ vi.mock('./orcad-app-paths', () => ({ resolveUserDataPath: () => state.root })) vi.mock('./orcad-browser-provider', () => ({ resolveOrcadBrowserProvider: state.browserProvider })) -vi.mock('./orcad-instance-lock', () => ({ acquireOrcadInstanceLock: () => ({ release() {} }) })) +vi.mock('./orcad-instance-lock', () => ({ + acquireOrcadInstanceLock: () => ({ + path: join(state.root, 'orcad.lock'), + record: { pid: process.pid, startedAtMs: null, nonce: 'headless-instance' }, + release() {} + }) +})) vi.mock('./orcad-daemon-supervision', () => ({ startOrcadDaemon: state.startDaemon, stopOrcadDaemon: async () => {} })) vi.mock('./orcad-health', () => ({ collectOrcadHealth: async () => ({}) })) +// The runtime stub has no automation surface; orcad-automations.test.ts covers that wiring. +vi.mock('./orcad-automations', () => ({ + startOrcadAutomations: () => {}, + stopOrcadAutomationScheduler: () => {}, + orcadAutomationsKeepHostBusy: () => false +})) // Why: the real updater would fetch rules from GitHub inside a unit test. vi.mock('../runtime/agent-state-rules/agent-state-rules-live-update', () => ({ startAgentStateRulesLiveUpdates: () => {} @@ -45,21 +58,27 @@ vi.mock('../ipc/pty', () => ({ getSshPtyProvider: () => null })) vi.mock('./orcad-profile-state-startup', () => ({ - createOrcadProfileStateStartup: async () => ({ - store: { - getSettings: () => ({}), - onSettingsChanged: state.onSettingsChanged, - flushFinalOrThrowAsync: async () => {}, - freezeWritesAsync: async () => {} - }, - authority: { - backend: 'sqlite', - classification: 'neither', - authority_mode: 'sqlite-candidate', - runtime: 'orcad', - migrated: false + createOrcadProfileStateStartup: async () => { + const error = state.profileStartupErrors.shift() + if (error) { + throw error } - }) + return { + store: { + getSettings: () => ({}), + onSettingsChanged: state.onSettingsChanged, + flushFinalOrThrowAsync: async () => {}, + freezeWritesAsync: async () => {} + }, + authority: { + backend: 'sqlite', + classification: 'neither', + authority_mode: 'sqlite-candidate', + runtime: 'orcad', + migrated: false + } + } + } })) vi.mock('../orca-profiles/profile-index-store', () => ({ initOrcaProfilePaths() {}, @@ -83,6 +102,8 @@ vi.mock('../runtime/orca-runtime', () => ({ rehydrateClientHostedBrowserPages() {} async refreshRestoredOrchestrationAuthority() {} async reconcileLegacyWorkerTerminals() {} + async stopLegacyWorkerTerminalRecovery() {} + syncWindowGraph() {} setMobilePushRegistrar( registrar: Parameters[0] ) { @@ -135,6 +156,7 @@ beforeEach(() => { afterEach(() => { rmSync(state.root, { recursive: true, force: true }) + state.profileStartupErrors.length = 0 vi.clearAllMocks() }) @@ -197,13 +219,32 @@ it('starts push after RPC identity is available and stops dispatch on shutdown', it('releases admission when host setup fails before a runtime exists', async () => { state.root = mkdtempSync(join(tmpdir(), 'orca-headless-setup-failure-')) - state.browserProvider.mockRejectedValueOnce(new Error('browser setup failed')) + state.profileStartupErrors.push(new Error('profile startup failed')) const { startOrcad } = await import('./orcad-entry') - await expect(startOrcad()).rejects.toThrow('browser setup failed') + await expect(startOrcad()).rejects.toThrow('profile startup failed') expect(readdirSync(profileStateAccessPaths(state.root).participants)).toEqual([]) acquireProfileStateMaintenance(state.root).release() }) +it('serves RPC without waiting for browser discovery', async () => { + state.root = mkdtempSync(join(tmpdir(), 'orca-headless-browser-pending-')) + let finishDiscovery!: () => void + state.browserProvider.mockReturnValueOnce( + new Promise((resolve) => { + finishDiscovery = () => resolve(null) + }) + ) + const { startOrcad } = await import('./orcad-entry') + const host = await startOrcad({ noPairing: true, json: true }) + expect(host.managedStop).toMatchObject({ + runtimeId: 'headless-runtime', + instance: { pid: process.pid, nonce: 'headless-instance' } + }) + finishDiscovery() + await host.stop() + expect(readdirSync(profileStateAccessPaths(state.root).participants)).toEqual([]) +}) + it('unsubscribes settings when daemon startup fails after hook setup', async () => { state.root = mkdtempSync(join(tmpdir(), 'orca-headless-daemon-failure-')) state.startDaemon.mockRejectedValueOnce(new Error('daemon setup failed')) diff --git a/src/main/orcad/orcad-runtime-entry-artifacts.test.ts b/src/main/orcad/orcad-runtime-entry-artifacts.test.ts new file mode 100644 index 00000000000..17c85a8e9ec --- /dev/null +++ b/src/main/orcad/orcad-runtime-entry-artifacts.test.ts @@ -0,0 +1,68 @@ +// Every separately built entry orcad starts by filename (a worker thread or a forked child) must +// ship in its slot: the bundle names it, the runtime looks for it beside orcad.js, and a missing +// file only shows up on a host as a feature that never starts. +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterAll, expect, it } from 'vitest' +import { runProcessSync } from '../../shared/child-process/run-process' +import { orcadArtifactFilenames } from '../../shared/orcad-artifacts' + +const REPO_ROOT = join(__dirname, '..', '..', '..') +// The desktop's built entries follow this naming; orcad resolves them by these literal names. +const RUNTIME_ENTRY_FILENAME = /[A-Za-z0-9_.-]+-(?:entry|worker)\.c?js/g +/** + * Named in orcad's bundle but not shipped yet. Each is a known gap, not an exemption: shipping + * one removes it here, and this list may only shrink. + */ +const KNOWN_UNSHIPPED_ENTRIES = new Set([ + 'session-scanner-service-entry.js', + 'wsl-transcript-fs-process-entry.js' +]) +const directory = mkdtempSync(join(tmpdir(), 'orcad-runtime-entries-')) + +afterAll(() => { + rmSync(directory, { recursive: true, force: true }) +}) + +/** orcad.js and every child it ships, built as build-orcad.mjs builds them. */ +function buildOrcadBundles(): string[] { + const builder = pathToFileURL(join(REPO_ROOT, 'config/scripts/orcad-entry-build.mjs')).href + const script = ` + import { build } from 'esbuild' + import { basename, join } from 'node:path' + import * as entries from ${JSON.stringify(builder)} + const out = ${JSON.stringify(directory)} + await entries.buildOrcadEntry(join(out, 'orcad.js')) + await Promise.all(Object.values(entries.ORCAD_CHILD_ENTRY_POINTS).map((entry) => build({ + entryPoints: [entry], bundle: true, platform: 'node', target: 'node18', format: 'cjs', + outfile: join(out, basename(entry).replace(/\\.ts$/, '.js')), + external: entries.ORCAD_EXTERNAL_MODULES, plugins: [entries.externalNativeAddons], + logLevel: 'error' + })))` + const built = runProcessSync({ + program: process.execPath, + args: ['--input-type=module', '-e', script], + cwd: REPO_ROOT, + env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }, + timeoutMs: 120_000 + }) + expect(built.code, built.stderr.slice(0, 2_000)).toBe(0) + return readdirSync(directory).map((file) => readFileSync(join(directory, file), 'utf8')) +} + +it('ships every worker and child entry orcad loads at runtime', () => { + const named = new Set( + buildOrcadBundles().flatMap((text) => text.match(RUNTIME_ENTRY_FILENAME) ?? []) + ) + const shipped = new Set(orcadArtifactFilenames('linux-x64-glibc')) + const missing = [...named].filter( + (filename) => !shipped.has(filename) && !KNOWN_UNSHIPPED_ENTRIES.has(filename) + ) + expect(missing, 'add these to ORCAD_ARTIFACTS and build them in build-orcad.mjs').toEqual([]) + // A gap that ships, or that orcad stops naming, must leave the known list. + for (const filename of KNOWN_UNSHIPPED_ENTRIES) { + expect(named.has(filename) && !shipped.has(filename), filename).toBe(true) + } +}) diff --git a/src/main/orcad/orcad-runtime-lifetime.test.ts b/src/main/orcad/orcad-runtime-lifetime.test.ts new file mode 100644 index 00000000000..110e7d2876b --- /dev/null +++ b/src/main/orcad/orcad-runtime-lifetime.test.ts @@ -0,0 +1,81 @@ +import { expect, it, vi } from 'vitest' +import { OrcadRuntimeLifetime } from './orcad-runtime-lifetime' + +it('stops in reverse acquisition order and releases the instance lock last', async () => { + const events: string[] = [] + const lifetime = new OrcadRuntimeLifetime(() => { + events.push('lock') + }) + lifetime.add(() => { + events.push('browser') + }) + lifetime.add(async () => { + events.push('daemon') + }) + lifetime.add(() => { + events.push('rpc') + }) + const stopping = lifetime.stop() + expect(lifetime.stop()).toBe(stopping) + expect(() => lifetime.add(() => {})).toThrow('lifetime_stopping') + await stopping + expect(events).toEqual(['rpc', 'daemon', 'browser', 'lock']) + await lifetime.stop() + expect(events).toHaveLength(4) +}) + +it('awaits pending cleanup before stopping dependencies or releasing the lock', async () => { + const release = vi.fn() + const lifetime = new OrcadRuntimeLifetime(release) + const dependency = vi.fn() + let finish!: () => void + lifetime.add(dependency) + lifetime.add( + () => + new Promise((resolve) => { + finish = resolve + }) + ) + const stopping = lifetime.stop() + await Promise.resolve() + expect(release).not.toHaveBeenCalled() + expect(dependency).not.toHaveBeenCalled() + finish() + await stopping + expect(dependency).toHaveBeenCalledOnce() + expect(release).toHaveBeenCalledOnce() +}) + +it('attempts every cleanup but retains the lock if any resource cannot stop', async () => { + const release = vi.fn() + const lifetime = new OrcadRuntimeLifetime(release) + const finalCleanup = vi.fn() + const first = new Error('rpc failed') + const second = new Error('browser failed') + lifetime.add(finalCleanup) + lifetime.add(() => { + throw second + }) + lifetime.add(async () => { + throw first + }) + const stopping = lifetime.stop() + await expect(stopping).rejects.toMatchObject({ errors: [first, second] }) + expect(finalCleanup).toHaveBeenCalledOnce() + expect(release).not.toHaveBeenCalled() + expect(lifetime.stop()).toBe(stopping) +}) + +it('tells later cleanups that an earlier one failed, and rethrows a single failure as-is', async () => { + const states: boolean[] = [] + const failure = new Error('profile writer still running') + const lifetime = new OrcadRuntimeLifetime() + lifetime.add(({ failed }) => { + states.push(failed) + }) + lifetime.add(() => { + throw failure + }) + await expect(lifetime.stop()).rejects.toBe(failure) + expect(states).toEqual([true]) +}) diff --git a/src/main/orcad/orcad-runtime-lifetime.ts b/src/main/orcad/orcad-runtime-lifetime.ts new file mode 100644 index 00000000000..e73de829323 --- /dev/null +++ b/src/main/orcad/orcad-runtime-lifetime.ts @@ -0,0 +1,42 @@ +export type OrcadRuntimeCleanup = (state: { failed: boolean }) => void | Promise + +/** + * Stops runtime resources in reverse acquisition order, then releases profile ownership. + * + * Every cleanup runs even after an earlier one fails, but a failed teardown never releases: + * a writer that may still be running cannot hand the data root to a second orcad. + */ +export class OrcadRuntimeLifetime { + private readonly cleanups: OrcadRuntimeCleanup[] = [] + private stopping?: Promise + + constructor(private readonly release: () => void = () => {}) {} + + add(cleanup: OrcadRuntimeCleanup): void { + if (this.stopping) { + throw new Error('orcad_runtime_lifetime_stopping') + } + this.cleanups.push(cleanup) + } + + stop(): Promise { + this.stopping ??= Promise.resolve().then(async () => { + const errors: unknown[] = [] + for (const cleanup of this.cleanups.splice(0).toReversed()) { + try { + await cleanup({ failed: errors.length > 0 }) + } catch (error) { + errors.push(error) + } + } + if (errors.length === 1) { + throw errors[0] + } + if (errors.length > 1) { + throw new AggregateError(errors, 'orcad_runtime_cleanup_failed') + } + this.release() + }) + return this.stopping + } +} diff --git a/src/main/orcad/orcad-runtime-native-preflight.test.ts b/src/main/orcad/orcad-runtime-native-preflight.test.ts index d53ed523dc2..f3edacde03d 100644 --- a/src/main/orcad/orcad-runtime-native-preflight.test.ts +++ b/src/main/orcad/orcad-runtime-native-preflight.test.ts @@ -4,7 +4,9 @@ import type { WatcherProcessCallback, WatcherProcessHooks } from '../ipc/parcel-watcher-process-subscription' +import { PtySpawnHealthTimeoutError } from '../daemon/pty-subprocess/spawn-preflight' import { preflightOrcadNativeRuntime } from './orcad-runtime-native-preflight' +import { WindowsProcessTableTimeoutError } from '../windows/windows-process-table-timeout-error' const fixture = vi.hoisted(() => ({ temp: vi.fn(), @@ -12,17 +14,27 @@ const fixture = vi.hoisted(() => ({ available: vi.fn(), startTime: vi.fn(), rows: vi.fn(), + creationTime: vi.fn(), subscribe: vi.fn(), unsubscribe: vi.fn(), dispose: vi.fn(), write: vi.fn(), remove: vi.fn() })) -vi.mock('../daemon/pty-subprocess/spawn-preflight', () => ({ runPtySpawnHealthProbe: fixture.pty })) +vi.mock('../daemon/pty-subprocess/spawn-preflight', () => ({ + PTY_SPAWN_HEALTH_TIMEOUT_MS: 4_000, + PtySpawnHealthTimeoutError: class extends Error { + constructor(timeoutMs: number) { + super(`PTY spawn health check timed out after ${timeoutMs}ms`) + } + }, + runPtySpawnHealthProbe: fixture.pty +})) vi.mock('../windows/windows-process-table', () => ({ isWindowsProcessTableAvailable: fixture.available, isWindowsProcessStartTimeAvailable: fixture.startTime, - readWindowsProcessIdentityTableFresh: fixture.rows + readWindowsProcessIdentityTableFresh: fixture.rows, + readWindowsProcessCreationTime: fixture.creationTime })) vi.mock('node:fs/promises', () => ({ mkdtemp: fixture.temp, @@ -77,11 +89,67 @@ describe('bundled native readiness', () => { ) }) + // A loaded Windows runner timed the whole-table snapshot out, and startup failed readiness. + it('identifies itself by one PID when the process-table snapshot times out', async () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('win32') + fixture.rows.mockRejectedValue( + new WindowsProcessTableTimeoutError('windows process table timed out') + ) + fixture.creationTime.mockReturnValue(Date.now() - 1_000) + await expect(preflightOrcadNativeRuntime({ nativeFeatures: false })).resolves.toBeUndefined() + expect(fixture.creationTime).toHaveBeenCalledWith(process.pid) + }) + + it('still fails when neither the snapshot nor the one-PID query can identify it', async () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('win32') + fixture.rows.mockRejectedValue( + new WindowsProcessTableTimeoutError('windows process table timed out') + ) + fixture.creationTime.mockReturnValue(null) + await expect(preflightOrcadNativeRuntime({ nativeFeatures: false })).rejects.toThrow( + 'windows process table timed out' + ) + }) + + it('never falls back for an unreadable table, only for a slow one', async () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('win32') + fixture.rows.mockRejectedValue(new Error('windows process table is unreadable')) + fixture.creationTime.mockReturnValue(Date.now() - 1_000) + await expect(preflightOrcadNativeRuntime({ nativeFeatures: false })).rejects.toThrow( + 'windows process table is unreadable' + ) + expect(fixture.creationTime).not.toHaveBeenCalled() + }) + it('does not admit a failed PTY in explicit qualification', async () => { fixture.pty.mockRejectedValue(new Error('PTY spawn health check timed out')) await expect(preflightOrcadNativeRuntime()).rejects.toThrow('PTY spawn health check timed out') }) + it('gives a cold first Windows PTY spawn a longer budget', async () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('win32') + await preflightOrcadNativeRuntime() + expect(fixture.pty).toHaveBeenCalledExactlyOnceWith(15_000) + }) + + it('retries a timed-out PTY probe once with the normal budget', async () => { + fixture.pty.mockRejectedValueOnce(new PtySpawnHealthTimeoutError(15_000)) + await preflightOrcadNativeRuntime() + expect(fixture.pty.mock.calls).toEqual([[4_000], [4_000]]) + }) + + it('fails when the retry also times out', async () => { + fixture.pty.mockRejectedValue(new PtySpawnHealthTimeoutError(4_000)) + await expect(preflightOrcadNativeRuntime()).rejects.toThrow('timed out after 4000ms') + expect(fixture.pty).toHaveBeenCalledTimes(2) + }) + + it('does not retry a PTY that spawned and failed', async () => { + fixture.pty.mockRejectedValue(new Error('PTY spawn health check exited with code 1')) + await expect(preflightOrcadNativeRuntime()).rejects.toThrow('exited with code 1') + expect(fixture.pty).toHaveBeenCalledOnce() + }) + it('awaits actual watcher delivery and unsubscribe before disposing temporary state', async () => { await preflightOrcadNativeRuntime() expect(fixture.pty).toHaveBeenCalledOnce() diff --git a/src/main/orcad/orcad-runtime-native-preflight.ts b/src/main/orcad/orcad-runtime-native-preflight.ts index 0510bc3c15d..cc9cbff9c1f 100644 --- a/src/main/orcad/orcad-runtime-native-preflight.ts +++ b/src/main/orcad/orcad-runtime-native-preflight.ts @@ -1,15 +1,24 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { runPtySpawnHealthProbe } from '../daemon/pty-subprocess/spawn-preflight' +import { + PtySpawnHealthTimeoutError, + PTY_SPAWN_HEALTH_TIMEOUT_MS, + runPtySpawnHealthProbe +} from '../daemon/pty-subprocess/spawn-preflight' import { WatcherProcessSupervisor } from '../ipc/parcel-watcher-process-supervisor' import { resolveWatcherProcessEntryPath } from '../ipc/parcel-watcher-entry-path' import { resolveOrcadInstallRoot } from './orcad-app-paths' import { isWindowsProcessTableAvailable, isWindowsProcessStartTimeAvailable, + readWindowsProcessCreationTime, readWindowsProcessIdentityTableFresh } from '../windows/windows-process-table' +import { WindowsProcessTableTimeoutError } from '../windows/windows-process-table-timeout-error' + +// A cold first conpty spawn on a slow (arm64, AV-scanned) Windows host can outlast the steady-state budget. +const WINDOWS_FIRST_PTY_PROBE_TIMEOUT_MS = 15_000 /** The candidate process owns disposable PTY and watcher probes before it touches user state. */ export async function preflightOrcadNativeRuntime( @@ -22,7 +31,7 @@ export async function preflightOrcadNativeRuntime( if (options.nativeFeatures === false) { return } - await runPtySpawnHealthProbe() + await probePtySpawn() const directory = await mkdtemp(join(tmpdir(), 'orca-native-ready-')) const supervisor = new WatcherProcessSupervisor({ entryPath: resolveWatcherProcessEntryPath(resolveOrcadInstallRoot(), false), @@ -70,14 +79,47 @@ export async function preflightOrcadNativeRuntime( } } +/** Retries once only after a timeout; a spawn error or non-zero exit fails immediately. */ +async function probePtySpawn(): Promise { + const firstTimeoutMs = + process.platform === 'win32' ? WINDOWS_FIRST_PTY_PROBE_TIMEOUT_MS : PTY_SPAWN_HEALTH_TIMEOUT_MS + try { + await runPtySpawnHealthProbe(firstTimeoutMs) + } catch (error) { + if (!(error instanceof PtySpawnHealthTimeoutError)) { + throw error + } + await runPtySpawnHealthProbe(PTY_SPAWN_HEALTH_TIMEOUT_MS) + } +} + async function preflightWindowsProcessIdentity(): Promise { if (!isWindowsProcessTableAvailable() || !isWindowsProcessStartTimeAvailable()) { throw new Error('The bundled Windows process table must support process creation times') } - const rows = await readWindowsProcessIdentityTableFresh() - const self = rows.find((row) => row.pid === process.pid) - const created = self?.creationTimeMs - if (created === undefined || !Number.isFinite(created) || created <= 0 || created > Date.now()) { + let created: number | null | undefined + try { + const rows = await readWindowsProcessIdentityTableFresh() + created = rows.find((row) => row.pid === process.pid)?.creationTimeMs + } catch (error) { + // Only slowness falls back: an unreadable table (EDR hook, restricted token) must still + // fail qualification, or every later liveness verdict on this host reads unverifiable. + if (!(error instanceof WindowsProcessTableTimeoutError)) { + throw error + } + // The addon's one-PID query proves this runtime can identify a process without a snapshot. + created = readWindowsProcessCreationTime(process.pid) + if (created === null) { + throw error + } + } + if ( + created === undefined || + created === null || + !Number.isFinite(created) || + created <= 0 || + created > Date.now() + ) { throw new Error('The bundled Windows process table could not identify this process') } } diff --git a/src/main/orcad/orcad-serve-parity.integration.test.ts b/src/main/orcad/orcad-serve-parity.integration.test.ts new file mode 100644 index 00000000000..99ac6746dfa --- /dev/null +++ b/src/main/orcad/orcad-serve-parity.integration.test.ts @@ -0,0 +1,226 @@ +/** + * `orca serve` on orcad, against the packaged slot: the native preflight the launcher asks + * first, desktop-serve flag parity on stdout (the readiness contract), and the shared-profile + * refusal while the desktop app holds the profile. + */ +import { existsSync, mkdtempSync, realpathSync, writeFileSync } from 'node:fs' +import { tmpdir, userInfo } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { ORCAD_NODE_RUNTIME_MARKER_FILENAME } from '../../shared/orcad-artifacts' +import { + ORCAD_NATIVE_PREFLIGHT_FLAG, + parseOrcadNativePreflightReport +} from '../../shared/orcad-native-preflight-report' +import { ORCAD_LOCK_FILE_NAME } from './orcad-instance-lock' +import { resolveBundledOrcadRuntime } from './orcad-bundled-runtime' +import { skipForMissingInputs } from './orcad-node-slot-fixture' +import { buildDaemonSessionClient } from './orcad-daemon-session-client-fixture' +import { + killAndAwaitExit, + killChildAndWait, + killProfileDaemons, + removeTestRoot +} from './orcad-daemon-teardown-fixture' + +const slotDir = resolve('out/orcad') +const runtime = existsSync(join(slotDir, ORCAD_NODE_RUNTIME_MARKER_FILENAME)) + ? resolveBundledOrcadRuntime(slotDir) + : null +const skip = skipForMissingInputs('artifact', runtime ? [] : ['a Node orcad slot in out/orcad']) +const roots: string[] = [] + +afterEach(async () => { + for (const root of roots.splice(0)) { + // The terminal daemon outlives orcad by design; it must be gone before its profile is. + await killProfileDaemons(root) + await removeTestRoot(root) + } +}) + +/** Without Vitest's markers: daemon-entry.js does not start its server under VITEST. */ +function serveEnv(userData: string): NodeJS.ProcessEnv { + return { + ...Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith('VITEST') && key !== 'NODE_ENV') + ), + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_DISABLE_MACOS_LOGIN_SHELL: '1', + ORCA_USER_DATA: userData + } +} + +function profile(): string { + const root = mkdtempSync(join(tmpdir(), 'orcad-serve-parity-')) + roots.push(root) + return root +} + +describe.skipIf(skip)('orca serve on orcad', () => { + it('answers the launcher native preflight with one report line', async () => { + const result = await runProcess({ + program: runtime!, + args: [join(slotDir, 'orcad.js'), ORCAD_NATIVE_PREFLIGHT_FLAG], + timeoutMs: 30_000 + }) + expect(result.code, result.stderr).toBe(0) + expect(parseOrcadNativePreflightReport(result.stdout)?.status).toMatch(/^(ok|unverifiable)$/u) + }) + + it('prints only the recipe line, exactly as Electron serve does', async () => { + const projectRoot = profile() + const child = spawnProcess({ + program: runtime!, + args: [ + join(slotDir, 'orcad.js'), + '--bind', + '127.0.0.1', + '--port', + '0', + '--recipe-json', + '--project-root', + projectRoot + ], + env: serveEnv(profile()), + timeoutMs: null + }) + let stdout = '' + let stderr = '' + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8'))) + try { + await new Promise((resolveLine, reject) => { + const timer = setTimeout(() => reject(new Error(`no recipe line: ${stderr}`)), 120_000) + child.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString('utf8') + if (stdout.includes('\n')) { + clearTimeout(timer) + resolveLine() + } + }) + child.once('exit', (code) => { + clearTimeout(timer) + reject(new Error(`orcad exited ${String(code)}: ${stderr}`)) + }) + }) + // Anything after the recipe line would break a reader of this contract. + await new Promise((settle) => setTimeout(settle, 1_000)) + } finally { + await killChildAndWait(child) + } + const lines = stdout.split('\n').filter(Boolean) + expect(lines, stderr).toHaveLength(1) + expect(JSON.parse(lines[0]!)).toEqual({ + schemaVersion: 1, + pairingCode: expect.any(String), + projectRoot + }) + }, 150_000) + + it('refuses with exit 78 while the desktop app holds the profile', async () => { + const userData = profile() + writeFileSync( + join(userData, ORCAD_LOCK_FILE_NAME), + JSON.stringify({ + pid: process.pid, + startedAtMs: null, + identity: process.platform === 'win32' ? userInfo().username : String(process.getuid?.()), + version: 'desktop-test', + acquiredAt: new Date().toISOString(), + nonce: 'desktop', + role: 'desktop' + }) + ) + const result = await runProcess({ + program: runtime!, + args: [join(slotDir, 'orcad.js'), '--bind', '127.0.0.1', '--port', '0', '--json'], + env: serveEnv(userData), + timeoutMs: 60_000 + }) + expect(result.code).toBe(78) + expect(result.stdout).toBe('') + expect(result.stderr).toContain('The Orca desktop app') + }, 90_000) + + // A short /tmp root keeps the POSIX daemon socket path legal; Windows uses a named pipe. + it('keeps a live terminal across a serve restart on the shared profile (D7)', async () => { + const userData = realpathSync( + mkdtempSync(join(process.platform === 'win32' ? tmpdir() : '/tmp', 'orca-serve-d7-')) + ) + roots.push(userData) + const client = join(userData, 'daemon-client.cjs') + await buildDaemonSessionClient(client) + const session = async (op: 'create' | 'attach', marker: string) => { + const result = await runProcess({ + program: runtime!, + args: [client, op, join(userData, 'daemon'), 'serve-d7', marker, userData], + env: serveEnv(userData), + timeoutMs: 30_000 + }) + expect(result.code, result.stderr).toBe(0) + return JSON.parse(result.stdout.trim().split('\n').at(-1) ?? '{}') + } + const first = await serveOnce(userData) + let daemonPid: number | null = first + try { + const created = await session('create', 'BEFORE') + expect(created).toMatchObject({ isReattach: false, output: true }) + await stopServe(userData) + // The same daemon, in the same \`/daemon\` Electron serve uses, is adopted. + expect(await serveOnce(userData)).toBe(first) + expect(await session('attach', 'AFTER')).toEqual({ + pid: created.pid, + isReattach: true, + output: true + }) + } finally { + await stopServe(userData) + if (daemonPid) { + await killAndAwaitExit([daemonPid]) + daemonPid = null + } + } + }, 300_000) +}) + +const running = new Map>() + +/** Starts `orca serve --json` on orcad and returns the terminal daemon pid its readiness names. */ +async function serveOnce(userData: string): Promise { + const child = spawnProcess({ + program: runtime!, + args: [join(slotDir, 'orcad.js'), '--bind', '127.0.0.1', '--port', '0', '--json'], + env: serveEnv(userData), + timeoutMs: null + }) + running.set(userData, child) + let stdout = '' + let stderr = '' + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8'))) + const line = await new Promise((resolveLine, reject) => { + const timer = setTimeout(() => reject(new Error(`no readiness: ${stderr}`)), 120_000) + child.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString('utf8') + const newline = stdout.indexOf('\n') + if (newline !== -1) { + clearTimeout(timer) + resolveLine(stdout.slice(0, newline)) + } + }) + }) + const daemon = JSON.parse(line).health?.terminalDaemon + expect(daemon?.state, stderr).toBe('live') + return daemon.pid +} + +/** SIGTERM is serve's graceful stop on POSIX; it leaves the daemon running by design. */ +async function stopServe(userData: string): Promise { + const child = running.get(userData) + running.delete(userData) + if (!child || child.exitCode !== null) { + return + } + const exited = new Promise((settle) => child.once('exit', settle)) + child.kill('SIGTERM') + await exited +} diff --git a/src/main/orcad/orcad-serve-readiness.ts b/src/main/orcad/orcad-serve-readiness.ts new file mode 100644 index 00000000000..2148f02291a --- /dev/null +++ b/src/main/orcad/orcad-serve-readiness.ts @@ -0,0 +1,55 @@ +import type { ServeReadiness } from '../server/serve-readiness' +import type { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc' +import { resolveAdvertisedPairingEndpoint } from '../runtime/pairing-endpoint' +import { renderServePairingQr } from '../server/serve-pairing-output' +import type { OrcadHealth } from './orcad-health' +import type { OrcadOptions } from './orcad-entry' + +/** The readiness payload orcad publishes once its RPC transport is listening. */ +export async function buildOrcadServeReadiness(input: { + options: OrcadOptions + runtimeId: string + rpc: Pick + collectHealth: () => Promise +}): Promise { + const { options, rpc } = input + const boundEndpoint = rpc.getWebSocketEndpoint() + const advertised = boundEndpoint + ? resolveAdvertisedPairingEndpoint(boundEndpoint, options.pairingAddress) + : null + const offer = options.noPairing + ? ({ + available: false, + reason: 'disabled_by_operator', + guidance: 'Restart without --no-pairing to create a client pairing offer.' + } as const) + : rpc.createPairingOffer({ + address: options.pairingAddress, + name: `${options.mobilePairing ? 'Mobile' : 'CLI'} ${new Date().toLocaleDateString()}`, + scope: options.mobilePairing ? 'mobile' : 'runtime' + }) + + return { + runtimeId: input.runtimeId, + boundEndpoint, + advertisedEndpoint: advertised?.ok ? advertised.endpoint : null, + // Why 'settled': the WSL CLI reconciliation barrier is a desktop-launch concern. + // orcad never runs it, so there is no pending repair a client could race. + managedWslCliReconciliation: 'settled', + pairing: offer.available + ? { + available: true, + url: offer.pairingUrl, + endpoint: offer.endpoint, + deviceId: offer.deviceId, + webClientUrl: offer.webClientUrl, + scope: options.mobilePairing ? 'mobile' : 'runtime', + qr: options.mobilePairing ? await renderServePairingQr(offer.pairingUrl) : null + } + : offer, + // Why in the readiness payload: this is the one message a supervisor and a deploy + // transaction both read, and a green orcad with a dead daemon is exactly the + // looks-healthy-but-useless state they must not activate. + health: await input.collectHealth() + } +} diff --git a/src/main/orcad/orcad-stop-deadlines.ts b/src/main/orcad/orcad-stop-deadlines.ts new file mode 100644 index 00000000000..faaf6d1ae56 --- /dev/null +++ b/src/main/orcad/orcad-stop-deadlines.ts @@ -0,0 +1,11 @@ +/** Kept dependency-free: the short-lived stop-completion command imports these too. */ +export const ORCAD_SHUTDOWN_DEADLINE_MS = 15_000 +export const ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS = 5_000 + +export const ORCAD_STOP_COMPLETION_POLL_MS = 250 +// Retirement (request + session count, each bounded) runs before the shutdown deadline starts. +const STOP_COMPLETION_BUDGET_MS = + 2 * ORCAD_DAEMON_RETIREMENT_TIMEOUT_MS + ORCAD_SHUTDOWN_DEADLINE_MS + 5_000 +export const ORCAD_STOP_COMPLETION_POLL_ATTEMPTS = Math.ceil( + STOP_COMPLETION_BUDGET_MS / ORCAD_STOP_COMPLETION_POLL_MS +) diff --git a/src/main/orcad/orcad-stop-request-listener.test.ts b/src/main/orcad/orcad-stop-request-listener.test.ts new file mode 100644 index 00000000000..dc98966f6c4 --- /dev/null +++ b/src/main/orcad/orcad-stop-request-listener.test.ts @@ -0,0 +1,164 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' +import { acquireOrcadInstanceLock } from './orcad-instance-lock' +import { orcadManagedStopRequestPath } from './orcad-managed-stop-request' +import { + installOrcadStopRequestListeners, + type OrcadStopRequestListener +} from './orcad-stop-request-listener' + +const roots: string[] = [] +const listeners: OrcadStopRequestListener[] = [] +afterEach(() => { + vi.restoreAllMocks() + for (const listener of listeners.splice(0)) { + listener.close() + } + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +function directory(): string { + const root = mkdtempSync(join(tmpdir(), 'orcad-stop-listener-')) + roots.push(root) + return root +} + +function managedContext() { + const lock = acquireOrcadInstanceLock(directory(), { identity: () => 'uid-1000' }) + const { pid, startedAtMs, nonce } = lock.record + return { + version: '1.0.0', + runtimeId: 'runtime-1', + instance: { pid, startedAtMs, nonce, lockPath: lock.path } + } +} + +function listen( + onRequest: () => void, + options: Parameters[1] +) { + const listener = installOrcadStopRequestListeners(onRequest, { pollIntervalMs: 10, ...options }) + listeners.push(listener) + return listener +} + +describe('orcad stop-request listeners', () => { + it('consumes a slot request written before the listener started', () => { + const installRoot = directory() + writeFileSync(join(installRoot, ORCAD_STOP_REQUEST_FILENAME), '') + const onRequest = vi.fn() + listen(onRequest, { installRoot }) + expect(onRequest).toHaveBeenCalledOnce() + expect(existsSync(join(installRoot, ORCAD_STOP_REQUEST_FILENAME))).toBe(false) + }) + + it('picks up a slot request written later, once', async () => { + const installRoot = directory() + const onRequest = vi.fn() + listen(onRequest, { installRoot }) + writeFileSync(join(installRoot, ORCAD_STOP_REQUEST_FILENAME), '') + await vi.waitFor(() => expect(onRequest).toHaveBeenCalledOnce()) + writeFileSync(join(installRoot, ORCAD_STOP_REQUEST_FILENAME), '') + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(onRequest).toHaveBeenCalledOnce() + }) + + it('stops after preparation even when preparation fails', async () => { + const managedStop = managedContext() + const onRequest = vi.fn() + const report = vi.spyOn(console, 'error').mockImplementation(() => {}) + listen(onRequest, { + installRoot: directory(), + managedStop, + beforeManagedStop: async () => { + throw new Error('daemon unreachable') + } + }) + writeFileSync( + orcadManagedStopRequestPath(managedStop.instance), + JSON.stringify({ + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + retireIdleDaemon: true, + ...managedStop + }) + ) + await vi.waitFor(() => expect(onRequest).toHaveBeenCalledOnce()) + expect(report).toHaveBeenCalledWith( + '[orcad] managed stop preparation failed:', + expect.any(Error) + ) + }) + + it('stops on a valid managed request and keeps it as evidence', async () => { + const managedStop = managedContext() + const onRequest = vi.fn() + listen(onRequest, { installRoot: directory(), managedStop }) + const path = orcadManagedStopRequestPath(managedStop.instance) + writeFileSync( + path, + JSON.stringify({ + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + ...managedStop + }) + ) + await vi.waitFor(() => expect(onRequest).toHaveBeenCalledOnce()) + expect(existsSync(path)).toBe(true) + }) + + it('acts once on a managed request written before the listeners were installed', async () => { + const managedStop = managedContext() + writeFileSync( + orcadManagedStopRequestPath(managedStop.instance), + JSON.stringify({ + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + ...managedStop + }) + ) + const onRequest = vi.fn() + const prepare = vi.fn(async () => {}) + const report = vi.spyOn(console, 'error').mockImplementation(() => {}) + listen(onRequest, { installRoot: directory(), managedStop, beforeManagedStop: prepare }) + await vi.waitFor(() => expect(onRequest).toHaveBeenCalledOnce()) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(prepare).toHaveBeenCalledOnce() + expect(onRequest).toHaveBeenCalledOnce() + expect(report).not.toHaveBeenCalled() + }) + + it('ignores a managed request for another runtime and reports it once', async () => { + const managedStop = managedContext() + const onRequest = vi.fn() + const report = vi.spyOn(console, 'error').mockImplementation(() => {}) + listen(onRequest, { installRoot: directory(), managedStop }) + writeFileSync( + orcadManagedStopRequestPath(managedStop.instance), + JSON.stringify({ + schemaVersion: 1, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + ...managedStop, + runtimeId: 'runtime-2' + }) + ) + await vi.waitFor(() => expect(report).toHaveBeenCalled()) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(onRequest).not.toHaveBeenCalled() + expect(report).toHaveBeenCalledOnce() + }) + + it('stops watching once closed', async () => { + const installRoot = directory() + const onRequest = vi.fn() + listen(onRequest, { installRoot }).close() + writeFileSync(join(installRoot, ORCAD_STOP_REQUEST_FILENAME), '') + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(onRequest).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/orcad/orcad-stop-request-listener.ts b/src/main/orcad/orcad-stop-request-listener.ts new file mode 100644 index 00000000000..1bde6a3a65a --- /dev/null +++ b/src/main/orcad/orcad-stop-request-listener.ts @@ -0,0 +1,149 @@ +/** + * Watching for stop requests addressed to this orcad. + * + * Two listeners share one shutdown: the plain slot request (`.orcad-stop-request` beside + * `orcad.js`, consumed by unlinking it) and the instance-bound managed request in the data + * root, which is validated and kept as evidence until the completion command proves exit. + */ +import { unlinkSync, watch, type FSWatcher } from 'node:fs' +import { basename, dirname, join } from 'node:path' +import { + ORCAD_STOP_REQUEST_FILENAME, + type OrcadManagedStopContext, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { + orcadManagedStopRequestPath, + validateOrcadManagedStopRequest +} from './orcad-managed-stop-request' +import { claimOrcadManagedStopDecision } from './orcad-managed-stop-decision' +import { withdrawOrcadManagedStopRequest } from './orcad-managed-stop-cancellation' +import { hasErrorCode } from '../daemon/daemon-process-inspection' + +export type OrcadStopRequestListener = { close(): void } + +const DEFAULT_POLL_INTERVAL_MS = 1_000 + +/** `consume` returns the request that should stop orcad, or null; a missing file throws ENOENT. */ +function listenForRequest( + requestPath: string, + consume: (path: string) => T | null, + onRequest: (request: T) => void, + pollIntervalMs: number +): OrcadStopRequestListener { + let closed = false + let lastFailure: string | null = null + const check = (): void => { + if (closed) { + return + } + let request: T | null + try { + request = consume(requestPath) + } catch (error) { + if (!hasErrorCode(error, 'ENOENT')) { + // Polling retries every interval; report each distinct refusal once. + const failure = error instanceof Error ? error.message : String(error) + if (failure !== lastFailure) { + lastFailure = failure + console.error(`[orcad] refused stop request at ${requestPath}:`, error) + } + } + return + } + if (request === null) { + return + } + closed = true + stop() + onRequest(request) + } + let watcher: FSWatcher | null = null + try { + watcher = watch(dirname(requestPath), (_event, changed) => { + if (!changed || basename(String(changed)) === basename(requestPath)) { + check() + } + }) + watcher.on('error', (error) => console.error('[orcad] stop-request watcher failed:', error)) + watcher.unref() + } catch (error) { + // The poll below still delivers requests; a watcher only makes them prompt. + console.error('[orcad] stop-request watcher could not start:', error) + } + const poll = setInterval(check, pollIntervalMs) + poll.unref() + const stop = (): void => { + watcher?.close() + clearInterval(poll) + } + // Covers a request written before this listener was installed. + check() + return { + close: () => { + closed = true + stop() + } + } +} + +export function installOrcadStopRequestListeners( + onRequest: () => void, + options: { + installRoot: string + managedStop?: OrcadManagedStopContext + /** Runs once for a validated managed request before the stop; it cannot prevent it. */ + beforeManagedStop?: (request: OrcadManagedStopRequest) => Promise + pollIntervalMs?: number + } +): OrcadStopRequestListener { + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS + // Declared first: a listener's initial check can consume a request and close before returning. + const listeners: OrcadStopRequestListener[] = [] + const close = (): void => { + for (const listener of listeners) { + listener.close() + } + } + listeners.push( + listenForRequest( + join(options.installRoot, ORCAD_STOP_REQUEST_FILENAME), + (path) => { + unlinkSync(path) + return true + }, + () => onRequest(), + pollIntervalMs + ) + ) + const managedStop = options.managedStop + if (managedStop) { + const prepare = options.beforeManagedStop ?? (async () => {}) + listeners.push( + listenForRequest( + orcadManagedStopRequestPath(managedStop.instance), + (path) => { + const request = validateOrcadManagedStopRequest(managedStop, path) + // A cancelled request is never acted on; keep listening for the next transaction. + if (claimOrcadManagedStopDecision(request, 'dispatched') === 'canceled') { + // A cancelled request left behind would block every later transaction's request. + withdrawOrcadManagedStopRequest(request) + throw new Error('orcad_managed_stop_canceled') + } + return request + }, + (request) => { + close() + // Preparation is best effort: whatever it reports, the stop proceeds. + void prepare(request) + .catch((error: unknown) => + console.error('[orcad] managed stop preparation failed:', error) + ) + .finally(onRequest) + }, + pollIntervalMs + ) + ) + } + return { close } +} diff --git a/src/main/orcad/orcad-stop-request-shutdown.integration.test.ts b/src/main/orcad/orcad-stop-request-shutdown.integration.test.ts new file mode 100644 index 00000000000..bf0aed4f78f --- /dev/null +++ b/src/main/orcad/orcad-stop-request-shutdown.integration.test.ts @@ -0,0 +1,112 @@ +/** + * The packaged orcad stops through its slot's stop-request file within the shutdown deadline. + * + * On Windows this is the only graceful stop: a signal there is TerminateProcess, which skips + * the flush and leaves the instance lock. So every lane, Windows included, proves the real + * server notices the file, shuts down cleanly (exit 0, lock released) before the 15 s + * deadline, and published an instance lock carrying a real process start time. + */ +import { existsSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { spawnProcess } from '../../shared/child-process/run-process' +import { ORCAD_NODE_RUNTIME_MARKER_FILENAME } from '../../shared/orcad-artifacts' +import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' +import { + killChildAndWait, + killProfileDaemons, + removeTestRoot +} from './orcad-daemon-teardown-fixture' +import { parseOrcadReadinessOutput } from '../ssh/orcad-remote-launch' +import { ORCAD_LOCK_FILE_NAME, readOrcadInstanceLockRecord } from './orcad-instance-lock' +import { ORCAD_SHUTDOWN_DEADLINE_MS } from './orcad-lifecycle' +import { + installPackagedOrcadSlotForTests, + locatePinnedNodeForTests, + skipForMissingInputs +} from './orcad-node-slot-fixture' + +const pinnedNode = locatePinnedNodeForTests() +const skip = skipForMissingInputs('artifact', [ + ...(pinnedNode ? [] : ['the pinned Node (ORCA_PINNED_NODE or out/runtimes)']), + ...(existsSync(join('out/orcad', ORCAD_NODE_RUNTIME_MARKER_FILENAME)) + ? [] + : ['a Node orcad slot in out/orcad (pnpm build:orcad)']) +]) + +let root = '' +const children: ReturnType[] = [] + +/** Without Vitest's markers: daemon-entry.js does not start its server under VITEST. */ +function hostEnv(userData: string): NodeJS.ProcessEnv { + return { + ...Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith('VITEST') && key !== 'NODE_ENV') + ), + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_DISABLE_MACOS_LOGIN_SHELL: '1', + ORCA_VERSION: '0.0.0-stop-request-test', + ORCA_USER_DATA: userData + } +} + +afterEach(async () => { + for (const child of children.splice(0)) { + await killChildAndWait(child) + } + if (root) { + // The terminal daemon deliberately outlives orcad; it must be gone before its profile is. + await killProfileDaemons(join(root, 'data')) + await removeTestRoot(root) + } +}) + +describe.skipIf(skip)('packaged orcad stop request', () => { + it(`stops cleanly within ${ORCAD_SHUTDOWN_DEADLINE_MS} ms of its slot's stop-request file`, async () => { + root = mkdtempSync(join(tmpdir(), 'orcad-stop-request-')) + const { slotDir, runtime } = installPackagedOrcadSlotForTests(root, pinnedNode!) + const userData = join(root, 'data') + const launchedAt = Date.now() + const child = spawnProcess({ + program: runtime, + args: [join(slotDir, 'orcad.js'), '--json', '--bind', '127.0.0.1', '--port', '0'], + cwd: slotDir, + env: hostEnv(userData), + timeoutMs: null + }) + children.push(child) + let stdout = '' + let stderr = '' + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString('utf8'))) + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8'))) + const exited = new Promise((resolve) => child.once('exit', resolve)) + + await vi.waitFor( + () => { + const parsed = parseOrcadReadinessOutput(stdout) + if (parsed.state !== 'ready') { + throw new Error(`orcad not ready (${parsed.state}): ${stderr.slice(-2000)}`) + } + expect(parsed.readiness.health?.stopRequests).toBe(1) + }, + { timeout: 90_000, interval: 250 } + ) + + const lock = readOrcadInstanceLockRecord(join(userData, ORCAD_LOCK_FILE_NAME)) + expect(lock?.pid).toBe(child.pid) + // Windows: the process-tree addon's creation time; POSIX: /proc or ps. Never null here. + expect(lock?.startedAtMs).toEqual(expect.any(Number)) + expect(Math.abs(lock!.startedAtMs! - launchedAt)).toBeLessThan(30_000) + + const requestedAt = Date.now() + writeFileSync(join(slotDir, ORCAD_STOP_REQUEST_FILENAME), '') + const code = await exited + const elapsedMs = Date.now() - requestedAt + expect(code, stderr.slice(-2000)).toBe(0) + // The listener polls once a second when the watcher misses the write. + expect(elapsedMs).toBeLessThan(ORCAD_SHUTDOWN_DEADLINE_MS) + expect(existsSync(join(userData, ORCAD_LOCK_FILE_NAME))).toBe(false) + expect(existsSync(join(slotDir, ORCAD_STOP_REQUEST_FILENAME))).toBe(false) + }, 150_000) +}) diff --git a/src/main/orcad/orcad-terminal-census.test.ts b/src/main/orcad/orcad-terminal-census.test.ts new file mode 100644 index 00000000000..3cd403079dc --- /dev/null +++ b/src/main/orcad/orcad-terminal-census.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest' +import type { DaemonSessionInfo } from '../daemon/types' +import { collectOrcadTerminalCensus } from './orcad-terminal-census' + +function session(createdAt: number, protocolVersion = 7): DaemonSessionInfo { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the census reads only createdAt and protocolVersion. + return { sessionId: `s-${createdAt}`, createdAt, protocolVersion } as DaemonSessionInfo +} + +describe('orcad terminal census', () => { + it('counts live sessions, those since activation, and their single owning protocol', async () => { + await expect( + collectOrcadTerminalCensus( + 100, + async () => [session(50), session(100), session(150)], + async () => 0 + ) + ).resolves.toEqual({ + liveSessions: 3, + startedSinceActivation: 2, + daemonProtocolVersion: 7, + inProcessSessions: 0 + }) + }) + + it('reports zero, not unknown, for an answered empty daemon', async () => { + await expect( + collectOrcadTerminalCensus( + 100, + async () => [], + async () => 0 + ) + ).resolves.toEqual({ + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: null, + inProcessSessions: 0 + }) + }) + + it('leaves the protocol unknown when sessions span daemon generations', async () => { + const census = await collectOrcadTerminalCensus( + 0, + async () => [session(1, 7), session(2, 6)], + async () => 0 + ) + expect(census.daemonProtocolVersion).toBeNull() + expect(census.liveSessions).toBe(2) + }) + + it('leaves the since-activation count unknown when a session has no creation time', async () => { + const census = await collectOrcadTerminalCensus( + 0, + async () => [session(0)], + async () => 0 + ) + expect(census).toMatchObject({ liveSessions: 1, startedSinceActivation: null }) + }) + + it.each([ + ['no inventory', async () => null], + [ + 'a failed inventory', + async () => { + throw new Error('daemon unreachable') + } + ] + ])('reads %s as unverifiable, never as zero', async (_label, list) => { + await expect(collectOrcadTerminalCensus(0, list, async () => 0)).resolves.toEqual({ + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null, + inProcessSessions: null + }) + }) + + it('counts terminals a degraded daemon left running in orcad itself', async () => { + // Empty daemon, one in-process agent: a restart would kill it, so the census must not read 0. + await expect( + collectOrcadTerminalCensus( + 100, + async () => [], + async () => 1 + ) + ).resolves.toEqual({ + liveSessions: 1, + startedSinceActivation: null, + daemonProtocolVersion: null, + inProcessSessions: 1 + }) + }) + + it('reads an unlistable in-process provider as unverifiable', async () => { + const census = await collectOrcadTerminalCensus( + 0, + async () => [], + async () => { + throw new Error('local provider failed') + } + ) + expect(census.liveSessions).toBeNull() + }) +}) diff --git a/src/main/orcad/orcad-terminal-census.ts b/src/main/orcad/orcad-terminal-census.ts new file mode 100644 index 00000000000..4dca703de12 --- /dev/null +++ b/src/main/orcad/orcad-terminal-census.ts @@ -0,0 +1,49 @@ +/** The terminal census a managed orcad reports to the client planning an update or stop. */ +import { + countInProcessFallbackTerminals, + listLiveDaemonSessionsWithProtocol +} from '../daemon/daemon-provider-state' +import type { DaemonSessionInfo } from '../daemon/types' +import type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' + +const UNVERIFIABLE: OrcadTerminalCensus = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null, + inProcessSessions: null +} + +export async function collectOrcadTerminalCensus( + activatedAt: number, + listSessions: () => Promise = listLiveDaemonSessionsWithProtocol, + countInProcess: () => Promise = countInProcessFallbackTerminals +): Promise { + // Degraded mode runs fresh terminals in orcad itself; a restart kills those too. + let inventory: [DaemonSessionInfo[] | null, number] + try { + inventory = await Promise.all([listSessions(), countInProcess()]) + } catch { + return UNVERIFIABLE + } + const [sessions, inProcess] = inventory + if (!sessions) { + return UNVERIFIABLE + } + const timestampsKnown = sessions.every( + (session) => Number.isFinite(session.createdAt) && session.createdAt > 0 + ) + const protocols = new Set(sessions.map((session) => session.protocolVersion)) + const [protocol] = protocols + return { + liveSessions: sessions.length + inProcess, + // In-process terminals carry no creation time to compare against activation. + startedSinceActivation: + timestampsKnown && inProcess === 0 + ? sessions.filter((session) => session.createdAt >= activatedAt).length + : null, + // Why one protocol only: sessions split across daemon generations have no single owner to + // check an incoming build against, so that reads as unverifiable. + daemonProtocolVersion: protocols.size === 1 && protocol !== undefined ? protocol : null, + inProcessSessions: inProcess + } +} diff --git a/src/main/orcad/orcad-windows-conpty-breakaway.integration.test.ts b/src/main/orcad/orcad-windows-conpty-breakaway.integration.test.ts new file mode 100644 index 00000000000..77164eb2cac --- /dev/null +++ b/src/main/orcad/orcad-windows-conpty-breakaway.integration.test.ts @@ -0,0 +1,110 @@ +/** + * ConPTY from a process started the way a Windows SSH host starts orcad: the slot's + * process-tree addon `spawnOutsideJob` (CREATE_BREAKAWAY_FROM_JOB | CREATE_NO_WINDOW, stdio to + * files, no inherited pipes). orcad's terminals live or die on node-pty working there, with no + * visible console and no parent session, so this spawns a shell, writes to it, reads its output + * and sees it exit from inside such a process. + */ +import { createRequire } from 'node:module' +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { quoteWindowsArgument } from '../../shared/child-process/windows-command-line' +import { + ORCAD_NODE_PTY_DIR, + ORCAD_WINDOWS_PROCESS_TREE_FILENAME +} from '../../shared/orcad-artifacts' +import { removeTreeSync } from '../../shared/windows-transient-lock-removal' +import { + installPackagedOrcadSlotForTests, + locatePinnedNodeForTests, + skipForMissingInputs +} from './orcad-node-slot-fixture' + +const pinnedNode = locatePinnedNodeForTests() +const windows = process.platform === 'win32' +const skip = skipForMissingInputs( + 'artifact', + windows + ? [ + ...(pinnedNode ? [] : ['the pinned Node (ORCA_PINNED_NODE or out/runtimes)']), + ...(existsSync(join('out/orcad', ORCAD_WINDOWS_PROCESS_TREE_FILENAME)) + ? [] + : [`out/orcad/${ORCAD_WINDOWS_PROCESS_TREE_FILENAME}`]) + ] + : [] +) + +const SMOKE = ` +const pty = require(process.argv[2]) +const shell = process.env.ComSpec || 'cmd.exe' +let output = '' +const deadline = setTimeout(() => { console.log(JSON.stringify({ timedOut: true, output })); process.exit(98) }, 20000) +const term = pty.spawn(shell, [], { cols: 80, rows: 24, cwd: process.cwd(), useConpty: true }) +term.onData((data) => { output += data }) +term.onExit(({ exitCode }) => { + clearTimeout(deadline) + console.log(JSON.stringify({ exitCode, read: output.includes('ORCA_CONPTY_SMOKE_OK') })) + process.exit(0) +}) +term.write('echo ORCA_CONPTY_SMOKE_%COMPUTERNAME:~0,0%OK & exit 17\\r') +` + +let root = '' +afterEach(() => { + if (root) { + removeTreeSync(root) + } +}) + +describe.skipIf(skip || !windows)('ConPTY in a broken-away, windowless orcad process', () => { + it('spawns a shell, writes, reads its output and sees it exit', async (context) => { + root = mkdtempSync(join(tmpdir(), 'orcad-conpty-breakaway-')) + const { slotDir, runtime } = installPackagedOrcadSlotForTests(root, pinnedNode!) + const script = join(root, 'conpty-smoke.cjs') + writeFileSync(script, SMOKE) + const stdoutPath = join(root, 'smoke.out') + const stderrPath = join(root, 'smoke.err') + // From out/orcad, not the temp slot: a loaded .node stays mapped and pins its directory. + const addon: unknown = createRequire(import.meta.url)( + resolve('out/orcad', ORCAD_WINDOWS_PROCESS_TREE_FILENAME) + ) + const spawn = + addon && typeof addon === 'object' && 'spawnOutsideJob' in addon + ? addon.spawnOutsideJob + : undefined + if (typeof spawn !== 'function') { + throw new Error('the slot addon does not export spawnOutsideJob') + } + const commandLine = [runtime, script, join(slotDir, ORCAD_NODE_PTY_DIR)] + .map(quoteWindowsArgument) + .join(' ') + const result: unknown = spawn(runtime, commandLine, root, stdoutPath, stderrPath) + const record = + result && typeof result === 'object' ? Object.fromEntries(Object.entries(result)) : {} + if (record.reason === 'breakaway-denied') { + // A runner whose own job forbids breakaway cannot host this check; Windows SSH hosts allow it. + context.skip() + } + expect(record, JSON.stringify(record)).toMatchObject({ ok: true }) + + let report: Record | null = null + await vi.waitFor( + () => { + const line = existsSync(stdoutPath) + ? readFileSync(stdoutPath, 'utf8') + .split(/\r?\n/u) + .find((candidate) => candidate.startsWith('{')) + : undefined + if (!line) { + const stderr = existsSync(stderrPath) ? readFileSync(stderrPath, 'utf8') : '' + throw new Error(`no smoke report yet: ${stderr.slice(-2000)}`) + } + report = JSON.parse(line) + }, + { timeout: 30_000, interval: 250 } + ) + expect(report).toEqual({ exitCode: 17, read: true }) + }, 60_000) +}) diff --git a/src/main/persistence-orcad-migration-catalog-fixture.ts b/src/main/persistence-orcad-migration-catalog-fixture.ts new file mode 100644 index 00000000000..f837413374a --- /dev/null +++ b/src/main/persistence-orcad-migration-catalog-fixture.ts @@ -0,0 +1,256 @@ +import type { FolderWorkspace } from '../shared/folder-workspace-types' +import type { Automation, AutomationRun } from '../shared/automations-types' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationDormantStatePayload, + type OrcadMigrationManifest +} from '../shared/orcad-migration-manifest' +import type { ProjectGroup } from '../shared/project-group-types' +import type { Repo } from '../shared/repo-types' +import { getDefaultWorkspaceSession } from '../shared/constants' +import { folderWorkspaceKey, worktreeWorkspaceKey } from '../shared/workspace-scope' +import { computeOrcadMigrationManifestSha256 } from './orcad/orcad-migration-manifest-digest' + +export const PROJECT_GROUP: ProjectGroup = { + id: 'group-1', + name: 'Production', + parentPath: '/srv', + connectionId: 'ssh-prod', + executionHostId: 'ssh:ssh-prod', + parentGroupId: null, + createdFrom: 'manual', + tabOrder: 1, + isCollapsed: false, + color: null, + createdAt: 1, + updatedAt: 2 +} + +export const REPOSITORY: Repo = { + id: 'repo-1', + path: '/srv/repo-1', + displayName: 'Repository', + badgeColor: '#737373', + addedAt: 3, + kind: 'git', + connectionId: 'ssh-prod', + executionHostId: 'ssh:ssh-prod', + projectGroupId: 'group-1' +} + +export const FOLDER_WORKSPACE: FolderWorkspace = { + id: 'folder-1', + projectGroupId: 'group-1', + name: 'Investigate', + folderPath: '/srv/investigate', + connectionId: 'ssh-prod', + executionHostId: 'ssh:ssh-prod', + linkedTask: null, + comment: 'Keep this note', + isArchived: false, + isUnread: true, + isPinned: true, + sortOrder: 4, + lastActivityAt: 5, + createdAt: 6, + updatedAt: 7 +} + +export const DORMANT_WORKTREE_ID = 'repo-1::/srv/repo-1-worktree' +export const DORMANT_NAMESPACE = 'local:/srv/orca-worktrees' +export const DORMANT_LEAF_ID = '11111111-1111-4111-8111-111111111111' +export const DORMANT_AUTOMATION: Automation = { + id: 'automation-dormant', + name: 'Nightly checks', + prompt: 'Run tests', + precheck: null, + agentId: 'codex', + runContext: { + kind: 'workspace-run', + projectId: 'repo:repo-1', + hostId: 'local', + projectHostSetupId: REPOSITORY.id, + repoId: REPOSITORY.id, + path: REPOSITORY.path + }, + sourceContext: null, + projectId: REPOSITORY.id, + executionTargetType: 'local', + executionTargetId: 'local', + schedulerOwner: 'remote_host_service', + workspaceMode: 'new_per_run', + workspaceId: null, + baseBranch: 'main', + reuseSession: false, + timezone: 'UTC', + rrule: 'FREQ=DAILY', + dtstart: 1, + enabled: false, + nextRunAt: 2, + missedRunPolicy: 'run_once_within_grace', + missedRunGraceMinutes: 60, + createdAt: 1, + updatedAt: 2 +} +export const DORMANT_AUTOMATION_RUN: AutomationRun = { + id: 'run-dormant', + automationId: DORMANT_AUTOMATION.id, + runContext: DORMANT_AUTOMATION.runContext, + sourceContext: null, + title: 'Nightly checks run 1', + scheduledFor: 1, + status: 'completed', + trigger: 'scheduled', + workspaceId: null, + sessionKind: 'terminal', + chatSessionId: null, + terminalSessionId: 'tab-history', + terminalPaneKey: null, + terminalPtyId: 'pty-history', + outputSnapshot: { + format: 'plain_text', + content: 'all green', + capturedAt: 2, + truncated: false + }, + precheckResult: null, + usage: null, + error: null, + startedAt: 1, + dispatchedAt: 1, + createdAt: 1, + runNumber: 1 +} + +export function dormantState(): OrcadMigrationDormantStatePayload { + return { + version: 1, + worktreeMeta: [ + { + sourceKey: DORMANT_WORKTREE_ID, + worktreeId: DORMANT_WORKTREE_ID, + meta: { + instanceId: 'instance-1', + displayName: 'Dormant worktree', + comment: 'Preserve me', + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + isArchived: false, + isUnread: true, + isPinned: false, + sortOrder: 1, + lastActivityAt: 4_102_444_800_000, + hostId: 'local' + } + } + ], + worktreeLineage: [ + { + sourceKey: DORMANT_WORKTREE_ID, + worktreeId: DORMANT_WORKTREE_ID, + lineage: { + worktreeId: DORMANT_WORKTREE_ID, + worktreeInstanceId: 'instance-1', + parentWorktreeId: REPOSITORY.id, + parentWorktreeInstanceId: 'main-instance', + origin: 'manual', + capture: { source: 'manual-action', confidence: 'explicit' }, + createdAt: 3 + } + } + ], + workspaceLineage: [ + { + sourceKey: worktreeWorkspaceKey(DORMANT_WORKTREE_ID), + childWorkspaceKey: worktreeWorkspaceKey(DORMANT_WORKTREE_ID), + lineage: { + childWorkspaceKey: worktreeWorkspaceKey(DORMANT_WORKTREE_ID), + childInstanceId: 'instance-1', + parentWorkspaceKey: folderWorkspaceKey(FOLDER_WORKSPACE.id), + parentInstanceId: null, + origin: 'manual', + capture: { source: 'manual-action', confidence: 'explicit' }, + createdAt: 4 + } + } + ], + sparsePresets: [ + { + id: 'preset-1', + repoId: REPOSITORY.id, + name: 'Renderer', + directories: ['src/renderer'], + createdAt: 5, + updatedAt: 6 + } + ], + retiredWorktreeNames: [ + { repoId: REPOSITORY.id, registry: { exhaustedTiers: 0, names: ['nautilus'] } } + ], + retiredWorktreeNamespaces: [ + { + sourceNamespaceKeys: ['ssh:source:/srv/orca-worktrees'], + namespaceKey: DORMANT_NAMESPACE, + registry: { exhaustedTiers: 0, names: ['seahorse'] } + } + ], + workspaceSession: { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [DORMANT_WORKTREE_ID]: [ + { + id: 'tab-dormant', + ptyId: null, + worktreeId: DORMANT_WORKTREE_ID, + title: 'Dormant terminal', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 7 + } + ] + }, + terminalLayoutsByTabId: { + 'tab-dormant': { + root: { type: 'leaf', leafId: DORMANT_LEAF_ID }, + activeLeafId: DORMANT_LEAF_ID, + expandedLeafId: null, + titlesByLeafId: { [DORMANT_LEAF_ID]: 'Investigating' } + } + } + }, + automations: [structuredClone(DORMANT_AUTOMATION)], + automationRuns: [structuredClone(DORMANT_AUTOMATION_RUN)] + } +} + +export function manifest(overrides: Partial = {}): OrcadMigrationManifest { + const unsigned = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-08-30T12:00:00.000Z', + source: { + sshTargetId: 'ssh-prod', + sshTargetGeneration: 8, + targetLabel: 'Production' + }, + payload: { + repositories: [REPOSITORY], + projectGroups: [PROJECT_GROUP], + folderWorkspaces: [FOLDER_WORKSPACE] + }, + ...withoutDigest(overrides) + } + return { + ...unsigned, + manifestSha256: overrides.manifestSha256 ?? computeOrcadMigrationManifestSha256(unsigned) + } +} + +export function withoutDigest( + value: Partial +): Partial> { + const { manifestSha256: _digest, ...rest } = value + return rest +} diff --git a/src/main/persistence-orcad-migration-catalog.test.ts b/src/main/persistence-orcad-migration-catalog.test.ts new file mode 100644 index 00000000000..30f089d6b74 --- /dev/null +++ b/src/main/persistence-orcad-migration-catalog.test.ts @@ -0,0 +1,551 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { MAX_ORCAD_MIGRATION_STAGED_CATALOGS } from '../shared/orcad-migration-manifest' +import { getDefaultWorkspaceSession } from '../shared/constants' +import { folderWorkspaceKey, worktreeWorkspaceKey } from '../shared/workspace-scope' +import { MAX_RETIREMENT_NAMESPACES } from './worktree-retirement-namespace' +import { + DORMANT_AUTOMATION, + DORMANT_AUTOMATION_RUN, + DORMANT_LEAF_ID, + DORMANT_NAMESPACE, + DORMANT_WORKTREE_ID, + FOLDER_WORKSPACE, + REPOSITORY, + dormantState, + manifest +} from './persistence-orcad-migration-catalog-fixture' +import { + createStore, + makeRepo, + readDataFile, + testState, + writeDataFile +} from './persistence-test-harness' + +const { trackMock, getCohortAtEmitMock } = vi.hoisted(() => ({ + trackMock: vi.fn(), + getCohortAtEmitMock: vi.fn(() => ({ nth_repo_added: 1 })) +})) + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { + isEncryptionAvailable: () => true, + encryptString: (plaintext: string) => Buffer.from(`encrypted:${plaintext}`, 'utf-8'), + decryptString: (ciphertext: Buffer) => ciphertext.toString('utf-8').slice('encrypted:'.length) + } +})) + +vi.mock('./telemetry/client', () => ({ track: trackMock })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: getCohortAtEmitMock })) + +/** Stage then commit: the only path a destination publishes a catalog through. */ +function commitCatalog( + store: ReturnType, + input: ReturnType, + options?: { now?: () => Date } +) { + store.stageOrcadMigrationCatalog(input, options) + return store.commitStagedOrcadMigrationCatalog(input, options) +} + +describe('orcad migration catalog persistence', () => { + beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-migration-catalog-')) + }) + + afterEach(() => { + rmSync(testState.dir, { recursive: true, force: true }) + }) + + it('commits the catalog durably, strips source ownership, and replays idempotently', async () => { + const store = createStore() + const input = manifest() + + const first = commitCatalog(store, input, { + now: () => new Date('2026-08-30T12:05:00.000Z') + }) + await store.flushPendingOrThrowAsync() + const restored = createStore() + const replay = commitCatalog(restored, input) + + expect(first).toMatchObject({ + state: 'committed', + receipt: { + migrationId: 'migration-1', + importedAt: '2026-08-30T12:05:00.000Z', + repositoryIds: ['repo-1'], + projectGroupIds: ['group-1'], + folderWorkspaceIds: ['folder-1'] + } + }) + expect(replay).toEqual(first) + expect(restored.getRepos()).toEqual([ + expect.not.objectContaining({ + connectionId: expect.anything(), + executionHostId: expect.anything() + }) + ]) + expect(restored.getProjectGroups()).toEqual([ + expect.not.objectContaining({ + connectionId: expect.anything(), + executionHostId: expect.anything() + }) + ]) + expect(restored.getFolderWorkspaces()).toEqual([ + expect.objectContaining({ id: 'folder-1', comment: 'Keep this note' }) + ]) + expect(restored.getFolderWorkspaces()[0]?.connectionId).toBeNull() + expect(restored.getFolderWorkspaces()[0]).not.toHaveProperty('executionHostId') + }) + + it('imports dormant source focus scalars alongside the inactive session', () => { + const store = createStore() + const incomingDormant = dormantState() + incomingDormant.workspaceSession = { + ...incomingDormant.workspaceSession!, + activeRepoId: REPOSITORY.id, + activeWorktreeId: DORMANT_WORKTREE_ID, + activeWorkspaceKey: worktreeWorkspaceKey(DORMANT_WORKTREE_ID), + activeWorkspaceExecutionHostId: 'local', + activeTabId: 'tab-dormant' + } + const base = manifest() + const input = manifest({ + payload: { ...base.payload, dormantState: incomingDormant } + }) + + commitCatalog(store, input) + + expect(store.getWorkspaceSession()).toMatchObject({ + activeRepoId: REPOSITORY.id, + activeWorktreeId: DORMANT_WORKTREE_ID, + activeWorkspaceKey: worktreeWorkspaceKey(DORMANT_WORKTREE_ID), + activeWorkspaceExecutionHostId: 'local', + activeTabId: 'tab-dormant' + }) + }) + + it('stages durably without publishing rows, then commits catalog and receipt together', async () => { + const store = createStore() + const input = manifest() + + expect( + store.stageOrcadMigrationCatalog(input, { + now: () => new Date('2026-08-30T12:03:00.000Z') + }) + ).toEqual({ + state: 'staged', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + stagedAt: '2026-08-30T12:03:00.000Z' + }) + expect(store.getRepos()).toEqual([]) + await store.flushPendingOrThrowAsync() + + const restored = createStore() + expect(restored.getOrcadMigrationCatalogState(input)).toMatchObject({ state: 'staged' }) + expect( + restored.commitStagedOrcadMigrationCatalog(input, { + now: () => new Date('2026-08-30T12:05:00.000Z') + }) + ).toMatchObject({ state: 'committed', receipt: { importedAt: '2026-08-30T12:05:00.000Z' } }) + expect(restored.getRepos()).toHaveLength(1) + await restored.flushPendingOrThrowAsync() + + const committed = createStore() + expect(committed.getOrcadMigrationCatalogState(input)).toMatchObject({ + state: 'committed', + receipt: { repositoryIds: ['repo-1'] } + }) + expect(readDataFile()).not.toHaveProperty('orcadMigrationStagedCatalogs.0') + }) + + it('commits dormant metadata, lineage, presets, and retirement state with the receipt', async () => { + const store = createStore() + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + store.mergeRetiredWorktreeNamesForNamespace(DORMANT_NAMESPACE, ['octopus']) + + store.stageOrcadMigrationCatalog(input) + expect(store.getAllWorktreeMeta()).toEqual({}) + expect(store.getSparsePresets(REPOSITORY.id)).toEqual([]) + + store.commitStagedOrcadMigrationCatalog(input) + expect(store.getWorktreeMeta(DORMANT_WORKTREE_ID)).toMatchObject({ + comment: 'Preserve me', + hostId: 'local' + }) + expect(store.getWorktreeLineage(DORMANT_WORKTREE_ID)?.parentWorktreeId).toBe(REPOSITORY.id) + expect( + store.getWorkspaceLineage(worktreeWorkspaceKey(DORMANT_WORKTREE_ID))?.parentWorkspaceKey + ).toBe(folderWorkspaceKey(FOLDER_WORKSPACE.id)) + expect(store.getSparsePresets(REPOSITORY.id).map((preset) => preset.id)).toEqual(['preset-1']) + expect(store.getWorkspaceSession().tabsByWorktree[DORMANT_WORKTREE_ID]?.[0]).toMatchObject({ + id: 'tab-dormant', + ptyId: null + }) + expect( + store.getWorkspaceSession().terminalLayoutsByTabId['tab-dormant']?.titlesByLeafId + ).toEqual({ [DORMANT_LEAF_ID]: 'Investigating' }) + expect(store.listAutomations()).toEqual([ + expect.objectContaining({ + id: DORMANT_AUTOMATION.id, + enabled: false, + schedulerOwner: 'remote_host_service' + }) + ]) + expect(store.listAutomationRuns(DORMANT_AUTOMATION.id)).toEqual([ + expect.objectContaining({ + id: DORMANT_AUTOMATION_RUN.id, + outputSnapshot: expect.objectContaining({ content: 'all green' }) + }) + ]) + expect(store.getRetiredWorktreeNameRegistry(REPOSITORY.id).names).toContain('nautilus') + expect(store.getRetiredWorktreeNameRegistryForNamespace(DORMANT_NAMESPACE).names).toEqual( + expect.arrayContaining(['octopus', 'seahorse']) + ) + await store.flushPendingOrThrowAsync() + + const restored = createStore() + expect(restored.getOrcadMigrationCatalogState(input)).toMatchObject({ state: 'committed' }) + expect(restored.commitStagedOrcadMigrationCatalog(input)).toMatchObject({ state: 'committed' }) + expect(restored.getWorktreeMeta(DORMANT_WORKTREE_ID)?.comment).toBe('Preserve me') + expect(restored.getSparsePresets(REPOSITORY.id)).toHaveLength(1) + expect(restored.getWorkspaceSession().tabsByWorktree[DORMANT_WORKTREE_ID]).toHaveLength(1) + expect(restored.listAutomationRuns(DORMANT_AUTOMATION.id)).toHaveLength(1) + }) + + it('recovers a dormant commit lost before flush from the durable stage', async () => { + const store = createStore() + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + store.stageOrcadMigrationCatalog(input) + await store.flushPendingOrThrowAsync() + const stagedDiskState = readDataFile() + + store.commitStagedOrcadMigrationCatalog(input) + expect(store.getWorktreeMeta(DORMANT_WORKTREE_ID)?.comment).toBe('Preserve me') + writeDataFile(stagedDiskState) + + const restored = createStore() + expect(restored.getOrcadMigrationCatalogState(input)).toMatchObject({ state: 'staged' }) + expect(restored.getWorktreeMeta(DORMANT_WORKTREE_ID)).toBeUndefined() + restored.commitStagedOrcadMigrationCatalog(input) + await restored.flushPendingOrThrowAsync() + + const committed = createStore() + expect(committed.getOrcadMigrationCatalogState(input)).toMatchObject({ state: 'committed' }) + expect(committed.getWorktreeMeta(DORMANT_WORKTREE_ID)?.comment).toBe('Preserve me') + }) + + it('merges a migrated retirement namespace while preserving the storage cap', async () => { + const store = createStore() + for (let index = 0; index < MAX_RETIREMENT_NAMESPACES; index += 1) { + store.mergeRetiredWorktreeNamesForNamespace(`local:/existing/${index}`, ['nautilus']) + } + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + + store.stageOrcadMigrationCatalog(input) + store.commitStagedOrcadMigrationCatalog(input) + await store.flushPendingOrThrowAsync() + + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the raw data file is untyped JSON this test edits to simulate an older writer. + const diskState = readDataFile() as Record + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the raw data file is untyped JSON this test edits to simulate an older writer. + const namespaces = (diskState.retiredWorktreeNamesByNamespace ?? {}) as Record + expect(Object.keys(namespaces)).toHaveLength(MAX_RETIREMENT_NAMESPACES) + expect(store.getRetiredWorktreeNameRegistryForNamespace(DORMANT_NAMESPACE).names).toContain( + 'seahorse' + ) + expect(store.getRetiredWorktreeNameRegistryForNamespace('local:/existing/0').names).toEqual([]) + }) + + it('rejects dormant-state conflicts before publishing any catalog row', () => { + const store = createStore() + store.setWorktreeMeta(DORMANT_WORKTREE_ID, { + ...dormantState().worktreeMeta[0].meta, + comment: 'Different owner' + }) + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + + expect(() => store.stageOrcadMigrationCatalog(input)).toThrow( + `orcad_migration_dormant_id_conflict:worktree_meta:${DORMANT_WORKTREE_ID}` + ) + expect(store.getRepos()).toEqual([]) + expect(store.getProjectGroups()).toEqual([]) + expect(store.getFolderWorkspaces()).toEqual([]) + }) + + it('rejects a dormant workspace-session owner conflict before publication', () => { + const store = createStore() + store.setWorkspaceSession({ + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [DORMANT_WORKTREE_ID]: [ + { + ...dormantState().workspaceSession!.tabsByWorktree[DORMANT_WORKTREE_ID][0], + title: 'Destination terminal' + } + ] + } + }) + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + + expect(() => store.stageOrcadMigrationCatalog(input)).toThrow( + `orcad_migration_dormant_id_conflict:workspace_session:tabsByWorktree:${DORMANT_WORKTREE_ID}` + ) + expect(store.getRepos()).toEqual([]) + }) + + it('rejects a dormant automation conflict before publishing a second migration', () => { + const store = createStore() + const base = manifest() + const first = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + commitCatalog(store, first) + const changedDormant = dormantState() + const changedAutomation = changedDormant.automations?.[0] + if (!changedAutomation) { + throw new Error('expected dormant automation') + } + changedDormant.automations = [{ ...changedAutomation, name: 'Different destination owner' }] + const second = manifest({ + migrationId: 'migration-automation-conflict', + payload: { ...base.payload, dormantState: changedDormant } + }) + + expect(() => store.stageOrcadMigrationCatalog(second)).toThrow( + `orcad_migration_dormant_id_conflict:automation:${DORMANT_AUTOMATION.id}` + ) + expect(store.listAutomations()).toEqual([ + expect.objectContaining({ id: DORMANT_AUTOMATION.id, name: 'Nightly checks' }) + ]) + }) + + it('rejects destination automation state with an in-flight run', () => { + const store = createStore() + const base = manifest() + const first = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + commitCatalog(store, first) + store.createAutomationRun(DORMANT_AUTOMATION, 10, 'manual') + const nextDormant = dormantState() + delete nextDormant.automationRuns + const second = manifest({ + migrationId: 'migration-automation-active', + payload: { ...base.payload, dormantState: nextDormant } + }) + + expect(() => store.stageOrcadMigrationCatalog(second)).toThrow( + `orcad_migration_dormant_automation_run_active:${DORMANT_AUTOMATION.id}` + ) + expect(store.getOrcadMigrationCatalogState(second)).toMatchObject({ state: 'absent' }) + }) + + it('stages idempotently and rejects migration-id reuse before catalog mutation', () => { + const store = createStore() + const input = manifest() + const first = store.stageOrcadMigrationCatalog(input) + + expect(store.stageOrcadMigrationCatalog(input)).toEqual(first) + expect(() => + store.stageOrcadMigrationCatalog( + manifest({ payload: { ...input.payload, repositories: [] } }) + ) + ).toThrow('orcad_migration_id_reused_with_different_manifest') + expect(store.getRepos()).toEqual([]) + }) + + it('aborts only matching staged state and never rolls back a committed catalog', () => { + const store = createStore() + const input = manifest() + + expect(store.abortStagedOrcadMigrationCatalog(input)).toMatchObject({ + state: 'absent', + aborted: false + }) + store.stageOrcadMigrationCatalog(input) + expect(store.abortStagedOrcadMigrationCatalog(input)).toMatchObject({ + state: 'absent', + aborted: true + }) + store.stageOrcadMigrationCatalog(input) + store.commitStagedOrcadMigrationCatalog(input) + expect(store.abortStagedOrcadMigrationCatalog(input)).toMatchObject({ + state: 'committed', + aborted: false + }) + expect(store.getRepos()).toHaveLength(1) + }) + + it('requires a durable stage before commit and bounds concurrent dormant catalogs', () => { + const store = createStore() + const input = manifest() + expect(() => store.commitStagedOrcadMigrationCatalog(input)).toThrow( + 'orcad_migration_catalog_not_staged' + ) + + for (let index = 0; index < MAX_ORCAD_MIGRATION_STAGED_CATALOGS; index++) { + store.stageOrcadMigrationCatalog( + manifest({ + migrationId: `migration-${index + 1}`, + payload: { repositories: [], projectGroups: [], folderWorkspaces: [] } + }) + ) + } + expect(() => + store.stageOrcadMigrationCatalog(manifest({ migrationId: 'migration-overflow' })) + ).toThrow('orcad_migration_staging_capacity_exceeded') + expect(store.getRepos()).toEqual([]) + + // A week later no client came back: the stages expire and stop holding the server awake. + const later = () => new Date(Date.now() + 8 * 24 * 60 * 60 * 1000) + expect(store.hasStagedOrcadMigrationCatalog(later().getTime())).toBe(false) + expect( + store.stageOrcadMigrationCatalog(manifest({ migrationId: 'migration-overflow' }), { + now: later + }) + ).toMatchObject({ state: 'staged' }) + }) + + it('retains exclusive staged claims after reload and releases them on dormant abort', async () => { + const store = createStore() + const first = manifest() + const second = manifest({ migrationId: 'competing-migration' }) + store.stageOrcadMigrationCatalog(first) + await store.flushPendingOrThrowAsync() + const restored = createStore() + expect(() => restored.stageOrcadMigrationCatalog(second)).toThrow( + 'orcad_migration_staged_claim_conflict:' + ) + expect(restored.getOrcadMigrationCatalogState(first)).toMatchObject({ state: 'staged' }) + expect(restored.getOrcadMigrationCatalogState(second)).toMatchObject({ state: 'absent' }) + expect(restored.getRepos()).toEqual([]) + expect(restored.abortStagedOrcadMigrationCatalog(first).aborted).toBe(true) + await restored.flushPendingOrThrowAsync() + const afterAbort = createStore() + expect(afterAbort.stageOrcadMigrationCatalog(second)).toMatchObject({ state: 'staged' }) + expect(afterAbort.commitStagedOrcadMigrationCatalog(second)).toMatchObject({ + state: 'committed' + }) + }) + + it('refuses committing overlapping stages loaded from an older writer', async () => { + const store = createStore() + const first = manifest() + const second = manifest({ migrationId: 'older-competing-stage' }) + store.stageOrcadMigrationCatalog(first) + await store.flushPendingOrThrowAsync() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the raw data file is untyped JSON this test edits to simulate an older writer. + const persisted = readDataFile() as Record + persisted.orcadMigrationStagedCatalogs = [first, second].map((entry) => ({ + version: 1, + manifest: entry, + stagedAt: entry.createdAt + })) + writeDataFile(persisted) + const restored = createStore() + for (const entry of [first, second]) { + expect(() => restored.commitStagedOrcadMigrationCatalog(entry)).toThrow( + 'orcad_migration_staged_claim_conflict:' + ) + expect(restored.getOrcadMigrationCatalogState(entry)).toMatchObject({ state: 'staged' }) + } + expect(restored.getRepos()).toEqual([]) + expect(restored.abortStagedOrcadMigrationCatalog(second).aborted).toBe(true) + expect(restored.commitStagedOrcadMigrationCatalog(first)).toMatchObject({ state: 'committed' }) + }) + + it('reconstructs a missing receipt without duplicating identical catalog or dormant state', async () => { + const store = createStore() + const base = manifest() + const input = manifest({ payload: { ...base.payload, dormantState: dormantState() } }) + commitCatalog(store, input) + await store.flushPendingOrThrowAsync() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the raw data file is untyped JSON this test edits to simulate an older writer. + const persisted = readDataFile() as Record + delete persisted.orcadMigrationImportReceipts + writeDataFile(persisted) + + const restored = createStore() + const replay = commitCatalog(restored, input) + + expect(replay.state).toBe('committed') + expect(restored.getRepos()).toHaveLength(1) + expect(restored.getProjectGroups()).toHaveLength(1) + expect(restored.getFolderWorkspaces()).toHaveLength(1) + expect(restored.getWorktreeMeta(DORMANT_WORKTREE_ID)?.comment).toBe('Preserve me') + expect(restored.getSparsePresets(REPOSITORY.id)).toHaveLength(1) + }) + + it('still reads a commit as committed after the live server changed what it imported', () => { + const store = createStore() + const input = manifest() + commitCatalog(store, input) + store.removeProject(REPOSITORY.id) + + expect(store.getOrcadMigrationCatalogState(input).state).toBe('committed') + expect(store.getRepos()).toEqual([]) + }) + + it('rejects an id reuse with a different manifest without duplicating state', () => { + const store = createStore() + const input = manifest() + commitCatalog(store, input) + const changed = manifest({ + payload: { + ...input.payload, + repositories: [{ ...REPOSITORY, displayName: 'Changed' }] + } + }) + + expect(() => commitCatalog(store, changed)).toThrow( + 'orcad_migration_id_reused_with_different_manifest' + ) + expect(store.getRepos()).toHaveLength(1) + }) + + it('validates all path conflicts before adding any manifest row', () => { + const store = createStore() + store.addRepo(makeRepo({ id: 'existing', path: REPOSITORY.path })) + + expect(() => commitCatalog(store, manifest())).toThrow( + `orcad_migration_repository_path_conflict:${REPOSITORY.path}` + ) + expect(store.getRepos().map((repo) => repo.id)).toEqual(['existing']) + expect(store.getProjectGroups()).toEqual([]) + expect(store.getFolderWorkspaces()).toEqual([]) + }) + + it('validates all id conflicts before adding any manifest row', () => { + const store = createStore() + store.addRepo(makeRepo({ id: REPOSITORY.id, path: '/srv/different' })) + + expect(() => commitCatalog(store, manifest())).toThrow( + `orcad_migration_repository_id_conflict:${REPOSITORY.id}` + ) + expect(store.getRepos().map((repo) => repo.path)).toEqual(['/srv/different']) + expect(store.getProjectGroups()).toEqual([]) + expect(store.getFolderWorkspaces()).toEqual([]) + }) + + it('rejects missing group references before mutation', () => { + const store = createStore() + const input = manifest() + const invalid = manifest({ + payload: { ...input.payload, projectGroups: [] } + }) + + expect(() => commitCatalog(store, invalid)).toThrow( + 'orcad_migration_repository_project_group_missing:repo-1' + ) + expect(store.getRepos()).toEqual([]) + expect(store.getProjectGroups()).toEqual([]) + expect(store.getFolderWorkspaces()).toEqual([]) + }) +}) diff --git a/src/main/persistence/applying-settings/terminal-settings-migrations.ts b/src/main/persistence/applying-settings/terminal-settings-migrations.ts index 551516e70b3..8f2cc0dc5ca 100644 --- a/src/main/persistence/applying-settings/terminal-settings-migrations.ts +++ b/src/main/persistence/applying-settings/terminal-settings-migrations.ts @@ -60,6 +60,8 @@ type RetiredGlobalSettings = { terminalScrollbackBytes?: unknown enableGitHubAttribution?: unknown showAgentsSidebar?: unknown + // Managed servers are the default SSH path now; an older build reads a missing key as off. + experimentalManagedServers?: unknown // Why: #22551 kept this key in settings; it now lives in a main-owned store and must never ride along. opencodeGoApiKey?: unknown } @@ -71,12 +73,14 @@ export function stripRetiredGlobalSettings( terminalScrollbackBytes: _legacyScrollbackBytes, enableGitHubAttribution: _legacyGitHubAttribution, showAgentsSidebar: _legacyShowAgentsSidebar, + experimentalManagedServers: _retiredManagedServersExperiment, opencodeGoApiKey: _legacyOpenCodeGoApiKey, ...rest } = (settings ?? {}) as Partial & RetiredGlobalSettings void _legacyScrollbackBytes void _legacyGitHubAttribution void _legacyShowAgentsSidebar + void _retiredManagedServersExperiment void _legacyOpenCodeGoApiKey return rest } diff --git a/src/main/persistence/leasing-ssh-ptys/ssh-normalization-orcad-fence.test.ts b/src/main/persistence/leasing-ssh-ptys/ssh-normalization-orcad-fence.test.ts new file mode 100644 index 00000000000..661c1fccdfb --- /dev/null +++ b/src/main/persistence/leasing-ssh-ptys/ssh-normalization-orcad-fence.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import type { SshTarget } from '../../../shared/ssh-types' +import { normalizeSshTarget } from './ssh-normalization' + +const base: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } + +describe('loading a managed Orca server fence', () => { + it('moves a phase-3 owner fence to orcadFence, so shipped builds stop hiding the host', () => { + const loaded = normalizeSshTarget({ + ...base, + owner: { type: 'on-demand-runtime', runtimeId: 'managed-orcad:env-1' } + }) + expect(loaded.owner).toBeUndefined() + expect(loaded.orcadFence).toEqual({ environmentId: 'env-1' }) + }) + + it('keeps ephemeral runtime owners, a valid fence, and drops a malformed one', () => { + const vm = normalizeSshTarget({ + ...base, + owner: { type: 'on-demand-runtime', runtimeId: 'vm' } + }) + expect(vm.owner).toEqual({ type: 'on-demand-runtime', runtimeId: 'vm' }) + expect( + normalizeSshTarget({ + ...base, + orcadFence: { environmentId: 'env-1', sourceChangedAt: '2026-10-05T00:00:00.000Z' } + }).orcadFence + ).toEqual({ environmentId: 'env-1', sourceChangedAt: '2026-10-05T00:00:00.000Z' }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: simulates a hand-edited or corrupt stored fence. + const malformed = { ...base, orcadFence: { environmentId: 7 } } as unknown as SshTarget + expect(normalizeSshTarget(malformed).orcadFence).toBeUndefined() + }) + + it('keeps the recorded move offer and drops a malformed one', () => { + expect( + normalizeSshTarget({ ...base, managedServerMoveOffered: { appVersion: '1.5.0' } }) + .managedServerMoveOffered + ).toEqual({ appVersion: '1.5.0' }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: simulates a hand-edited or corrupt stored record. + const malformed = { + ...base, + managedServerMoveOffered: { appVersion: 7 } + } as unknown as SshTarget + expect(normalizeSshTarget(malformed).managedServerMoveOffered).toBeUndefined() + }) + + it('falls back to the legacy owner when the stored fence is malformed', () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: simulates a corrupt fence next to a legacy owner. + const loaded = normalizeSshTarget({ + ...base, + owner: { type: 'on-demand-runtime', runtimeId: 'managed-orcad:env-1' }, + orcadFence: { environmentId: 7, sourceChangedAt: 3 } + } as unknown as SshTarget) + expect(loaded.owner).toBeUndefined() + expect(loaded.orcadFence).toEqual({ environmentId: 'env-1' }) + }) + + it("keeps a newer build's unknown fields on the fence and the server notes", () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: simulates fields a newer build adds. + const loaded = normalizeSshTarget({ + ...base, + orcadFence: { environmentId: 'env-1', sourceChangedAt: 4, addedLater: true }, + managedServerUnavailable: { reason: 'r', appVersion: '1', addedLater: 1 }, + managedServerMoveOffered: { appVersion: '1', addedLater: 'x' } + } as unknown as SshTarget) + expect(loaded.orcadFence).toEqual({ environmentId: 'env-1', addedLater: true }) + expect(loaded.managedServerUnavailable).toEqual({ reason: 'r', appVersion: '1', addedLater: 1 }) + expect(loaded.managedServerMoveOffered).toEqual({ appVersion: '1', addedLater: 'x' }) + }) +}) diff --git a/src/main/persistence/leasing-ssh-ptys/ssh-normalization.ts b/src/main/persistence/leasing-ssh-ptys/ssh-normalization.ts index 58414b934a5..d81fa4bd5cf 100644 --- a/src/main/persistence/leasing-ssh-ptys/ssh-normalization.ts +++ b/src/main/persistence/leasing-ssh-ptys/ssh-normalization.ts @@ -11,6 +11,7 @@ import { SSH_REMOTE_RUNTIMES } from '../../../shared/ssh-types' import { normalizeSshPendingPtyKill } from '../../../shared/ssh-pending-pty-kill' +import { getLegacyManagedOrcadOwnerEnvironmentId } from '../../../shared/managed-orcad-ssh-owner' export type LegacySshTarget = SshTarget & { remoteWorkspaceSyncEnabled?: unknown @@ -65,7 +66,83 @@ export function normalizeSshTarget(t: SshTarget): SshTarget { if (remoteRuntimeResolution) { normalized.remoteRuntimeResolution = remoteRuntimeResolution } - return normalized + return normalizeManagedServerMoveOffered( + normalizeAppVersionNote( + normalizeAppVersionNote( + migrateLegacyManagedOrcadOwner(normalized), + 'managedServerUnavailable' + ), + 'managedServerUpdateFailure' + ) + ) +} + +/** + * Phase-3 builds fenced a managed host through `owner`, which shipped builds hide. Moving the fence + * to `orcadFence` keeps the host visible, and reachable over its relay, after a downgrade. + */ +function migrateLegacyManagedOrcadOwner(target: SshTarget): SshTarget { + const environmentId = getLegacyManagedOrcadOwnerEnvironmentId(target.owner) + if (!environmentId) { + return normalizeOrcadFence(target) + } + const { owner: _legacyOwner, ...rest } = target + const fenced = normalizeOrcadFence(rest) + // A malformed fence must not discard the owner's valid environment id. + return fenced.orcadFence ? fenced : { ...fenced, orcadFence: { environmentId } } +} + +// Unknown keys pass through every note below, so a newer build's optional fields survive this one. +function noteFields(value: unknown): Record | null { + return value && typeof value === 'object' && !Array.isArray(value) ? { ...value } : null +} + +function normalizeAppVersionNote( + target: SshTarget, + key: 'managedServerUnavailable' | 'managedServerUpdateFailure' +): SshTarget { + if (target[key] === undefined) { + return target + } + const note = noteFields(target[key]) + if (typeof note?.reason === 'string' && typeof note.appVersion === 'string') { + return { ...target, [key]: { ...note, reason: note.reason, appVersion: note.appVersion } } + } + const { [key]: _malformed, ...rest } = target + return rest +} + +function normalizeManagedServerMoveOffered(target: SshTarget): SshTarget { + if (target.managedServerMoveOffered === undefined) { + return target + } + const note = noteFields(target.managedServerMoveOffered) + if (typeof note?.appVersion === 'string') { + return { ...target, managedServerMoveOffered: { ...note, appVersion: note.appVersion } } + } + const { managedServerMoveOffered: _malformed, ...rest } = target + return rest +} + +function normalizeOrcadFence(target: SshTarget): SshTarget { + if (target.orcadFence === undefined) { + return target + } + const fence = noteFields(target.orcadFence) + const environmentId = fence?.environmentId + if (fence && typeof environmentId === 'string' && environmentId.length > 0) { + const { sourceChangedAt, ...unknownAndId } = fence + return { + ...target, + orcadFence: { + ...unknownAndId, + environmentId, + ...(typeof sourceChangedAt === 'string' ? { sourceChangedAt } : {}) + } + } + } + const { orcadFence: _malformed, ...rest } = target + return rest } // Why strict: a malformed cache entry is dropped, which only costs one rung A attempt. diff --git a/src/main/persistence/leasing-ssh-ptys/ssh-pty-lease-operations.ts b/src/main/persistence/leasing-ssh-ptys/ssh-pty-lease-operations.ts index b2e3cef030b..9eb5870c19c 100644 --- a/src/main/persistence/leasing-ssh-ptys/ssh-pty-lease-operations.ts +++ b/src/main/persistence/leasing-ssh-ptys/ssh-pty-lease-operations.ts @@ -1,3 +1,4 @@ +import { isLiveSshPtyLease } from '../../../shared/ssh-pty-lease-liveness' import type { StoreRuntimeState } from '../loading-store/store-runtime-state' import type { PersistedState } from '../../../shared/persisted-state-types' import type { SshRemotePtyLease } from '../../../shared/ssh-types' @@ -77,7 +78,7 @@ export function upsertSshRemotePtyLease( // A relay renumbers from `pty-1` on every start, so `existing` can be a RECYCLED id. Route // retirement belongs to the shell that lost, never to whatever claims the id next — drop both // marks the moment this id is claimed live again, and let supersession re-derive them below. - if (next.state === 'attached' || next.state === 'detached') { + if (isLiveSshPtyLease(next)) { delete next.supersededBy delete next.relayIdRecycled } diff --git a/src/main/persistence/loading-store/normalize-loaded-global-settings.test.ts b/src/main/persistence/loading-store/normalize-loaded-global-settings.test.ts index 1fb35f2533f..7cbce85fe9b 100644 --- a/src/main/persistence/loading-store/normalize-loaded-global-settings.test.ts +++ b/src/main/persistence/loading-store/normalize-loaded-global-settings.test.ts @@ -36,6 +36,15 @@ describe('retired Agents sidebar setting', () => { }) }) +describe('retired managed servers experiment', () => { + it('drops the stored toggle, since managed servers are the default SSH path', () => { + expect('experimentalManagedServers' in normalizeLegacyProfile({})).toBe(false) + expect( + 'experimentalManagedServers' in normalizeLegacyProfile({ experimentalManagedServers: true }) + ).toBe(false) + }) +}) + describe('structured chat shell environment settings', () => { it('keeps a valid saved list and an explicit opt-out', () => { const normalized = normalizeLegacyProfile({ diff --git a/src/main/persistence/loading-store/normalize-loaded-profile-state.ts b/src/main/persistence/loading-store/normalize-loaded-profile-state.ts index 987a6e81a17..1a3ba807c5a 100644 --- a/src/main/persistence/loading-store/normalize-loaded-profile-state.ts +++ b/src/main/persistence/loading-store/normalize-loaded-profile-state.ts @@ -3,6 +3,9 @@ import type { SshRemotePtyLease } from '../../../shared/ssh-types' import { normalizeFeatureInteractionTelemetryBuckets } from '../../../shared/feature-interactions' import { normalizeFolderWorkspaceDiffComments } from '../../folder-workspace-diff-comments' import { normalizeFolderWorkspaces } from '../../../shared/folder-workspaces' +import { normalizeOrcadMigrationEvictedReceipts } from '../../../shared/orcad-migration-evicted-receipts' +import { normalizeOrcadMigrationImportReceipts } from '../../../shared/orcad-migration-manifest-validation' +import { normalizeOrcadMigrationStagedCatalogs } from '../../../shared/orcad-migration-staged-catalog-validation' import { normalizeWorkspaceLineageByChildKey } from '../applying-settings/ui-interaction-merge' import { normalizeSshRemotePtyLease, @@ -106,6 +109,15 @@ export function normalizeLoadedProfileState( legacyPaneKeyAliasEntries: normalizeLegacyPaneKeyAliasEntries(parsed.legacyPaneKeyAliasEntries), automations: Array.isArray(parsed.automations) ? parsed.automations : [], automationRuns: normalizeLoadedAutomationRuns(parsed, markNeedsSave), + orcadMigrationImportReceipts: normalizeOrcadMigrationImportReceipts( + parsed.orcadMigrationImportReceipts + ), + orcadMigrationEvictedReceipts: normalizeOrcadMigrationEvictedReceipts( + parsed.orcadMigrationEvictedReceipts + ), + orcadMigrationStagedCatalogs: normalizeOrcadMigrationStagedCatalogs( + parsed.orcadMigrationStagedCatalogs + ), onboarding: normalizedOnboarding } } diff --git a/src/main/persistence/loading-store/session-host-partitions.ts b/src/main/persistence/loading-store/session-host-partitions.ts index 12efd45a19e..74c34ec1a68 100644 --- a/src/main/persistence/loading-store/session-host-partitions.ts +++ b/src/main/persistence/loading-store/session-host-partitions.ts @@ -80,6 +80,21 @@ export class SessionHostPartitionOperations { return [...hostIds] } + /** Drops a removed host's whole session partition; local's session is never dropped. */ + removeWorkspaceSessionHost(hostId: ExecutionHostId): void { + const runtime = this[sessionHostPartitionOperationsContext].runtime + const partitions = runtime.state.workspaceSessionsByHostId + if (hostId === LOCAL_EXECUTION_HOST_ID || partitions?.[hostId] === undefined) { + return + } + const { [hostId]: _removed, ...remaining } = partitions + runtime.state.workspaceSessionsByHostId = remaining + invalidateLocalWorktreeMetadataPruneInputs() + scheduleSave(this[sessionHostPartitionOperationsContext].scheduling, [ + 'workspaceSessionsByHostId' + ]) + } + readTerminalScrollbackSnapshot(ref: string): string | null { return readTerminalScrollbackSnapshotSync( ref, @@ -207,7 +222,7 @@ export function setHostWorkspaceSession( ) owner[sessionHostPartitionOperationsContext].runtime.state.workspaceSessionsByHostId = { ...owner[sessionHostPartitionOperationsContext].runtime.state.workspaceSessionsByHostId, - [hostId]: pruned + [hostId]: withRequiredWorkspaceSessionMaps(pruned) } scheduleSave(owner[sessionHostPartitionOperationsContext].scheduling, [ 'workspaceSessionsByHostId' @@ -222,3 +237,17 @@ export function installSessionHostPartitionOperationsContext( value: source[sessionHostPartitionOperationsContext] }) } + +/** + * The renderer splits a full snapshot per host and leaves out maps a host has no rows in, so a + * host partition written as sent lacks maps the type requires and every reader iterates. + */ +export function withRequiredWorkspaceSessionMaps( + session: WorkspaceSessionState +): WorkspaceSessionState { + return { + ...session, + tabsByWorktree: session.tabsByWorktree ?? {}, + terminalLayoutsByTabId: session.terminalLayoutsByTabId ?? {} + } +} diff --git a/src/main/persistence/loading-store/session-snapshot-operations.ts b/src/main/persistence/loading-store/session-snapshot-operations.ts index 9c3cf23be7b..13174ef18b5 100644 --- a/src/main/persistence/loading-store/session-snapshot-operations.ts +++ b/src/main/persistence/loading-store/session-snapshot-operations.ts @@ -13,7 +13,11 @@ import type { StoreRuntimeState } from './store-runtime-state' import type { SessionHostPartitionOperations } from './session-host-partitions' import type { TerminalBindingRecoveryOperations } from './terminal-binding-recovery' import type { WriteSchedulingOperations } from './write-scheduling' -import { resolveHostId, setHostWorkspaceSession } from './session-host-partitions' +import { + resolveHostId, + setHostWorkspaceSession, + withRequiredWorkspaceSessionMaps +} from './session-host-partitions' import { scheduleSave } from './write-scheduling' type SessionSnapshotOperationsRuntime = Pick< @@ -24,6 +28,7 @@ type SessionSnapshotOperationsRuntime = Pick< | 'quitFlushStarted' | 'state' | 'terminalScrollbackSnapshotStorage' + | 'retainedScrollbackRefsByMigrationId' | 'writesFrozen' > @@ -101,7 +106,7 @@ export class SessionSnapshotOperations { } else { runtime.state.workspaceSessionsByHostId = { ...runtime.state.workspaceSessionsByHostId, - [hostId]: session + [hostId]: withRequiredWorkspaceSessionMaps(session) } } scheduleSave( diff --git a/src/main/persistence/loading-store/store-domain-composition.ts b/src/main/persistence/loading-store/store-domain-composition.ts index 70f22e90c5c..4ba2e58881a 100644 --- a/src/main/persistence/loading-store/store-domain-composition.ts +++ b/src/main/persistence/loading-store/store-domain-composition.ts @@ -63,6 +63,14 @@ import { SshLeaseRecoveryOperations, installSshLeaseRecoveryOperationsContext } from './ssh-lease-recovery-operations' +import { + OrcadSourceExportPersistence, + installOrcadSourceExportPersistenceContext +} from '../migrating-orcad-catalog/orcad-source-export' +import { + OrcadCatalogImportPersistence, + installOrcadCatalogImportPersistenceContext +} from '../migrating-orcad-catalog/orcad-catalog-import' export type StoreDomainOperations = WriteSchedulingOperations & PrimaryStateWriteOperations & @@ -79,6 +87,8 @@ export type StoreDomainOperations = WriteSchedulingOperations & SshProfileOperations & RetiredWorktreeNamePersistence & SshLeaseRecoveryOperations & + OrcadSourceExportPersistence & + OrcadCatalogImportPersistence & WriteFlushBarrierOperations export type StoreDomains = { @@ -103,6 +113,8 @@ export type StoreDomains = { sshProfiles: SshProfileOperations retiredWorktreeNames: RetiredWorktreeNamePersistence sshLeases: SshLeaseRecoveryOperations + orcadSourceExport: OrcadSourceExportPersistence + orcadCatalogImports: OrcadCatalogImportPersistence } export const STORE_DOMAIN_OPERATION_CLASSES = [ @@ -121,6 +133,8 @@ export const STORE_DOMAIN_OPERATION_CLASSES = [ SshProfileOperations, RetiredWorktreeNamePersistence, SshLeaseRecoveryOperations, + OrcadSourceExportPersistence, + OrcadCatalogImportPersistence, WriteFlushBarrierOperations ] as const @@ -140,6 +154,8 @@ export function installStoreDomainContexts(target: Store, domains: StoreDomains) installSshProfileOperationsContext(target, domains.sshProfiles) installRetiredWorktreeNamePersistenceContext(target, domains.retiredWorktreeNames) installSshLeaseRecoveryOperationsContext(target, domains.sshLeases) + installOrcadSourceExportPersistenceContext(target, domains.orcadSourceExport) + installOrcadCatalogImportPersistenceContext(target, domains.orcadCatalogImports) installWriteFlushBarrierOperationsContext(target, domains.flushBarriers) } @@ -175,6 +191,7 @@ export function createStoreDomains(runtime: StoreRuntimeState): StoreDomains { bindingRecovery, scheduling ) + const orcadCatalogImports = new OrcadCatalogImportPersistence(runtime, repos, scheduling) return { adaptation, cohorts, @@ -196,6 +213,9 @@ export function createStoreDomains(runtime: StoreRuntimeState): StoreDomains { ptyBindings, sshProfiles, retiredWorktreeNames, - sshLeases + sshLeases, + // Read-only: holds the runtime state and nothing that writes. + orcadSourceExport: new OrcadSourceExportPersistence(runtime), + orcadCatalogImports } } diff --git a/src/main/persistence/loading-store/store-runtime-authored-session-writes.test.ts b/src/main/persistence/loading-store/store-runtime-authored-session-writes.test.ts index 8007bfb7ece..1cc673e52b4 100644 --- a/src/main/persistence/loading-store/store-runtime-authored-session-writes.test.ts +++ b/src/main/persistence/loading-store/store-runtime-authored-session-writes.test.ts @@ -185,3 +185,23 @@ describe('Store keeps runtime-authored session fields across desktop writes', () ).toEqual([other]) }) }) + +describe('a host partition written from a per-host renderer snapshot', () => { + it('keeps the maps the snapshot left out, on both desktop write paths', () => { + const store = createStore() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the renderer's per-host split omits maps a host has no rows in. + const slice = { + activeRepoId: null, + activeWorktreeId: null, + unifiedTabs: {} + } as unknown as WorkspaceSessionState + store.setWorkspaceSession(slice, 'runtime:env-1') + store.stageWorkspaceSessionBeforeUnload(slice, HOST_ID) + for (const hostId of ['runtime:env-1', HOST_ID]) { + expect(store.getWorkspaceSession(hostId)).toMatchObject({ + tabsByWorktree: {}, + terminalLayoutsByTabId: {} + }) + } + }) +}) diff --git a/src/main/persistence/loading-store/store-runtime-state.ts b/src/main/persistence/loading-store/store-runtime-state.ts index a580eafcf36..f72bba42ffe 100644 --- a/src/main/persistence/loading-store/store-runtime-state.ts +++ b/src/main/persistence/loading-store/store-runtime-state.ts @@ -44,6 +44,8 @@ export class StoreRuntimeState { automationListProjectionCache: AutomationListProjectionCache | null = null activeViewPreference!: ActiveViewPreference readonly terminalScrollbackSnapshotStorage: TerminalScrollbackSnapshotStorage + /** Scrollback refs each in-flight migration export still reads, keyed by migration id. */ + readonly retainedScrollbackRefsByMigrationId = new Map>() writeTimer: ReturnType | null = null pendingWrite: Promise | null = null pendingSnapshotFileWork: Promise | null = null diff --git a/src/main/persistence/loading-store/store-workspace-session-host-removal.test.ts b/src/main/persistence/loading-store/store-workspace-session-host-removal.test.ts new file mode 100644 index 00000000000..899d5c70abf --- /dev/null +++ b/src/main/persistence/loading-store/store-workspace-session-host-removal.test.ts @@ -0,0 +1,79 @@ +import { closeTestStores, createSqliteTestStore } from '../../persistence-test-harness' +import { mkdtempSync, realpathSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' + +vi.mock('electron', () => ({ + app: { + getPath: () => tmpdir(), + getName: () => 'orca-test', + getVersion: () => '0.0.0-test', + isPackaged: false, + on: () => {}, + whenReady: () => Promise.resolve() + }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + }, + ipcMain: { on: () => {}, handle: () => {} }, + BrowserWindow: { getAllWindows: () => [] } +})) + +const { Store } = await import('./store') + +const REMOVED = 'runtime:removed-env' +const KEPT = 'runtime:kept-env' +const stores: InstanceType[] = [] + +afterEach(async () => { + for (const store of stores.splice(0)) { + store.freezeWrites() + } + await closeTestStores() +}) + +function createStore(dataFile: string): InstanceType { + const store = createSqliteTestStore(Store, { dataFile }) + stores.push(store) + return store +} + +function session(worktreeId: string): WorkspaceSessionState { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the partition's presence is asserted. + return { + activeRepoId: 'repo-1', + activeWorktreeId: worktreeId, + activeTabId: 'tab-1', + tabsByWorktree: { [worktreeId]: [{ id: 'tab-1', worktreeId }] }, + terminalLayoutsByTabId: {} + } as unknown as WorkspaceSessionState +} + +describe('removing a host’s workspace session partition', () => { + it('drops only that host, keeps local, and stays dropped after a reload', async () => { + const dataFile = join( + realpathSync(mkdtempSync(join(tmpdir(), 'orca-host-removal-'))), + 'orca-data.json' + ) + const store = createStore(dataFile) + store.setWorkspaceSession(session('repo-1::/a'), REMOVED) + store.setWorkspaceSession(session('repo-1::/b'), KEPT) + expect(store.getWorkspaceSessionHostIds()).toEqual(expect.arrayContaining([REMOVED, KEPT])) + + store.removeWorkspaceSessionHost(REMOVED) + store.removeWorkspaceSessionHost('local') + + expect(store.getWorkspaceSessionHostIds()).not.toContain(REMOVED) + expect(store.getWorkspaceSessionHostIds()).toEqual(expect.arrayContaining(['local', KEPT])) + store.flush() + store.freezeWrites() + + const reloaded = createStore(dataFile) + expect(reloaded.getWorkspaceSessionHostIds()).not.toContain(REMOVED) + expect(reloaded.getWorkspaceSessionHostIds()).toContain(KEPT) + }) +}) diff --git a/src/main/persistence/loading-store/terminal-session-cleanup.ts b/src/main/persistence/loading-store/terminal-session-cleanup.ts index dede6bf8747..7c9b3b10b11 100644 --- a/src/main/persistence/loading-store/terminal-session-cleanup.ts +++ b/src/main/persistence/loading-store/terminal-session-cleanup.ts @@ -27,14 +27,16 @@ export function workspaceSessionPatchNeedsFullNormalization(patch: WorkspaceSess export function deleteRemovedTerminalScrollbackSnapshots( prior: WorkspaceSessionState | undefined, next: WorkspaceSessionState, - storage?: TerminalScrollbackSnapshotStorage + storage?: TerminalScrollbackSnapshotStorage, + /** Refs a pending migration export still reads; they outlive the session row. */ + retainedRefs: ReadonlySet = new Set() ): void { if (!prior) { return } const nextRefs = collectTerminalScrollbackSnapshotRefs(next) for (const ref of collectTerminalScrollbackSnapshotRefs(prior)) { - if (!nextRefs.has(ref)) { + if (!nextRefs.has(ref) && !retainedRefs.has(ref)) { deleteTerminalScrollbackSnapshotSync(ref, storage) } } diff --git a/src/main/persistence/loading-store/workspace-session-snapshot-publication.ts b/src/main/persistence/loading-store/workspace-session-snapshot-publication.ts index 96834ffbcf9..178f1d833ff 100644 --- a/src/main/persistence/loading-store/workspace-session-snapshot-publication.ts +++ b/src/main/persistence/loading-store/workspace-session-snapshot-publication.ts @@ -104,7 +104,8 @@ export function setLocalWorkspaceSession( deleteRemovedTerminalScrollbackSnapshots( prior, session, - context.runtime.terminalScrollbackSnapshotStorage + context.runtime.terminalScrollbackSnapshotStorage, + retainedScrollbackRefs(context.runtime) ) } context.runtime.state.workspaceSession = session @@ -140,7 +141,8 @@ export function enqueueTerminalScrollbackSnapshotWork( await deleteRemovedTerminalScrollbackSnapshotsAsync( prior, context.runtime.state.workspaceSession, - context.runtime.terminalScrollbackSnapshotStorage + context.runtime.terminalScrollbackSnapshotStorage, + retainedScrollbackRefs(context.runtime) ) } return @@ -159,14 +161,16 @@ export function enqueueTerminalScrollbackSnapshotWork( await deleteRemovedTerminalScrollbackSnapshotsAsync( migrated, current, - context.runtime.terminalScrollbackSnapshotStorage + context.runtime.terminalScrollbackSnapshotStorage, + retainedScrollbackRefs(context.runtime) ) } if (current) { await deleteRemovedTerminalScrollbackSnapshotsAsync( prior, current, - context.runtime.terminalScrollbackSnapshotStorage + context.runtime.terminalScrollbackSnapshotStorage, + retainedScrollbackRefs(context.runtime) ) } }) @@ -180,3 +184,11 @@ export function enqueueTerminalScrollbackSnapshotWork( }) context.runtime.pendingSnapshotFileWork = work } + +function retainedScrollbackRefs(runtime: { + retainedScrollbackRefsByMigrationId: ReadonlyMap> +}): ReadonlySet { + return new Set( + [...runtime.retainedScrollbackRefsByMigrationId.values()].flatMap((refs) => [...refs]) + ) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts new file mode 100644 index 00000000000..fefa4e43b6e --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-import.ts @@ -0,0 +1,284 @@ +import { + MAX_ORCAD_MIGRATION_STAGED_CATALOGS, + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationCatalogAbortResult, + type OrcadMigrationCatalogState, + type OrcadMigrationImportReceipt, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { + OrcadMigrationSnapshotChunkRequest, + OrcadMigrationSnapshotChunkResult +} from '../../../shared/orcad-migration-scrollback' +import type { RepoLifecycleOperations } from '../loading-store/repo-lifecycle-operations' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import type { WriteSchedulingOperations } from '../loading-store/write-scheduling' +import { scheduleSave } from '../loading-store/write-scheduling' +import { + applyPreparedOrcadMigrationCatalog, + assertOrcadMigrationReceiptMatchesManifest, + assertSameOrcadMigrationManifest, + prepareOrcadMigrationCatalog, + type PreparedOrcadMigrationCatalog +} from './orcad-catalog-records' +import { + expireOrcadMigrationStages, + findOrcadMigrationImportReceipt, + isLiveOrcadMigrationStage, + recordOrcadMigrationImportReceipt +} from './orcad-catalog-receipt-ledger' +import { + abortOrcadMigrationSnapshots, + assertOrcadMigrationSnapshotsReady, + commitOrcadMigrationSnapshots, + inspectOrcadMigrationSnapshotUploads, + pruneOrcadMigrationSnapshotStaging, + stageOrcadMigrationSnapshotChunk +} from './orcad-scrollback-snapshot-transfer' + +type OrcadCatalogImportRuntime = Pick< + StoreRuntimeState, + 'state' | 'terminalScrollbackSnapshotStorage' +> + +const orcadCatalogImportContext = Symbol('OrcadCatalogImportPersistence') +type OrcadCatalogImportContext = { + runtime: OrcadCatalogImportRuntime + repos: RepoLifecycleOperations + scheduling: WriteSchedulingOperations +} + +// Manifests arrive parsed; the RPC boundary verified the digest over the bytes as sent. +export class OrcadCatalogImportPersistence { + readonly [orcadCatalogImportContext]: OrcadCatalogImportContext + + constructor( + runtime: OrcadCatalogImportRuntime, + repos: RepoLifecycleOperations, + scheduling: WriteSchedulingOperations + ) { + this[orcadCatalogImportContext] = { runtime, repos, scheduling } + } + + stageOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + options: { now?: () => Date } = {} + ): OrcadMigrationCatalogState { + const context = this[orcadCatalogImportContext] + const now = options.now ?? (() => new Date()) + if (expireOrcadMigrationStages(context.runtime.state, now().getTime())) { + scheduleSave(context.scheduling) + } + pruneOrcadMigrationSnapshotStaging( + stagedManifests(context.runtime.state), + context.runtime.terminalScrollbackSnapshotStorage + ) + const current = migrationCatalogState( + context.runtime.state, + manifest, + context.runtime.terminalScrollbackSnapshotStorage + ) + if (current.state === 'committed') { + return current + } + prepareOrcadMigrationCatalog(manifest, context.runtime.state) + if (current.state === 'staged') { + return current + } + const staged = context.runtime.state.orcadMigrationStagedCatalogs ?? [] + if (staged.length >= MAX_ORCAD_MIGRATION_STAGED_CATALOGS) { + throw new Error('orcad_migration_staging_capacity_exceeded') + } + const stagedAt = now().toISOString() + context.runtime.state.orcadMigrationStagedCatalogs = [ + ...staged, + { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + manifest: structuredClone(manifest), + stagedAt + } + ] + scheduleSave(context.scheduling) + return stagedCatalogState(manifest, stagedAt, context.runtime.terminalScrollbackSnapshotStorage) + } + + stageOrcadMigrationSnapshotChunk( + request: OrcadMigrationSnapshotChunkRequest + ): OrcadMigrationSnapshotChunkResult { + const context = this[orcadCatalogImportContext] + return stageOrcadMigrationSnapshotChunk({ + stagedManifest: + stagedManifests(context.runtime.state).find( + (manifest) => manifest.migrationId === request.migrationId + ) ?? null, + storage: context.runtime.terminalScrollbackSnapshotStorage, + request + }) + } + + commitStagedOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + options: { now?: () => Date } = {} + ): OrcadMigrationCatalogState { + const context = this[orcadCatalogImportContext] + const current = migrationCatalogState( + context.runtime.state, + manifest, + context.runtime.terminalScrollbackSnapshotStorage + ) + if (current.state === 'committed') { + return current + } + if (current.state !== 'staged') { + throw new Error('orcad_migration_catalog_not_staged') + } + const prepared = prepareOrcadMigrationCatalog(manifest, context.runtime.state) + assertOrcadMigrationSnapshotsReady(manifest, context.runtime.terminalScrollbackSnapshotStorage) + commitOrcadMigrationSnapshots(manifest, context.runtime.terminalScrollbackSnapshotStorage) + const receipt = commitPreparedCatalog(context, manifest, prepared, options.now) + scheduleSave(context.scheduling) + return committedCatalogState(receipt) + } + + abortStagedOrcadMigrationCatalog( + manifest: OrcadMigrationManifest + ): OrcadMigrationCatalogAbortResult { + const context = this[orcadCatalogImportContext] + const current = migrationCatalogState( + context.runtime.state, + manifest, + context.runtime.terminalScrollbackSnapshotStorage + ) + if (current.state === 'committed' || current.state === 'absent') { + return { ...current, aborted: false } + } + // Dormant import only: nothing but this stage references the staged rows before commit. + context.runtime.state.orcadMigrationStagedCatalogs = ( + context.runtime.state.orcadMigrationStagedCatalogs ?? [] + ).filter((entry) => entry.manifest.migrationId !== manifest.migrationId) + abortOrcadMigrationSnapshots(manifest, context.runtime.terminalScrollbackSnapshotStorage) + scheduleSave(context.scheduling) + return { ...absentCatalogState(manifest), aborted: true } + } + + getOrcadMigrationCatalogState(manifest: OrcadMigrationManifest): OrcadMigrationCatalogState { + const context = this[orcadCatalogImportContext] + return migrationCatalogState( + context.runtime.state, + manifest, + context.runtime.terminalScrollbackSnapshotStorage + ) + } + + /** A migration into this server staged recently and neither committed nor aborted. */ + hasStagedOrcadMigrationCatalog(now: number = Date.now()): boolean { + return (this[orcadCatalogImportContext].runtime.state.orcadMigrationStagedCatalogs ?? []).some( + (entry) => isLiveOrcadMigrationStage(entry.stagedAt, now) + ) + } +} + +function commitPreparedCatalog( + context: OrcadCatalogImportContext, + manifest: OrcadMigrationManifest, + prepared: PreparedOrcadMigrationCatalog, + now?: () => Date +): OrcadMigrationImportReceipt { + applyPreparedOrcadMigrationCatalog(prepared, context.runtime.state, context.repos) + const receipt: OrcadMigrationImportReceipt = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256, + source: structuredClone(manifest.source), + importedAt: (now ?? (() => new Date()))().toISOString(), + repositoryIds: prepared.repositories.map((repo) => repo.id), + projectGroupIds: prepared.projectGroups.map((group) => group.id), + folderWorkspaceIds: prepared.folderWorkspaces.map((workspace) => workspace.id) + } + context.runtime.state.orcadMigrationStagedCatalogs = ( + context.runtime.state.orcadMigrationStagedCatalogs ?? [] + ).filter((entry) => entry.manifest.migrationId !== manifest.migrationId) + recordOrcadMigrationImportReceipt(context.runtime.state, receipt) + return receipt +} + +function migrationCatalogState( + state: StoreRuntimeState['state'], + manifest: OrcadMigrationManifest, + storage?: StoreRuntimeState['terminalScrollbackSnapshotStorage'] +): OrcadMigrationCatalogState { + const receipt = findOrcadMigrationImportReceipt(state, manifest) + if (receipt) { + assertCommittedReceipt(receipt, manifest) + return committedCatalogState(receipt) + } + const staged = state.orcadMigrationStagedCatalogs?.find( + (entry) => entry.manifest.migrationId === manifest.migrationId + ) + if (!staged) { + return absentCatalogState(manifest) + } + assertSameOrcadMigrationManifest(staged.manifest, manifest) + return stagedCatalogState(manifest, staged.stagedAt, storage) +} + +function stagedManifests(state: StoreRuntimeState['state']): OrcadMigrationManifest[] { + return (state.orcadMigrationStagedCatalogs ?? []).map((entry) => entry.manifest) +} + +/** + * The receipt alone proves the commit: the server is live afterwards, so its rows and snapshot + * files may legitimately change, and a later read must still say "committed". + */ +function assertCommittedReceipt( + receipt: OrcadMigrationImportReceipt, + manifest: OrcadMigrationManifest +): void { + if (receipt.manifestSha256 !== manifest.manifestSha256) { + throw new Error('orcad_migration_id_reused_with_different_manifest') + } + assertOrcadMigrationReceiptMatchesManifest(receipt, manifest) +} + +function absentCatalogState(manifest: OrcadMigrationManifest): OrcadMigrationCatalogState { + return { + state: 'absent', + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256 + } +} + +function stagedCatalogState( + manifest: OrcadMigrationManifest, + stagedAt: string, + storage?: StoreRuntimeState['terminalScrollbackSnapshotStorage'] +): OrcadMigrationCatalogState { + const snapshotUploads = storage + ? inspectOrcadMigrationSnapshotUploads(manifest, storage) + : undefined + return { + state: 'staged', + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256, + stagedAt, + ...(snapshotUploads ? { snapshotUploads } : {}) + } +} + +function committedCatalogState(receipt: OrcadMigrationImportReceipt): OrcadMigrationCatalogState { + return { + state: 'committed', + migrationId: receipt.migrationId, + manifestSha256: receipt.manifestSha256, + receipt: structuredClone(receipt) + } +} + +export function installOrcadCatalogImportPersistenceContext( + target: OrcadCatalogImportPersistence, + source: OrcadCatalogImportPersistence +): void { + Object.defineProperty(target, orcadCatalogImportContext, { + value: source[orcadCatalogImportContext] + }) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.test.ts new file mode 100644 index 00000000000..05911af392a --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import { MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS } from '../../../shared/orcad-migration-manifest' +import { manifest } from '../../persistence-orcad-migration-catalog-fixture' +import { + expireOrcadMigrationStages, + findOrcadMigrationImportReceipt, + ORCAD_MIGRATION_STAGE_TTL_MS, + recordOrcadMigrationImportReceipt +} from './orcad-catalog-receipt-ledger' + +const empty = { repositories: [], projectGroups: [], folderWorkspaces: [] } + +function receiptFor(migrationId: string) { + const input = manifest({ migrationId, payload: empty }) + return { + input, + receipt: { + version: input.version, + migrationId, + manifestSha256: input.manifestSha256, + source: input.source, + importedAt: '2026-10-01T00:00:00.000Z', + repositoryIds: [], + projectGroupIds: [], + folderWorkspaceIds: [] + } + } +} + +describe('orcad migration receipt ledger', () => { + it('keeps answering for a commit whose receipt aged out of the bounded list', () => { + const state = getDefaultPersistedState('/tmp/orcad-receipt-ledger') + const first = receiptFor('migration-0') + recordOrcadMigrationImportReceipt(state, first.receipt) + for (let index = 1; index <= MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS; index++) { + recordOrcadMigrationImportReceipt(state, receiptFor(`migration-${index}`).receipt) + } + expect(state.orcadMigrationImportReceipts).toHaveLength(MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS) + expect(findOrcadMigrationImportReceipt(state, first.input)).toEqual(first.receipt) + expect(findOrcadMigrationImportReceipt(state, receiptFor('never').input)).toBeUndefined() + }) + + it('expires a stage no client came back for', () => { + const state = getDefaultPersistedState('/tmp/orcad-receipt-ledger') + const stagedAt = '2026-10-01T00:00:00.000Z' + state.orcadMigrationStagedCatalogs = [{ version: 1, manifest: manifest(), stagedAt }] + const start = Date.parse(stagedAt) + expect(expireOrcadMigrationStages(state, start + ORCAD_MIGRATION_STAGE_TTL_MS - 1)).toBe(false) + expect(expireOrcadMigrationStages(state, start + ORCAD_MIGRATION_STAGE_TTL_MS)).toBe(true) + expect(state.orcadMigrationStagedCatalogs).toEqual([]) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.ts new file mode 100644 index 00000000000..9e3ec1461bf --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-receipt-ledger.ts @@ -0,0 +1,78 @@ +/** + * The server's record of which migrations it committed and which are still staged. A commit must + * read as committed for as long as a client could ask, and an abandoned stage must not hold the + * server awake or take a staging slot forever. + */ +import { + MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS, + type OrcadMigrationImportReceipt, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import { MAX_ORCAD_MIGRATION_EVICTED_RECEIPTS } from '../../../shared/orcad-migration-evicted-receipts' +import type { PersistedState } from '../../../shared/persisted-state-types' + +// A client resumes a stage within minutes; a week only covers a host left offline meanwhile. +export const ORCAD_MIGRATION_STAGE_TTL_MS = 7 * 24 * 60 * 60 * 1000 + +export function findOrcadMigrationImportReceipt( + state: PersistedState, + manifest: OrcadMigrationManifest +): OrcadMigrationImportReceipt | undefined { + const receipt = state.orcadMigrationImportReceipts?.find( + (entry) => entry.migrationId === manifest.migrationId + ) + if (receipt) { + return receipt + } + const evicted = state.orcadMigrationEvictedReceipts?.find( + (entry) => entry.migrationId === manifest.migrationId + ) + // Everything else a receipt holds is the manifest's own: rebuilt, it still checks against it. + return evicted + ? { + version: manifest.version, + migrationId: evicted.migrationId, + manifestSha256: evicted.manifestSha256, + source: structuredClone(manifest.source), + importedAt: evicted.importedAt, + repositoryIds: manifest.payload.repositories.map((repo) => repo.id), + projectGroupIds: manifest.payload.projectGroups.map((group) => group.id), + folderWorkspaceIds: manifest.payload.folderWorkspaces.map((workspace) => workspace.id) + } + : undefined +} + +/** Appends a receipt; one that ages out of the bounded list keeps its commit as a compact record. */ +export function recordOrcadMigrationImportReceipt( + state: PersistedState, + receipt: OrcadMigrationImportReceipt +): void { + const receipts = [...(state.orcadMigrationImportReceipts ?? []), receipt] + const overflow = receipts.length - MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS + if (overflow > 0) { + state.orcadMigrationEvictedReceipts = [ + ...(state.orcadMigrationEvictedReceipts ?? []), + ...receipts.slice(0, overflow).map(({ migrationId, manifestSha256, importedAt }) => ({ + migrationId, + manifestSha256, + importedAt + })) + ].slice(-MAX_ORCAD_MIGRATION_EVICTED_RECEIPTS) + } + state.orcadMigrationImportReceipts = receipts.slice(-MAX_ORCAD_MIGRATION_IMPORT_RECEIPTS) +} + +export function isLiveOrcadMigrationStage(stagedAt: string, now: number): boolean { + return now - Date.parse(stagedAt) < ORCAD_MIGRATION_STAGE_TTL_MS +} + +/** Drops stages no client came back for; returns whether any went. */ +export function expireOrcadMigrationStages(state: PersistedState, now: number): boolean { + const staged = state.orcadMigrationStagedCatalogs ?? [] + const live = staged.filter((entry) => isLiveOrcadMigrationStage(entry.stagedAt, now)) + if (live.length === staged.length) { + return false + } + state.orcadMigrationStagedCatalogs = live + return true +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-records.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-records.ts new file mode 100644 index 00000000000..f1d375baa04 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-records.ts @@ -0,0 +1,171 @@ +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import { + serializeOrcadMigrationValue, + type OrcadMigrationImportReceipt, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { Repo } from '../../../shared/repo-types' +import { normalizeRuntimePathForComparison } from '../../../shared/cross-platform-path' +import { assertOrcadMigrationStagedCatalogClaims } from './orcad-staged-catalog-claims' +import { + toOrcadDestinationFolderWorkspace, + toOrcadDestinationProjectGroup, + toOrcadDestinationRepository +} from './orcad-destination-catalog-projection' +import { + syncProjectHostSetupCompatibilityState, + type RepoLifecycleOperations +} from '../loading-store/repo-lifecycle-operations' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import { + applyPreparedOrcadMigrationDormantState, + prepareOrcadMigrationDormantState, + type PreparedOrcadMigrationDormantState +} from './orcad-dormant-state-records' +import { selectNewRows } from './orcad-catalog-row-identity' + +export type PreparedOrcadMigrationCatalog = { + repositories: Repo[] + projectGroups: ProjectGroup[] + folderWorkspaces: FolderWorkspace[] + newRepositories: Repo[] + newProjectGroups: ProjectGroup[] + newFolderWorkspaces: FolderWorkspace[] + dormantState: PreparedOrcadMigrationDormantState +} + +export function assertSameOrcadMigrationManifest( + staged: OrcadMigrationManifest, + requested: OrcadMigrationManifest +): void { + if (staged.manifestSha256 !== requested.manifestSha256) { + throw new Error('orcad_migration_id_reused_with_different_manifest') + } +} + +// Dormant only: no live PTY reaches the destination, so the incoming session is compared as is. +export function prepareOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + state: StoreRuntimeState['state'] +): PreparedOrcadMigrationCatalog { + assertOrcadMigrationStagedCatalogClaims(manifest, state.orcadMigrationStagedCatalogs ?? []) + const repositories = manifest.payload.repositories.map(toOrcadDestinationRepository) + const projectGroups = manifest.payload.projectGroups.map(toOrcadDestinationProjectGroup) + const folderWorkspaces = manifest.payload.folderWorkspaces.map(toOrcadDestinationFolderWorkspace) + const newRepositories = selectNewRows(repositories, state.repos, catalogConflict('repository')) + const newProjectGroups = selectNewRows( + projectGroups, + state.projectGroups, + catalogConflict('project_group') + ) + const newFolderWorkspaces = selectNewRows( + folderWorkspaces, + state.folderWorkspaces, + catalogConflict('folder_workspace') + ) + assertNoRepositoryPathConflicts(repositories, state.repos) + assertCatalogReferences({ + repositories, + projectGroups, + folderWorkspaces, + existingProjectGroups: state.projectGroups + }) + const dormantState = prepareOrcadMigrationDormantState(manifest, state) + return { + repositories, + projectGroups, + folderWorkspaces, + newRepositories, + newProjectGroups, + newFolderWorkspaces, + dormantState + } +} + +export function applyPreparedOrcadMigrationCatalog( + prepared: PreparedOrcadMigrationCatalog, + state: StoreRuntimeState['state'], + repos: RepoLifecycleOperations +): void { + state.projectGroups.push(...prepared.newProjectGroups) + state.repos.push(...prepared.newRepositories) + state.folderWorkspaces.push(...prepared.newFolderWorkspaces) + applyPreparedOrcadMigrationDormantState(prepared.dormantState, state) + if (prepared.newRepositories.length > 0) { + syncProjectHostSetupCompatibilityState(repos) + } +} + +export function assertOrcadMigrationReceiptMatchesManifest( + receipt: OrcadMigrationImportReceipt, + manifest: OrcadMigrationManifest +): void { + const expected = { + source: manifest.source, + repositoryIds: manifest.payload.repositories.map((repo) => repo.id), + projectGroupIds: manifest.payload.projectGroups.map((group) => group.id), + folderWorkspaceIds: manifest.payload.folderWorkspaces.map((workspace) => workspace.id) + } + const actual = { + source: receipt.source, + repositoryIds: receipt.repositoryIds, + projectGroupIds: receipt.projectGroupIds, + folderWorkspaceIds: receipt.folderWorkspaceIds + } + if (serializeOrcadMigrationValue(actual) !== serializeOrcadMigrationValue(expected)) { + throw new Error('orcad_migration_receipt_manifest_mismatch') + } +} + +function catalogConflict(label: string): (id: string) => string { + return (id) => `orcad_migration_${label}_id_conflict:${id}` +} + +function assertNoRepositoryPathConflicts(incoming: Repo[], existing: Repo[]): void { + const incomingIdSet = new Set(incoming.map((repo) => repo.id)) + const incomingByPath = new Map() + for (const repo of incoming) { + const key = normalizeRuntimePathForComparison(repo.path) + const priorId = incomingByPath.get(key) + if (priorId && priorId !== repo.id) { + throw new Error(`orcad_migration_repository_path_conflict:${repo.path}`) + } + incomingByPath.set(key, repo.id) + } + for (const repo of existing) { + if ( + !incomingIdSet.has(repo.id) && + incomingByPath.has(normalizeRuntimePathForComparison(repo.path)) + ) { + throw new Error(`orcad_migration_repository_path_conflict:${repo.path}`) + } + } +} + +function assertCatalogReferences(args: { + repositories: Repo[] + projectGroups: ProjectGroup[] + folderWorkspaces: FolderWorkspace[] + existingProjectGroups: ProjectGroup[] +}): void { + const groupIds = new Set([ + ...args.existingProjectGroups.map((group) => group.id), + ...args.projectGroups.map((group) => group.id) + ]) + for (const group of args.projectGroups) { + if (group.parentGroupId && !groupIds.has(group.parentGroupId)) { + throw new Error(`orcad_migration_project_group_parent_missing:${group.id}`) + } + } + for (const repo of args.repositories) { + if (repo.projectGroupId && !groupIds.has(repo.projectGroupId)) { + throw new Error(`orcad_migration_repository_project_group_missing:${repo.id}`) + } + } + for (const workspace of args.folderWorkspaces) { + if (!groupIds.has(workspace.projectGroupId)) { + throw new Error(`orcad_migration_folder_workspace_project_group_missing:${workspace.id}`) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-catalog-row-identity.ts b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-row-identity.ts new file mode 100644 index 00000000000..f78c4f48cd4 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-catalog-row-identity.ts @@ -0,0 +1,26 @@ +/** Merging migrated rows by id: a new id is added, a known id must carry the identical row. */ +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' + +export function selectNewRows( + incoming: T[], + existing: T[], + conflictError: (id: string) => string +): T[] { + const existingById = new Map(existing.map((row) => [row.id, row])) + return incoming.filter((row) => { + const current = existingById.get(row.id) + if (!current) { + return true + } + if (serializeOrcadMigrationValue(current) !== serializeOrcadMigrationValue(row)) { + throw new Error(conflictError(row.id)) + } + return false + }) +} + +export function assertSameValue(left: unknown, right: unknown, label: string): void { + if (serializeOrcadMigrationValue(left) !== serializeOrcadMigrationValue(right)) { + throw new Error(`orcad_migration_dormant_id_conflict:${label}`) + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-automation-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-automation-state.ts new file mode 100644 index 00000000000..5dd4031eebd --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-automation-state.ts @@ -0,0 +1,94 @@ +import { MAX_AUTOMATION_RUNS_PER_AUTOMATION } from '../../../shared/automation-run-retention' +import { + isFinalAutomationRunStatus, + type Automation, + type AutomationRun +} from '../../../shared/automations-types' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { selectNewRows } from './orcad-catalog-row-identity' + +export type PreparedOrcadMigrationAutomationState = { + incomingAutomations: Automation[] + incomingRuns: AutomationRun[] + newAutomations: Automation[] + newRuns: AutomationRun[] +} + +export function prepareOrcadMigrationAutomationState( + automations: readonly Automation[] | undefined, + runs: readonly AutomationRun[] | undefined, + state: PersistedState +): PreparedOrcadMigrationAutomationState { + const incomingAutomations = (automations ?? []).map((entry) => structuredClone(entry)) + const incomingRuns = (runs ?? []).map((entry) => structuredClone(entry)) + const newAutomations = selectNewRows( + incomingAutomations, + state.automations, + dormantConflict('automation') + ) + const newRuns = selectNewRows( + incomingRuns, + state.automationRuns, + dormantConflict('automation_run') + ) + assertRunOwnersExist(incomingAutomations, incomingRuns, state.automations) + assertRunRetentionCapacity(incomingAutomations, incomingRuns, state.automationRuns) + return { incomingAutomations, incomingRuns, newAutomations, newRuns } +} + +export function applyPreparedOrcadMigrationAutomationState( + prepared: PreparedOrcadMigrationAutomationState, + state: PersistedState +): void { + if (prepared.newAutomations.length > 0) { + state.automations = [...state.automations, ...prepared.newAutomations] + } + if (prepared.newRuns.length > 0) { + state.automationRuns = [...state.automationRuns, ...prepared.newRuns] + } +} + +function assertRunOwnersExist( + incomingAutomations: readonly Automation[], + incomingRuns: readonly AutomationRun[], + existingAutomations: readonly Automation[] +): void { + const automationIds = new Set([ + ...existingAutomations.map((entry) => entry.id), + ...incomingAutomations.map((entry) => entry.id) + ]) + for (const run of incomingRuns) { + if (!automationIds.has(run.automationId)) { + throw new Error(`orcad_migration_dormant_automation_run_owner_missing:${run.id}`) + } + } +} + +function assertRunRetentionCapacity( + incomingAutomations: readonly Automation[], + incomingRuns: readonly AutomationRun[], + existingRuns: readonly AutomationRun[] +): void { + const incomingAutomationIds = new Set(incomingAutomations.map((entry) => entry.id)) + const idsByAutomation = new Map>() + for (const run of [ + ...existingRuns.filter((entry) => incomingAutomationIds.has(entry.automationId)), + ...incomingRuns + ]) { + if (!isFinalAutomationRunStatus(run.status)) { + throw new Error(`orcad_migration_dormant_automation_run_active:${run.automationId}`) + } + const ids = idsByAutomation.get(run.automationId) ?? new Set() + ids.add(run.id) + idsByAutomation.set(run.automationId, ids) + } + for (const [automationId, ids] of idsByAutomation) { + if (ids.size > MAX_AUTOMATION_RUNS_PER_AUTOMATION) { + throw new Error(`orcad_migration_dormant_automation_run_capacity:${automationId}`) + } + } +} + +function dormantConflict(label: string): (id: string) => string { + return (id) => `orcad_migration_dormant_id_conflict:${label}:${id}` +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-catalog-projection.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-catalog-projection.ts new file mode 100644 index 00000000000..20cd8b9c8bd --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-catalog-projection.ts @@ -0,0 +1,26 @@ +/** How a source catalog row looks once a local orcad owns it: no SSH connection or host. */ +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { Repo } from '../../../shared/repo-types' + +export function toOrcadDestinationRepository(source: Repo): Repo { + const destination = structuredClone(source) + delete destination.connectionId + delete destination.executionHostId + return destination +} + +export function toOrcadDestinationProjectGroup(source: ProjectGroup): ProjectGroup { + const destination = structuredClone(source) + destination.connectionId = null + delete destination.executionHostId + return destination +} + +export function toOrcadDestinationFolderWorkspace(source: FolderWorkspace): FolderWorkspace { + const destination = structuredClone(source) + destination.connectionId = null + delete destination.executionHostId + destination.linkedTaskSourceContext ??= null + return destination +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.test.ts new file mode 100644 index 00000000000..2c8d4f8e80a --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { OrcadMigrationClientStatePayload } from '../../../shared/orcad-migration-client-state' +import { + applyPreparedOrcadMigrationClientState, + prepareOrcadMigrationClientState +} from './orcad-destination-client-state' + +function state(): PersistedState { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a minimal persisted state; the code under test reads only the client-state fields set here. + return { + ui: { + lastActiveRepoId: null, + lastActiveWorktreeId: null, + filterRepoIds: [], + showDotfilesByWorktree: {}, + setupScriptPromptDismissedRepoIds: [], + manualRepoOrder: [], + workspaceHostScope: undefined, + visibleWorkspaceHostIds: null, + workspaceHostOrder: [], + automationHostFilter: { kind: 'all' }, + acknowledgedAgentsByPaneKey: {} + } + } as unknown as PersistedState +} + +const incoming: OrcadMigrationClientStatePayload = { + mobileClientTabSelectionsByDeviceId: { + phone: { + 'repo-1::/worktree': { + activeTabId: 'tab-1', + activeGroupId: null, + activeTabIdByGroupId: {} + } + } + }, + uiRouting: { + lastActiveRepoId: 'repo-1', + lastActiveWorktreeId: 'repo-1::/worktree', + filterRepoIds: ['repo-1'], + showDotfilesByWorktree: { 'repo-1::/worktree': false }, + manualRepoOrder: [{ hostId: 'local', repoId: 'repo-1' }] + } +} + +describe('destination client-state migration', () => { + it('applies selections and routing while preserving unrelated device state', () => { + const destination = state() + destination.mobileClientTabSelectionsByDeviceId = { + tablet: { + 'repo-other::/worktree': { + activeTabId: null, + activeGroupId: null, + activeTabIdByGroupId: {} + } + } + } + const prepared = prepareOrcadMigrationClientState(incoming, destination) + applyPreparedOrcadMigrationClientState(prepared, destination) + + expect(destination.mobileClientTabSelectionsByDeviceId).toMatchObject({ + phone: incoming.mobileClientTabSelectionsByDeviceId?.phone, + tablet: expect.any(Object) + }) + expect(destination.ui).toMatchObject({ + lastActiveRepoId: 'repo-1', + lastActiveWorktreeId: 'repo-1::/worktree', + filterRepoIds: ['repo-1'], + showDotfilesByWorktree: { 'repo-1::/worktree': false } + }) + }) + + it("keeps a device's selections for workspaces outside the import", () => { + const destination = state() + const kept = { activeTabId: 'kept', activeGroupId: null, activeTabIdByGroupId: {} } + destination.mobileClientTabSelectionsByDeviceId = { phone: { 'old::/workspace': kept } } + applyPreparedOrcadMigrationClientState( + prepareOrcadMigrationClientState(incoming, destination), + destination + ) + expect(destination.mobileClientTabSelectionsByDeviceId?.phone).toEqual({ + 'old::/workspace': kept, + ...incoming.mobileClientTabSelectionsByDeviceId?.phone + }) + }) + + it('rejects a conflicting destination selection before publication', () => { + const destination = state() + destination.mobileClientTabSelectionsByDeviceId = { + phone: { + 'repo-1::/worktree': { + activeTabId: 'different-tab', + activeGroupId: null, + activeTabIdByGroupId: {} + } + } + } + expect(() => prepareOrcadMigrationClientState(incoming, destination)).toThrow( + 'orcad_migration_client_state_conflict:mobile' + ) + }) + + it('rejects non-default UI routing conflicts', () => { + const destination = state() + destination.ui.lastActiveRepoId = 'other-repo' + expect(() => prepareOrcadMigrationClientState(incoming, destination)).toThrow( + 'orcad_migration_client_state_conflict:ui:lastActiveRepoId' + ) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.ts new file mode 100644 index 00000000000..2d1d3e02d26 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.ts @@ -0,0 +1,181 @@ +import type { PersistedState } from '../../../shared/persisted-state-types' +import { hostStableKey, parseHostStableKey } from '../../../shared/automation-owner-key' +import type { + OrcadMigrationClientStatePayload, + OrcadMigrationUiRoutingState +} from '../../../shared/orcad-migration-client-state' +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' + +export type PreparedOrcadMigrationClientState = { + incoming: OrcadMigrationClientStatePayload | undefined + mobileSelections: NonNullable< + OrcadMigrationClientStatePayload['mobileClientTabSelectionsByDeviceId'] + > + uiRouting: OrcadMigrationUiRoutingState | undefined +} + +export function prepareOrcadMigrationClientState( + incoming: OrcadMigrationClientStatePayload | undefined, + state: PersistedState +): PreparedOrcadMigrationClientState { + const mobileSelections = prepareMobileSelections( + incoming?.mobileClientTabSelectionsByDeviceId, + state + ) + const uiRouting = incoming?.uiRouting ? normalizeUiRouting(incoming.uiRouting) : undefined + if (uiRouting) { + assertUiRoutingCompatible(uiRouting, state) + } + return { + incoming, + mobileSelections, + uiRouting + } +} + +export function applyPreparedOrcadMigrationClientState( + prepared: PreparedOrcadMigrationClientState, + state: PersistedState +): void { + if (Object.keys(prepared.mobileSelections).length > 0) { + // Per workspace: a device's selections outside this import stay as they are. + state.mobileClientTabSelectionsByDeviceId = { ...state.mobileClientTabSelectionsByDeviceId } + for (const [deviceId, selections] of Object.entries(prepared.mobileSelections)) { + state.mobileClientTabSelectionsByDeviceId[deviceId] = { + ...state.mobileClientTabSelectionsByDeviceId[deviceId], + ...selections + } + } + } + if (prepared.uiRouting) { + applyUiRouting(prepared.uiRouting, state) + } +} + +function normalizeUiRouting(route: OrcadMigrationUiRoutingState): OrcadMigrationUiRoutingState { + const filter = route.automationHostFilter + if (filter?.kind !== 'host') { + return structuredClone(route) + } + const parsed = parseHostStableKey(filter.hostKey) + if (parsed?.authority.kind !== 'runtime' || parsed.selector.kind !== 'self') { + return structuredClone(route) + } + return { + ...structuredClone(route), + automationHostFilter: { + kind: 'host', + hostKey: hostStableKey({ authority: { kind: 'desktop' }, selector: { kind: 'self' } }) + } + } +} + +function prepareMobileSelections( + incoming: OrcadMigrationClientStatePayload['mobileClientTabSelectionsByDeviceId'], + state: PersistedState +): NonNullable { + const result: NonNullable< + OrcadMigrationClientStatePayload['mobileClientTabSelectionsByDeviceId'] + > = {} + const current = state.mobileClientTabSelectionsByDeviceId ?? {} + for (const [deviceId, selections] of Object.entries(incoming ?? {})) { + for (const [worktreeId, selection] of Object.entries(selections)) { + const existing = current[deviceId]?.[worktreeId] + if ( + existing && + serializeOrcadMigrationValue(existing) !== serializeOrcadMigrationValue(selection) + ) { + throw new Error(`orcad_migration_client_state_conflict:mobile:${deviceId}:${worktreeId}`) + } + const deviceSelections = (result[deviceId] ??= {}) + deviceSelections[worktreeId] = structuredClone(selection) + } + } + return result +} + +// What each routed field reads in this profile, and what it holds when never set. +const UI_ROUTING_FIELDS: readonly { + key: keyof OrcadMigrationUiRoutingState + read: (ui: PersistedState['ui']) => unknown + empty: unknown +}[] = [ + { key: 'lastActiveRepoId', read: (ui) => ui.lastActiveRepoId, empty: null }, + { key: 'lastActiveWorktreeId', read: (ui) => ui.lastActiveWorktreeId, empty: null }, + { key: 'filterRepoIds', read: (ui) => ui.filterRepoIds, empty: [] }, + { key: 'showDotfilesByWorktree', read: (ui) => ui.showDotfilesByWorktree ?? {}, empty: {} }, + { + key: 'setupScriptPromptDismissedRepoIds', + read: (ui) => ui.setupScriptPromptDismissedRepoIds ?? [], + empty: [] + }, + { key: 'manualRepoOrder', read: (ui) => ui.manualRepoOrder ?? [], empty: [] }, + { key: 'workspaceHostScope', read: (ui) => ui.workspaceHostScope, empty: undefined }, + { key: 'visibleWorkspaceHostIds', read: (ui) => ui.visibleWorkspaceHostIds, empty: null }, + { key: 'workspaceHostOrder', read: (ui) => ui.workspaceHostOrder ?? [], empty: [] }, + { key: 'automationHostFilter', read: (ui) => ui.automationHostFilter, empty: undefined }, + { + key: 'acknowledgedAgentsByPaneKey', + read: (ui) => ui.acknowledgedAgentsByPaneKey ?? {}, + empty: {} + } +] + +function assertUiRoutingCompatible( + route: OrcadMigrationUiRoutingState, + state: PersistedState +): void { + for (const { key, read, empty } of UI_ROUTING_FIELDS) { + const current = serializeOrcadMigrationValue(read(state.ui)) + if ( + route[key] !== undefined && + current !== serializeOrcadMigrationValue(route[key]) && + current !== serializeOrcadMigrationValue(empty) + ) { + throw new Error(`orcad_migration_client_state_conflict:ui:${key}`) + } + } +} + +function applyUiRouting(route: OrcadMigrationUiRoutingState, state: PersistedState): void { + const ui = state.ui + if (route.lastActiveRepoId !== undefined) { + ui.lastActiveRepoId = route.lastActiveRepoId + } + if (route.lastActiveWorktreeId !== undefined) { + ui.lastActiveWorktreeId = route.lastActiveWorktreeId + } + if (route.filterRepoIds !== undefined) { + ui.filterRepoIds = structuredClone(route.filterRepoIds) + } + if (route.showDotfilesByWorktree !== undefined) { + ui.showDotfilesByWorktree = { + ...ui.showDotfilesByWorktree, + ...structuredClone(route.showDotfilesByWorktree) + } + } + if (route.setupScriptPromptDismissedRepoIds !== undefined) { + ui.setupScriptPromptDismissedRepoIds = structuredClone(route.setupScriptPromptDismissedRepoIds) + } + if (route.manualRepoOrder !== undefined) { + ui.manualRepoOrder = structuredClone(route.manualRepoOrder) + } + if (route.workspaceHostScope !== undefined) { + ui.workspaceHostScope = route.workspaceHostScope + } + if (route.visibleWorkspaceHostIds !== undefined) { + ui.visibleWorkspaceHostIds = structuredClone(route.visibleWorkspaceHostIds) + } + if (route.workspaceHostOrder !== undefined) { + ui.workspaceHostOrder = structuredClone(route.workspaceHostOrder) + } + if (route.automationHostFilter !== undefined) { + ui.automationHostFilter = structuredClone(route.automationHostFilter) + } + if (route.acknowledgedAgentsByPaneKey !== undefined) { + ui.acknowledgedAgentsByPaneKey = { + ...ui.acknowledgedAgentsByPaneKey, + ...structuredClone(route.acknowledgedAgentsByPaneKey) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.test.ts new file mode 100644 index 00000000000..882eca77c8f --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../../../shared/constants' +import { + CLIENT_HOSTED_BROWSER_PAGE_RECORD_VERSION, + type PersistedClientHostedBrowserPage +} from '../../../shared/client-hosted-browser-page-record' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import type { TerminalLayoutSnapshot } from '../../../shared/terminal-tab-types' +import { prepareOrcadMigrationWorkspaceSession } from './orcad-destination-workspace-session' + +const OWNER = 'repo-1::/srv/worktree' +const LEAF = '11111111-1111-4111-8111-111111111111' +const OTHER_LEAF = '22222222-2222-4222-8222-222222222222' + +function layout(leafId = LEAF): TerminalLayoutSnapshot { + return { root: { type: 'leaf', leafId }, activeLeafId: leafId, expandedLeafId: null } +} + +function page(browserPageId: string): PersistedClientHostedBrowserPage { + return { + v: CLIENT_HOSTED_BROWSER_PAGE_RECORD_VERSION, + browserPageId, + workspaceId: 'browser-workspace-1', + browserProfileId: 'profile-1', + url: 'https://example.test/', + title: 'Example', + pairedDeviceId: 'device-1', + savedAt: 1 + } +} + +function state(): PersistedState { + return getDefaultPersistedState('/tmp/orca-test') +} + +function incoming(pages: ReturnType[]): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + clientHostedBrowserPagesByWorktree: { [OWNER]: pages } + } +} + +describe('destination workspace-session merge validation', () => { + it('accepts a unique client-hosted page identity', () => { + expect(() => + prepareOrcadMigrationWorkspaceSession(incoming([page('page-1')]), state()) + ).not.toThrow() + }) + + it('rejects duplicate client-hosted page identities before merge', () => { + expect(() => + prepareOrcadMigrationWorkspaceSession(incoming([page('page-1'), page('page-1')]), state()) + ).toThrow('orcad_migration_dormant_id_conflict:workspace_session:client-browser-page:page-1') + }) + + it.each(['existing', 'incoming'] as const)( + 'rejects a pane identity already claimed by another %s tab without mutation', + (location) => { + const current = state() + const next = incoming([]) + next.terminalLayoutsByTabId['incoming-tab'] = layout() + const competing = location === 'existing' ? current.workspaceSession : next + competing.terminalLayoutsByTabId['other-tab'] = layout() + const before = structuredClone({ current, next }) + expect(() => prepareOrcadMigrationWorkspaceSession(next, current)).toThrow( + `orcad_migration_dormant_id_conflict:workspace_session:terminal-leaf:${LEAF}` + ) + expect({ current, next }).toEqual(before) + } + ) + + it('rejects duplicate leaves within an incoming split', () => { + const next = incoming([]) + next.terminalLayoutsByTabId.tab = { + ...layout(), + root: { + type: 'split', + direction: 'horizontal', + first: { type: 'leaf', leafId: LEAF }, + second: { type: 'leaf', leafId: LEAF } + } + } + expect(() => prepareOrcadMigrationWorkspaceSession(next, state())).toThrow( + `orcad_migration_dormant_id_conflict:workspace_session:terminal-leaf:${LEAF}` + ) + }) + + it.each([ + 'ptyIdsByLeafId', + 'buffersByLeafId', + 'scrollbackRefsByLeafId', + 'titlesByLeafId' + ] as const)('preserves a competing pane claim in %s even outside its topology', (field) => { + const current = state() + current.workspaceSession.terminalLayoutsByTabId.other = { + ...layout(OTHER_LEAF), + [field]: { [LEAF]: 'preserved-value' } + } + const next = incoming([]) + next.terminalLayoutsByTabId.tab = layout() + const before = structuredClone(current) + expect(() => prepareOrcadMigrationWorkspaceSession(next, current)).toThrow( + `orcad_migration_dormant_id_conflict:workspace_session:terminal-leaf:${LEAF}` + ) + expect(current).toEqual(before) + }) + + it('preserves a unique split and accepts an exact retry with per-pane records', () => { + const next = incoming([]) + next.terminalLayoutsByTabId.tab = { + ...layout(), + root: { + type: 'split', + direction: 'vertical', + ratio: 0.3, + first: { type: 'leaf', leafId: LEAF }, + second: { type: 'leaf', leafId: OTHER_LEAF } + }, + buffersByLeafId: { [LEAF]: 'first output', [OTHER_LEAF]: 'second output' }, + titlesByLeafId: { [LEAF]: 'first', [OTHER_LEAF]: 'second' } + } + const current = state() + const prepared = prepareOrcadMigrationWorkspaceSession(next, current) + expect(prepared.merged?.terminalLayoutsByTabId).toEqual(next.terminalLayoutsByTabId) + current.workspaceSession = prepared.merged! + expect(prepareOrcadMigrationWorkspaceSession(next, current).merged).toEqual( + current.workspaceSession + ) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.ts new file mode 100644 index 00000000000..09a42df978b --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.ts @@ -0,0 +1,166 @@ +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' +import { isRecord } from '../../../shared/orcad-migration-manifest-fields' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { SESSION_FOCUS_FIELDS } from '../../../shared/workspace-session-host-field-ownership' +import { mergeWorkspaceSessions } from '../../orca-profiles/profile-project-session-state' +import { collectLayoutLeafIdsInOrder } from '../restoring-sessions/terminal-layout-normalization' + +const MERGED_SESSION_RECORD_FIELDS = [ + 'tabsByWorktree', + 'terminalLayoutsByTabId', + 'openFilesByWorktree', + 'markdownFrontmatterVisible', + 'browserTabsByWorktree', + 'browserPagesByWorkspace', + 'clientHostedBrowserPagesByWorktree', + 'activeBrowserTabIdByWorktree', + 'activeFileIdByWorktree', + 'activeTabTypeByWorktree', + 'activeTabIdByWorktree', + 'unifiedTabs', + 'tabGroups', + 'tabGroupLayouts', + 'activeGroupIdByWorktree', + 'lastVisitedAtByWorktreeId', + 'defaultTerminalTabsAppliedByWorktreeId', + 'terminalPtyIncarnationsByPaneKey', + 'terminalSurfaceTombstonesByPaneKey', + 'sleepingAgentSessionsByPaneKey' +] as const satisfies readonly (keyof WorkspaceSessionState)[] + +export type PreparedOrcadMigrationWorkspaceSession = { + incoming: WorkspaceSessionState | undefined + merged: WorkspaceSessionState | undefined +} + +export function prepareOrcadMigrationWorkspaceSession( + incoming: WorkspaceSessionState | undefined, + state: PersistedState +): PreparedOrcadMigrationWorkspaceSession { + if (!incoming) { + return { incoming: undefined, merged: undefined } + } + assertNoSessionRecordConflicts(state.workspaceSession, incoming) + assertNoSessionFocusConflicts(state.workspaceSession, incoming) + assertNoSessionEntityConflicts(state.workspaceSession, incoming) + return { + incoming, + merged: mergeWorkspaceSessions(state.workspaceSession, incoming) + } +} + +function assertNoSessionFocusConflicts( + existing: WorkspaceSessionState, + incoming: WorkspaceSessionState +): void { + for (const field of SESSION_FOCUS_FIELDS) { + const current = existing[field] + const next = incoming[field] + if ( + next !== undefined && + next !== null && + current !== undefined && + current !== null && + serializeOrcadMigrationValue(current) !== serializeOrcadMigrationValue(next) + ) { + throw new Error(`orcad_migration_dormant_focus_conflict:${field}`) + } + } +} + +export function applyPreparedOrcadMigrationWorkspaceSession( + prepared: PreparedOrcadMigrationWorkspaceSession, + state: PersistedState +): void { + if (prepared.merged) { + state.workspaceSession = prepared.merged + } +} + +function assertNoSessionRecordConflicts( + existing: WorkspaceSessionState, + incoming: WorkspaceSessionState +): void { + for (const field of MERGED_SESSION_RECORD_FIELDS) { + const existingValue: unknown = existing[field] + const incomingValue: unknown = incoming[field] + const existingRecord = isRecord(existingValue) ? existingValue : {} + const incomingRecord = isRecord(incomingValue) ? incomingValue : {} + for (const [key, value] of Object.entries(incomingRecord)) { + const current = existingRecord[key] + if ( + current !== undefined && + serializeOrcadMigrationValue(current) !== serializeOrcadMigrationValue(value) + ) { + throw new Error( + `orcad_migration_dormant_id_conflict:workspace_session:${String(field)}:${key}` + ) + } + } + } +} + +function assertNoSessionEntityConflicts( + existing: WorkspaceSessionState, + incoming: WorkspaceSessionState +): void { + const existingOwners = collectOrcadMigrationSessionEntityOwners(existing) + for (const [key, owner] of collectOrcadMigrationSessionEntityOwners(incoming)) { + if (owner === '\0') { + throw new Error(`orcad_migration_dormant_id_conflict:workspace_session:${key}`) + } + const currentOwner = existingOwners.get(key) + if (currentOwner !== undefined && currentOwner !== owner) { + throw new Error(`orcad_migration_dormant_id_conflict:workspace_session:${key}`) + } + } +} + +export function collectOrcadMigrationSessionEntityOwners( + session: WorkspaceSessionState +): Map { + const owners = new Map() + const add = (key: string, owner: string): void => { + const current = owners.get(key) + // A duplicate entity id is invalid even when both rows claim the same owner. Marking every + // repeat lets the caller reject duplicates within the incoming payload as well as conflicts + // with already-committed state. + owners.set(key, current === undefined ? owner : '\0') + } + for (const [owner, tabs] of Object.entries(session.tabsByWorktree ?? {})) { + tabs.forEach((tab) => add(`terminal:${tab.id}`, owner)) + } + for (const [tabId, layout] of Object.entries(session.terminalLayoutsByTabId ?? {})) { + const topologyLeaves = collectLayoutLeafIdsInOrder(layout.root) + topologyLeaves.forEach((leafId) => add(`terminal-leaf:${leafId}`, tabId)) + const topologyIds = new Set(topologyLeaves) + const recordIds = new Set( + [ + layout.ptyIdsByLeafId, + layout.buffersByLeafId, + layout.scrollbackRefsByLeafId, + layout.titlesByLeafId + ].flatMap((record) => Object.keys(record ?? {})) + ) + // Stale per-pane records still own data; importing their identity must not overwrite it. + for (const leafId of recordIds) { + if (!topologyIds.has(leafId)) { + add(`terminal-leaf:${leafId}`, tabId) + } + } + } + for (const [owner, workspaces] of Object.entries(session.browserTabsByWorktree ?? {})) { + workspaces.forEach((workspace) => add(`browser:${workspace.id}`, owner)) + } + for (const [owner, pages] of Object.entries(session.clientHostedBrowserPagesByWorktree ?? {})) { + pages.forEach((page) => add(`client-browser-page:${page.browserPageId}`, owner)) + } + for (const [owner, tabs] of Object.entries(session.unifiedTabs ?? {})) { + tabs.forEach((tab) => add(`tab:${tab.id}`, owner)) + } + for (const [owner, groups] of Object.entries(session.tabGroups ?? {})) { + groups.forEach((group) => add(`group:${group.id}`, owner)) + } + return owners +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-destination-worktree-metadata.ts b/src/main/persistence/migrating-orcad-catalog/orcad-destination-worktree-metadata.ts new file mode 100644 index 00000000000..7315229a873 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-destination-worktree-metadata.ts @@ -0,0 +1,41 @@ +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { OrcadMigrationDormantStatePayload } from '../../../shared/orcad-migration-manifest' +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' +import { canonicalWorktreeIdentity } from '../../../shared/worktree/identity' +import { composeWorktreeHostIdentity } from '../../../shared/worktree/host-qualified-identity' +import { omitDefaultWorktreeMetaFields } from '../../../shared/worktree/meta-persisted-defaults' + +export function assertOrcadDestinationCanonicalMetadata( + state: PersistedState, + entries: OrcadMigrationDormantStatePayload['worktreeMeta'] +): void { + for (const { worktreeId, meta } of entries) { + const alias = composeWorktreeHostIdentity('local', worktreeId) + const identities = state.worktreeIdentityAliases?.[alias] ?? [] + const expectedKey = meta.instanceId + ? canonicalWorktreeIdentity({ + worktreeId, + executionHostId: 'local', + instanceId: meta.instanceId + }) + : undefined + const candidates = new Set([ + ...identities, + ...(expectedKey && state.worktreeMetaByIdentity?.[expectedKey] ? [expectedKey] : []) + ]) + if ( + candidates.size > 1 || + [...candidates].some((key) => { + const current = state.worktreeMetaByIdentity?.[key] + return ( + !current || + key !== expectedKey || + serializeOrcadMigrationValue(omitDefaultWorktreeMetaFields(current)) !== + serializeOrcadMigrationValue(omitDefaultWorktreeMetaFields(meta)) + ) + }) + ) { + throw new Error(`orcad_migration_dormant_id_conflict:worktree_meta:${worktreeId}`) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-dormant-metadata-defaults.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-dormant-metadata-defaults.test.ts new file mode 100644 index 00000000000..bdddd017d28 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-dormant-metadata-defaults.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { fillDefaultWorktreeMetaFields } from '../../../shared/worktree/meta-persisted-defaults' +import { prepareOrcadMigrationDormantState } from './orcad-dormant-state-records' + +const KEY = 'repo::/srv/worktree' + +function fixture() { + const state = getDefaultPersistedState('/tmp/orcad-metadata-defaults') + const meta = { + displayName: 'Workspace', + comment: 'preserve', + isUnread: true, + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + isArchived: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 1 + } + const manifest: OrcadMigrationManifest = { + version: 1, + migrationId: 'migration', + createdAt: '2026-09-06T00:00:00.000Z', + source: { sshTargetId: 'ssh', sshTargetGeneration: 1, targetLabel: 'host' }, + manifestSha256: 'unused-by-dormant-validator', + payload: { + repositories: [], + projectGroups: [], + folderWorkspaces: [], + dormantState: { + version: 1, + worktreeMeta: [{ sourceKey: KEY, worktreeId: KEY, meta }], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [] + } + } + } + state.worktreeMeta[KEY] = { ...meta } + fillDefaultWorktreeMetaFields(state.worktreeMeta[KEY]) + return { state, manifest } +} + +describe('migration metadata default equivalence', () => { + it('accepts load-materialized defaults without changing the manifest or state', () => { + const { state, manifest } = fixture() + const before = structuredClone({ state, manifest }) + expect(prepareOrcadMigrationDormantState(manifest, state).newWorktreeMeta).toEqual([]) + expect({ state, manifest }).toEqual(before) + }) + + it.each(['isPinned', 'isArchived'] as const)('still refuses a changed %s value', (field) => { + const { state, manifest } = fixture() + state.worktreeMeta[KEY][field] = true + expect(() => prepareOrcadMigrationDormantState(manifest, state)).toThrow( + `orcad_migration_dormant_id_conflict:worktree_meta:${KEY}` + ) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-dormant-state-records.ts b/src/main/persistence/migrating-orcad-catalog/orcad-dormant-state-records.ts new file mode 100644 index 00000000000..4a0af58276a --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-dormant-state-records.ts @@ -0,0 +1,210 @@ +import type { + OrcadMigrationDormantStatePayload, + OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { SparsePreset } from '../../../shared/worktree/create-types' +import type { WorkspaceLineage, WorktreeLineage } from '../../../shared/worktree/lineage-types' +import type { WorktreeMeta } from '../../../shared/worktree/meta-types' +import { omitDefaultWorktreeMetaFields } from '../../../shared/worktree/meta-persisted-defaults' +import { assertOrcadDestinationCanonicalMetadata } from './orcad-destination-worktree-metadata' +import { assertSameValue } from './orcad-catalog-row-identity' +import { + mergeRetiredNameRegistries, + type RetiredNameRegistry +} from '../../../shared/worktree/retired-name-registry' +import { recordRetirementNamespaceRegistry } from '../../worktree-retirement-namespace' +import { + applyPreparedOrcadMigrationWorkspaceSession, + prepareOrcadMigrationWorkspaceSession, + type PreparedOrcadMigrationWorkspaceSession +} from './orcad-destination-workspace-session' +import { + applyPreparedOrcadMigrationAutomationState, + prepareOrcadMigrationAutomationState, + type PreparedOrcadMigrationAutomationState +} from './orcad-destination-automation-state' +import { + applyPreparedOrcadMigrationClientState, + prepareOrcadMigrationClientState, + type PreparedOrcadMigrationClientState +} from './orcad-destination-client-state' + +type KeyedRow = { key: string; value: T } +type RegistryUpdate = { key: string; value: RetiredNameRegistry } + +export type PreparedOrcadMigrationDormantState = { + payload: OrcadMigrationDormantStatePayload | undefined + newWorktreeMeta: KeyedRow[] + newWorktreeLineage: KeyedRow[] + newWorkspaceLineage: KeyedRow[] + newSparsePresets: SparsePreset[] + retiredNameUpdates: RegistryUpdate[] + retirementNamespaceUpdates: RegistryUpdate[] + workspaceSession: PreparedOrcadMigrationWorkspaceSession + automationState: PreparedOrcadMigrationAutomationState + clientState: PreparedOrcadMigrationClientState +} + +export function prepareOrcadMigrationDormantState( + manifest: OrcadMigrationManifest, + state: PersistedState +): PreparedOrcadMigrationDormantState { + const payload = manifest.payload.dormantState + if (!payload) { + return emptyPreparedDormantState() + } + assertOrcadDestinationCanonicalMetadata(state, payload.worktreeMeta) + const worktreeMeta = payload.worktreeMeta.map((entry) => ({ + key: entry.worktreeId, + value: structuredClone(entry.meta) + })) + const worktreeLineage = payload.worktreeLineage.map((entry) => ({ + key: entry.worktreeId, + value: structuredClone(entry.lineage) + })) + const workspaceLineage = payload.workspaceLineage.map((entry) => ({ + key: entry.childWorkspaceKey, + value: structuredClone(entry.lineage) + })) + const existingPresets = new Map( + Object.values(state.sparsePresetsByRepo) + .flat() + .map((preset) => [sparsePresetKey(preset), preset]) + ) + const newSparsePresets = payload.sparsePresets.filter((preset) => { + const existing = existingPresets.get(sparsePresetKey(preset)) + if (!existing) { + return true + } + assertSameValue(existing, preset, `sparse_preset:${preset.repoId}:${preset.id}`) + return false + }) + return { + payload, + newWorktreeMeta: selectNewKeyedRows( + worktreeMeta, + state.worktreeMeta, + 'worktree_meta', + omitDefaultWorktreeMetaFields + ), + newWorktreeLineage: selectNewKeyedRows( + worktreeLineage, + state.worktreeLineageById, + 'worktree_lineage' + ), + newWorkspaceLineage: selectNewKeyedRows( + workspaceLineage, + state.workspaceLineageByChildKey, + 'workspace_lineage' + ), + newSparsePresets, + retiredNameUpdates: payload.retiredWorktreeNames.map((entry) => ({ + key: entry.repoId, + value: mergeRetiredNameRegistries( + state.retiredWorktreeNamesByRepo?.[entry.repoId] ?? { exhaustedTiers: 0, names: [] }, + entry.registry + ) + })), + retirementNamespaceUpdates: payload.retiredWorktreeNamespaces.map((entry) => ({ + key: entry.namespaceKey, + value: mergeRetiredNameRegistries( + state.retiredWorktreeNamesByNamespace?.[entry.namespaceKey] ?? { + exhaustedTiers: 0, + names: [] + }, + entry.registry + ) + })), + workspaceSession: prepareOrcadMigrationWorkspaceSession(payload.workspaceSession, state), + automationState: prepareOrcadMigrationAutomationState( + payload.automations, + payload.automationRuns, + state + ), + clientState: prepareOrcadMigrationClientState(payload.clientState, state) + } +} + +export function applyPreparedOrcadMigrationDormantState( + prepared: PreparedOrcadMigrationDormantState, + state: PersistedState +): void { + for (const entry of prepared.newWorktreeMeta) { + state.worktreeMeta[entry.key] = entry.value + } + for (const entry of prepared.newWorktreeLineage) { + state.worktreeLineageById[entry.key] = entry.value + } + for (const entry of prepared.newWorkspaceLineage) { + state.workspaceLineageByChildKey[entry.key] = entry.value + } + for (const preset of prepared.newSparsePresets) { + state.sparsePresetsByRepo[preset.repoId] = [ + ...(state.sparsePresetsByRepo[preset.repoId] ?? []), + preset + ] + } + if (prepared.retiredNameUpdates.length > 0) { + state.retiredWorktreeNamesByRepo ??= {} + for (const entry of prepared.retiredNameUpdates) { + state.retiredWorktreeNamesByRepo[entry.key] = entry.value + } + } + if (prepared.retirementNamespaceUpdates.length > 0) { + state.retiredWorktreeNamesByNamespace ??= {} + for (const entry of prepared.retirementNamespaceUpdates) { + recordRetirementNamespaceRegistry( + state.retiredWorktreeNamesByNamespace, + entry.key, + entry.value + ) + } + } + applyPreparedOrcadMigrationWorkspaceSession(prepared.workspaceSession, state) + applyPreparedOrcadMigrationAutomationState(prepared.automationState, state) + applyPreparedOrcadMigrationClientState(prepared.clientState, state) +} + +function selectNewKeyedRows( + incoming: KeyedRow[], + existing: Record, + label: string, + canonicalize: (value: T) => T = (value) => value +): KeyedRow[] { + return incoming.filter((entry) => { + const current = existing[entry.key] + if (current === undefined) { + return true + } + assertSameValue(canonicalize(current), canonicalize(entry.value), `${label}:${entry.key}`) + return false + }) +} + +const sparsePresetKey = (preset: Pick): string => + `${preset.repoId}\0${preset.id}` + +function emptyPreparedDormantState(): PreparedOrcadMigrationDormantState { + return { + payload: undefined, + newWorktreeMeta: [], + newWorktreeLineage: [], + newWorkspaceLineage: [], + newSparsePresets: [], + retiredNameUpdates: [], + retirementNamespaceUpdates: [], + workspaceSession: { incoming: undefined, merged: undefined }, + automationState: { + incomingAutomations: [], + incomingRuns: [], + newAutomations: [], + newRuns: [] + }, + clientState: { + incoming: undefined, + mobileSelections: {}, + uiRouting: undefined + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.test.ts new file mode 100644 index 00000000000..8e3efdeefe6 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.test.ts @@ -0,0 +1,231 @@ +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../../shared/constants' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { + OrcadMigrationSnapshotChunkRequest, + OrcadMigrationTerminalScrollbackSnapshot +} from '../../../shared/orcad-migration-scrollback' +import { + getTerminalScrollbackSnapshotPath, + type TerminalScrollbackSnapshotStorage +} from '../../terminal-scrollback-snapshots' +import { + abortOrcadMigrationSnapshots, + assertOrcadMigrationSnapshotsReady, + commitOrcadMigrationSnapshots, + inspectOrcadMigrationSnapshotUploads, + pruneOrcadMigrationSnapshotStaging, + stageOrcadMigrationSnapshotChunk +} from './orcad-scrollback-snapshot-transfer' + +let root: string +let storage: TerminalScrollbackSnapshotStorage + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orcad-migration-snapshots-')) + storage = { snapshotRoot: join(root, 'terminal-scrollback') } +}) + +afterEach(() => rmSync(root, { recursive: true, force: true })) + +describe('orcad scrollback snapshot transfer', () => { + it('resumes from staged byte length and accepts only exact idempotent retries', () => { + const bytes = Buffer.from('first line\nmultibyte: 🐋\nlast line', 'utf8') + const descriptor = snapshot('1', 'tab-1', 'leaf-1', bytes) + const manifest = migrationManifest('resume', [descriptor]) + const state = stagedState(manifest) + const first = bytes.subarray(0, 11) + + expect(stage(state, manifest, descriptor, 0, first).acknowledgedOffset).toBe(first.length) + expect(inspectOrcadMigrationSnapshotUploads(manifest, storage)?.[0]?.receivedBytes).toBe( + first.length + ) + expect(stage(state, manifest, descriptor, 0, first).acknowledgedOffset).toBe(first.length) + expect(() => stage(state, manifest, descriptor, 0, Buffer.from('different!!'))).toThrow( + 'orcad_migration_snapshot_retry_mismatch' + ) + expect(() => stage(state, manifest, descriptor, first.length + 1, Buffer.from('x'))).toThrow( + 'orcad_migration_snapshot_offset_invalid' + ) + + stage(state, manifest, descriptor, first.length, bytes.subarray(first.length)) + assertOrcadMigrationSnapshotsReady(manifest, storage) + commitOrcadMigrationSnapshots(manifest, storage) + expect(readFinal(descriptor)).toEqual(bytes) + }) + + it('refuses incomplete and digest-mismatched staged bytes', () => { + const bytes = Buffer.from('expected bytes', 'utf8') + const descriptor = snapshot('2', 'tab-2', 'leaf-2', bytes) + const manifest = migrationManifest('refuse', [descriptor]) + const state = stagedState(manifest) + + stage(state, manifest, descriptor, 0, bytes.subarray(0, 4)) + expect(() => assertOrcadMigrationSnapshotsReady(manifest, storage)).toThrow( + 'orcad_migration_snapshot_incomplete' + ) + stage(state, manifest, descriptor, 4, Buffer.alloc(bytes.length - 4, 0x78)) + expect(() => assertOrcadMigrationSnapshotsReady(manifest, storage)).toThrow( + 'orcad_migration_snapshot_digest_mismatch' + ) + expect(readFinal(descriptor)).toBeNull() + }) + + it('recognizes an already-materialized matching final file after restart', () => { + const bytes = Buffer.from('already committed bytes', 'utf8') + const descriptor = snapshot('3', 'tab-3', 'leaf-3', bytes) + const manifest = migrationManifest('materialized', [descriptor]) + stagedState(manifest) + writeFinal(descriptor, bytes) + + expect(inspectOrcadMigrationSnapshotUploads(manifest, storage)?.[0]?.receivedBytes).toBe( + bytes.length + ) + assertOrcadMigrationSnapshotsReady(manifest, storage) + commitOrcadMigrationSnapshots(manifest, storage) + expect(readFinal(descriptor)).toEqual(bytes) + }) + + it('rejects a same-ref content conflict before materializing any snapshot', () => { + const firstBytes = Buffer.from('first snapshot', 'utf8') + const secondBytes = Buffer.from('second snapshot', 'utf8') + const first = snapshot('4', 'tab-4', 'leaf-4', firstBytes) + const second = snapshot('5', 'tab-5', 'leaf-5', secondBytes) + const manifest = migrationManifest('conflict', [first, second]) + const state = stagedState(manifest) + stage(state, manifest, first, 0, firstBytes) + stage(state, manifest, second, 0, secondBytes) + writeFinal(second, Buffer.alloc(secondBytes.length, 0x78)) + + expect(() => commitOrcadMigrationSnapshots(manifest, storage)).toThrow( + `orcad_migration_snapshot_destination_conflict:${second.ref}` + ) + expect(readFinal(first)).toBeNull() + expect(readFinal(second)).toEqual(Buffer.alloc(secondBytes.length, 0x78)) + }) + + it('removes only the selected staging tree and prunes unjournaled trees', () => { + const bytes = Buffer.from('staged bytes', 'utf8') + const firstDescriptor = snapshot('6', 'tab-6', 'leaf-6', bytes) + const secondDescriptor = snapshot('7', 'tab-7', 'leaf-7', bytes) + const first = migrationManifest('abort-first', [firstDescriptor]) + const second = migrationManifest('retain-second', [secondDescriptor]) + const state = stagedState(first, second) + stage(state, first, firstDescriptor, 0, bytes) + stage(state, second, secondDescriptor, 0, bytes) + + abortOrcadMigrationSnapshots(first, storage) + expect(inspectOrcadMigrationSnapshotUploads(first, storage)?.[0]?.receivedBytes).toBe(0) + expect(inspectOrcadMigrationSnapshotUploads(second, storage)?.[0]?.receivedBytes).toBe( + bytes.length + ) + pruneOrcadMigrationSnapshotStaging( + state.filter((entry) => entry.migrationId !== second.migrationId), + storage + ) + expect(inspectOrcadMigrationSnapshotUploads(second, storage)?.[0]?.receivedBytes).toBe(0) + }) +}) + +function snapshot( + suffix: string, + tabId: string, + leafId: string, + bytes: Buffer +): OrcadMigrationTerminalScrollbackSnapshot { + return { + tabId, + leafId, + ref: `v1-${suffix.repeat(32)}`, + sha256: createHash('sha256').update(bytes).digest('hex'), + byteLength: bytes.length + } +} + +function migrationManifest( + migrationId: string, + snapshots: OrcadMigrationTerminalScrollbackSnapshot[] +): OrcadMigrationManifest { + const workspaceSession = getDefaultWorkspaceSession() + workspaceSession.terminalLayoutsByTabId = Object.fromEntries( + snapshots.map((entry) => [ + entry.tabId, + { + root: { type: 'leaf' as const, leafId: entry.leafId }, + activeLeafId: entry.leafId, + expandedLeafId: null, + scrollbackRefsByLeafId: { [entry.leafId]: entry.ref } + } + ]) + ) + return { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId, + createdAt: '2026-08-30T12:00:00.000Z', + source: { sshTargetId: 'source', sshTargetGeneration: 1, targetLabel: 'Source' }, + payload: { + repositories: [], + projectGroups: [], + folderWorkspaces: [], + dormantState: { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [], + workspaceSession, + terminalScrollbackSnapshots: snapshots + } + }, + manifestSha256: 'a'.repeat(64) + } +} + +/** The manifests the destination importer holds staged. */ +function stagedState(...manifests: OrcadMigrationManifest[]): OrcadMigrationManifest[] { + return manifests +} + +function stage( + state: OrcadMigrationManifest[], + manifest: OrcadMigrationManifest, + descriptor: OrcadMigrationTerminalScrollbackSnapshot, + offset: number, + bytes: Buffer +) { + const request: OrcadMigrationSnapshotChunkRequest = { + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256, + ref: descriptor.ref, + offset, + bytesBase64: bytes.toString('base64') + } + const stagedManifest = state.find((entry) => entry.migrationId === manifest.migrationId) ?? null + return stageOrcadMigrationSnapshotChunk({ stagedManifest, storage, request }) +} + +function writeFinal(descriptor: OrcadMigrationTerminalScrollbackSnapshot, bytes: Buffer): void { + const path = getTerminalScrollbackSnapshotPath(descriptor.ref, storage) + if (!path) { + throw new Error('expected snapshot path') + } + if (!storage.snapshotRoot) { + throw new Error('expected snapshot root') + } + mkdirSync(storage.snapshotRoot, { recursive: true }) + writeFileSync(path, bytes) +} + +function readFinal(descriptor: OrcadMigrationTerminalScrollbackSnapshot): Buffer | null { + const path = getTerminalScrollbackSnapshotPath(descriptor.ref, storage) + return path && existsSync(path) ? readFileSync(path) : null +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.ts b/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.ts new file mode 100644 index 00000000000..80a7e566b71 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-scrollback-snapshot-transfer.ts @@ -0,0 +1,302 @@ +import { createHash } from 'node:crypto' +import { + closeSync, + existsSync, + fsyncSync, + linkSync, + mkdirSync, + openSync, + readFileSync, + readSync, + readdirSync, + rmSync, + statSync, + writeSync +} from 'node:fs' +import { join } from 'node:path' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { + decodeOrcadMigrationSnapshotChunk, + type OrcadMigrationSnapshotChunkRequest, + type OrcadMigrationSnapshotChunkResult, + type OrcadMigrationSnapshotUploadState, + type OrcadMigrationTerminalScrollbackSnapshot +} from '../../../shared/orcad-migration-scrollback' +import { syncDirectoryDurablySync } from '../../durable-file-write' +import { + getTerminalScrollbackSnapshotPath, + getTerminalScrollbackSnapshotRoot, + type TerminalScrollbackSnapshotStorage +} from '../../terminal-scrollback-snapshots' + +const STAGING_DIRECTORY = '.orcad-migration-staging' + +/** + * Destination side: append one verified chunk to the staging file for a snapshot the staged + * manifest names. `stagedManifest` is the manifest the importer accepted; a request for any + * other migration or digest is refused. Retries of an already-written range must match it. + */ +export function stageOrcadMigrationSnapshotChunk(args: { + stagedManifest: OrcadMigrationManifest | null + storage: TerminalScrollbackSnapshotStorage + request: OrcadMigrationSnapshotChunkRequest +}): OrcadMigrationSnapshotChunkResult { + const manifest = requireStagedManifest(args.stagedManifest, args.request) + const descriptor = requireDescriptor(manifest, args.request.ref) + const bytes = decodeOrcadMigrationSnapshotChunk(args.request.bytesBase64) + const path = stagedSnapshotPath(args.storage, manifest, descriptor) + mkdirSync(stagingDirectory(args.storage, manifest), { recursive: true, mode: 0o700 }) + const descriptorFd = openStagedFile(path) + try { + const size = statSync(path).size + if (args.request.offset > size || args.request.offset + bytes.length > descriptor.byteLength) { + throw new Error('orcad_migration_snapshot_offset_invalid') + } + if (args.request.offset < size) { + if (args.request.offset + bytes.length > size) { + throw new Error('orcad_migration_snapshot_offset_invalid') + } + const existing = Buffer.alloc(bytes.length) + const read = readSync(descriptorFd, existing, 0, existing.length, args.request.offset) + if (read !== bytes.length || !existing.equals(bytes)) { + throw new Error('orcad_migration_snapshot_retry_mismatch') + } + return chunkResult(args.request, size) + } + const written = writeSync(descriptorFd, bytes, 0, bytes.length, args.request.offset) + if (written !== bytes.length) { + throw new Error('orcad_migration_snapshot_write_incomplete') + } + fsyncSync(descriptorFd) + return chunkResult(args.request, size + bytes.length) + } finally { + closeSync(descriptorFd) + } +} + +export function inspectOrcadMigrationSnapshotUploads( + manifest: OrcadMigrationManifest, + storage: TerminalScrollbackSnapshotStorage +): OrcadMigrationSnapshotUploadState[] | undefined { + const snapshots = manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + if (snapshots.length === 0) { + return undefined + } + return snapshots.map((descriptor) => ({ + ...descriptor, + receivedBytes: matchingFinalSnapshot(storage, descriptor) + ? descriptor.byteLength + : stagedSnapshotSize(storage, manifest, descriptor) + })) +} + +export function assertOrcadMigrationSnapshotsReady( + manifest: OrcadMigrationManifest, + storage: TerminalScrollbackSnapshotStorage +): void { + for (const upload of inspectOrcadMigrationSnapshotUploads(manifest, storage) ?? []) { + const finalStatus = inspectFinalSnapshot(storage, upload) + if (finalStatus === 'conflict') { + throw new Error(`orcad_migration_snapshot_destination_conflict:${upload.ref}`) + } + if (upload.receivedBytes !== upload.byteLength) { + throw new Error(`orcad_migration_snapshot_incomplete:${upload.ref}`) + } + const path = + finalStatus === 'matching' + ? getTerminalScrollbackSnapshotPath(upload.ref, storage) + : stagedSnapshotPath(storage, manifest, upload) + if (!path || !fileMatches(path, upload)) { + throw new Error(`orcad_migration_snapshot_digest_mismatch:${upload.ref}`) + } + } +} + +export function commitOrcadMigrationSnapshots( + manifest: OrcadMigrationManifest, + storage: TerminalScrollbackSnapshotStorage +): void { + const snapshots = manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + assertOrcadMigrationSnapshotsReady(manifest, storage) + const root = getTerminalScrollbackSnapshotRoot(storage) + mkdirSync(root, { recursive: true, mode: 0o700 }) + for (const descriptor of snapshots) { + if (matchingFinalSnapshot(storage, descriptor)) { + rmSync(stagedSnapshotPath(storage, manifest, descriptor), { force: true }) + continue + } + const stagedPath = stagedSnapshotPath(storage, manifest, descriptor) + const finalPath = getTerminalScrollbackSnapshotPath(descriptor.ref, storage) + if (!finalPath) { + throw new Error('orcad_migration_snapshot_ref_invalid') + } + try { + linkSync(stagedPath, finalPath) + } catch (error) { + const finalStatus = inspectFinalSnapshot(storage, descriptor) + if (finalStatus === 'conflict') { + throw new Error(`orcad_migration_snapshot_destination_conflict:${descriptor.ref}`) + } + if (finalStatus !== 'matching') { + throw error + } + } + rmSync(stagedPath, { force: true }) + } + syncDirectoryDurablySync(root) + removeStagingDirectory(storage, manifest) +} + +export function abortOrcadMigrationSnapshots( + manifest: OrcadMigrationManifest, + storage: TerminalScrollbackSnapshotStorage +): void { + removeStagingDirectory(storage, manifest) +} + +/** Removes staging for every migration the importer no longer holds staged. */ +export function pruneOrcadMigrationSnapshotStaging( + stagedManifests: readonly OrcadMigrationManifest[], + storage: TerminalScrollbackSnapshotStorage +): void { + const root = join(getTerminalScrollbackSnapshotRoot(storage), STAGING_DIRECTORY) + const retained = new Set(stagedManifests.map(stagingKey)) + let entries: string[] + try { + entries = readdirSync(root) + } catch { + return + } + for (const entry of entries) { + if (/^[a-f0-9]{32}$/.test(entry) && !retained.has(entry)) { + rmSync(join(root, entry), { recursive: true, force: true }) + } + } +} + +function requireStagedManifest( + staged: OrcadMigrationManifest | null, + request: Pick +): OrcadMigrationManifest { + if ( + !staged || + staged.migrationId !== request.migrationId || + staged.manifestSha256 !== request.manifestSha256 + ) { + throw new Error('orcad_migration_snapshot_catalog_not_staged') + } + return staged +} + +function requireDescriptor( + manifest: OrcadMigrationManifest, + ref: string +): OrcadMigrationTerminalScrollbackSnapshot { + const descriptor = manifest.payload.dormantState?.terminalScrollbackSnapshots?.find( + (entry) => entry.ref === ref + ) + if (!descriptor) { + throw new Error('orcad_migration_snapshot_unknown') + } + return descriptor +} + +function stagedSnapshotSize( + storage: TerminalScrollbackSnapshotStorage, + manifest: OrcadMigrationManifest, + descriptor: OrcadMigrationTerminalScrollbackSnapshot +): number { + try { + return Math.min( + statSync(stagedSnapshotPath(storage, manifest, descriptor)).size, + descriptor.byteLength + ) + } catch { + return 0 + } +} + +function matchingFinalSnapshot( + storage: TerminalScrollbackSnapshotStorage, + descriptor: OrcadMigrationTerminalScrollbackSnapshot +): boolean { + return inspectFinalSnapshot(storage, descriptor) === 'matching' +} + +function inspectFinalSnapshot( + storage: TerminalScrollbackSnapshotStorage, + descriptor: OrcadMigrationTerminalScrollbackSnapshot +): 'absent' | 'matching' | 'conflict' { + const path = getTerminalScrollbackSnapshotPath(descriptor.ref, storage) + if (!path || !existsSync(path)) { + return 'absent' + } + return fileMatches(path, descriptor) ? 'matching' : 'conflict' +} + +function fileMatches(path: string, descriptor: OrcadMigrationTerminalScrollbackSnapshot): boolean { + try { + const bytes = readFileSync(path) + return ( + bytes.length === descriptor.byteLength && + createHash('sha256').update(bytes).digest('hex') === descriptor.sha256 + ) + } catch { + return false + } +} + +function openStagedFile(path: string): number { + try { + return openSync(path, 'r+') + } catch { + try { + return openSync(path, 'wx+', 0o600) + } catch { + return openSync(path, 'r+') + } + } +} + +function stagingDirectory( + storage: TerminalScrollbackSnapshotStorage, + manifest: OrcadMigrationManifest +): string { + return join(getTerminalScrollbackSnapshotRoot(storage), STAGING_DIRECTORY, stagingKey(manifest)) +} + +function stagedSnapshotPath( + storage: TerminalScrollbackSnapshotStorage, + manifest: OrcadMigrationManifest, + descriptor: OrcadMigrationTerminalScrollbackSnapshot +): string { + return join(stagingDirectory(storage, manifest), `${descriptor.ref}.${descriptor.sha256}.part`) +} + +function removeStagingDirectory( + storage: TerminalScrollbackSnapshotStorage, + manifest: OrcadMigrationManifest +): void { + const stagingRoot = join(getTerminalScrollbackSnapshotRoot(storage), STAGING_DIRECTORY) + rmSync(stagingDirectory(storage, manifest), { recursive: true, force: true }) + syncDirectoryDurablySync(stagingRoot) +} + +function stagingKey(manifest: OrcadMigrationManifest): string { + return createHash('sha256') + .update(`${manifest.migrationId}\0${manifest.manifestSha256}`) + .digest('hex') + .slice(0, 32) +} + +function chunkResult( + request: OrcadMigrationSnapshotChunkRequest, + acknowledgedOffset: number +): OrcadMigrationSnapshotChunkResult { + return { + migrationId: request.migrationId, + manifestSha256: request.manifestSha256, + ref: request.ref, + acknowledgedOffset + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-state.ts new file mode 100644 index 00000000000..6ff14758448 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-state.ts @@ -0,0 +1,190 @@ +import { + isFinalAutomationRunStatus, + type Automation, + type AutomationRun +} from '../../../shared/automations-types' +import { getAutomationRunRepoId } from '../../../shared/automation-run-identity' +import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { TaskSourceContext, WorkspaceRunContext } from '../../../shared/task-source-context' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + orcadMigrationOwnsRepoId, + type OrcadMigrationSourceScope, + unqualifyOrcadMigrationOwnerKey +} from './orcad-source-scope' +import { compareKeys } from './orcad-source-key-order' + +export type OrcadMigrationSourceAutomationInspection = { + automations: Automation[] + automationRuns: AutomationRun[] + blockedAutomationCount: number + blockedRunCount: number +} + +export function collectOrcadMigrationSourceAutomationState( + state: PersistedState, + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload +): OrcadMigrationSourceAutomationInspection { + const scope = createOrcadMigrationSourceScope({ source, catalog, repos: state.repos }) + const touchedAutomations = state.automations.filter((entry) => + automationTouchesScope(entry, scope) + ) + const touchedAutomationIds = new Set(touchedAutomations.map((entry) => entry.id)) + const touchedRuns = state.automationRuns.filter( + (entry) => touchedAutomationIds.has(entry.automationId) || runTouchesScope(entry, scope) + ) + const runsByAutomationId = Map.groupBy(touchedRuns, (entry) => entry.automationId) + const automationsById = Map.groupBy(touchedAutomations, (entry) => entry.id) + const automations: Automation[] = [] + const automationRuns: AutomationRun[] = [] + let blockedAutomationCount = 0 + let blockedRunCount = 0 + + for (const [automationId, matching] of automationsById) { + const runs = runsByAutomationId.get(automationId) ?? [] + const eligible = + matching.length === 1 && + automationCanTransfer(matching[0], scope) && + runs.every((run) => runCanTransfer(run, scope)) + if (!eligible) { + blockedAutomationCount += matching.length + blockedRunCount += runs.length + continue + } + automations.push(projectAutomationToDestination(matching[0])) + automationRuns.push(...runs.map(projectAutomationRunToDestination)) + } + + for (const [automationId, runs] of runsByAutomationId) { + if (!automationsById.has(automationId)) { + blockedRunCount += runs.length + } + } + automations.sort((left, right) => compareKeys(left.id, right.id)) + automationRuns.sort((left, right) => compareKeys(left.id, right.id)) + return { automations, automationRuns, blockedAutomationCount, blockedRunCount } +} + +export function automationTouchesScope( + automation: Automation, + scope: OrcadMigrationSourceScope +): boolean { + return ( + (automation.executionTargetType === 'ssh' && automation.executionTargetId === scope.targetId) || + // Why the target too: a generation is per target, so another host's can share the number. + (scope.targetGeneration !== null && + automation.executionTargetId === scope.targetId && + automation.executionTargetGeneration === scope.targetGeneration) || + orcadMigrationOwnsRepoId(scope, getAutomationRunRepoId(automation)) || + orcadMigrationOwnerMatchesScope(automation.workspaceId, scope) || + contextTouchesScope(automation.runContext, scope) || + contextTouchesScope(automation.sourceContext, scope) + ) +} + +function runTouchesScope(run: AutomationRun, scope: OrcadMigrationSourceScope): boolean { + return ( + orcadMigrationOwnerMatchesScope(run.workspaceId, scope) || + contextTouchesScope(run.runContext, scope) || + contextTouchesScope(run.sourceContext, scope) + ) +} + +function automationCanTransfer(automation: Automation, scope: OrcadMigrationSourceScope): boolean { + return ( + automation.enabled === false && + automation.executionTargetType === 'ssh' && + automation.executionTargetId === scope.targetId && + automation.schedulerOwner === 'ssh_bridge' && + (automation.executionTargetGeneration === undefined || + automation.executionTargetGeneration === scope.targetGeneration) && + scope.repoIds.has(getAutomationRunRepoId(automation)) && + ownerCanTransfer(automation.workspaceId, scope) && + contextCanTransfer(automation.runContext, scope) && + contextCanTransfer(automation.sourceContext, scope) + ) +} + +function runCanTransfer(run: AutomationRun, scope: OrcadMigrationSourceScope): boolean { + return ( + isFinalAutomationRunStatus(run.status) && + ownerCanTransfer(run.workspaceId, scope) && + contextCanTransfer(run.runContext, scope) && + contextCanTransfer(run.sourceContext, scope) + ) +} + +function ownerCanTransfer(value: string | null, scope: OrcadMigrationSourceScope): boolean { + return value === null || orcadMigrationOwnerMatchesScope(value, scope) +} + +function contextTouchesScope( + context: WorkspaceRunContext | TaskSourceContext | null | undefined, + scope: OrcadMigrationSourceScope +): boolean { + return ( + context?.hostId === scope.hostId || + (!context?.hostId && orcadMigrationOwnsRepoId(scope, context?.repoId)) + ) +} + +function contextCanTransfer( + context: WorkspaceRunContext | TaskSourceContext | null | undefined, + scope: OrcadMigrationSourceScope +): boolean { + if (!context) { + return true + } + return ( + context.hostId === scope.hostId && + typeof context.repoId === 'string' && + scope.repoIds.has(context.repoId) && + (!context.projectHostSetupId || context.projectHostSetupId === context.repoId) + ) +} + +function projectAutomationToDestination(source: Automation): Automation { + const destination: Automation = { + ...structuredClone(source), + runContext: projectContextToDestination(source.runContext), + sourceContext: projectContextToDestination(source.sourceContext), + executionTargetType: 'local', + executionTargetId: 'local', + schedulerOwner: 'remote_host_service', + workspaceId: projectOwnerToDestination(source.workspaceId) + } + delete destination.executionTargetGeneration + return destination +} + +function projectAutomationRunToDestination(source: AutomationRun): AutomationRun { + return { + ...structuredClone(source), + runContext: projectContextToDestination(source.runContext), + sourceContext: projectContextToDestination(source.sourceContext), + workspaceId: projectOwnerToDestination(source.workspaceId) + } +} + +function projectContextToDestination( + context: T | null | undefined +): T | null { + if (!context) { + return null + } + return Object.assign(structuredClone(context), { + hostId: LOCAL_EXECUTION_HOST_ID, + projectHostSetupId: context.repoId ?? null + }) +} + +function projectOwnerToDestination(value: string | null): string | null { + return value === null ? null : unqualifyOrcadMigrationOwnerKey(value) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-subtraction.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-subtraction.ts new file mode 100644 index 00000000000..a8604c71988 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-automation-subtraction.ts @@ -0,0 +1,31 @@ +import { getAutomationRunRepoId } from '../../../shared/automation-run-identity' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { createOrcadMigrationSourceScope, orcadMigrationOwnsRepoId } from './orcad-source-scope' + +/** The manifest's automations, and any a downgraded build added to a moved project since. */ +export function subtractOrcadMigrationSourceAutomationState( + state: PersistedState, + manifest: OrcadMigrationManifest +): void { + const scope = createOrcadMigrationSourceScope({ + source: manifest.source, + catalog: manifest.payload, + repos: state.repos + }) + const automationIds = new Set([ + ...(manifest.payload.dormantState?.automations ?? []).map((entry) => entry.id), + ...state.automations + .filter( + (entry) => + orcadMigrationOwnsRepoId(scope, getAutomationRunRepoId(entry)) && + (entry.executionTargetType !== 'ssh' || entry.executionTargetId === scope.targetId) + ) + .map((entry) => entry.id) + ]) + const runIds = new Set((manifest.payload.dormantState?.automationRuns ?? []).map((run) => run.id)) + state.automations = state.automations.filter((entry) => !automationIds.has(entry.id)) + state.automationRuns = state.automationRuns.filter( + (run) => !runIds.has(run.id) && !automationIds.has(run.automationId) + ) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog-subtraction.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog-subtraction.ts new file mode 100644 index 00000000000..3322494e17a --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog-subtraction.ts @@ -0,0 +1,52 @@ +/** + * Subtracts a migrated target's catalog rows from a copy of the source profile (the delta view). + * Only rows the manifest names and the target still owns go; a group stays while anything outside + * the migration references it. The live profile is never passed here. + */ +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { orcadSourceFolderWorkspaceIds, repoBelongsToOrcadSource } from './orcad-source-ownership' + +export function subtractOrcadSourceCatalogState( + state: StoreRuntimeState['state'], + manifest: OrcadMigrationManifest +): void { + const repoIds = new Set(manifest.payload.repositories.map((repo) => repo.id)) + const folderIds = new Set(manifest.payload.folderWorkspaces.map((workspace) => workspace.id)) + const groupIds = new Set(manifest.payload.projectGroups.map((group) => group.id)) + const targetId = manifest.source.sshTargetId + // Before any repo goes: a folder owned through the repos inside it would read as local after. + const ownedFolderIds = orcadSourceFolderWorkspaceIds(state, targetId) + state.repos = state.repos.filter( + (repo) => !(repoIds.has(repo.id) && repoBelongsToOrcadSource(repo, targetId)) + ) + state.folderWorkspaces = state.folderWorkspaces.filter( + (workspace) => !(folderIds.has(workspace.id) && ownedFolderIds.has(workspace.id)) + ) + removeUnreferencedSourceGroups(state, groupIds, manifest.source.sshTargetId) +} + +function removeUnreferencedSourceGroups( + state: StoreRuntimeState['state'], + candidateIds: ReadonlySet, + targetId: string +): void { + let removed = true + while (removed) { + removed = false + const referencedIds = new Set([ + ...state.repos.flatMap((repo) => (repo.projectGroupId ? [repo.projectGroupId] : [])), + ...state.folderWorkspaces.map((workspace) => workspace.projectGroupId), + ...state.projectGroups.flatMap((group) => (group.parentGroupId ? [group.parentGroupId] : [])) + ]) + const next = state.projectGroups.filter((group) => { + const shouldRemove = + candidateIds.has(group.id) && + group.connectionId === targetId && + !referencedIds.has(group.id) + removed ||= shouldRemove + return !shouldRemove + }) + state.projectGroups = next + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog.ts new file mode 100644 index 00000000000..16ad2845469 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-catalog.ts @@ -0,0 +1,57 @@ +import type { OrcadMigrationCatalogPayload } from '../../../shared/orcad-migration-manifest' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { SshTarget } from '../../../shared/ssh-types' +import type { Store } from '../../persistence' +import { orcadSourceFolderWorkspaceIds, repoBelongsToOrcadSource } from './orcad-source-ownership' + +type OrcadSourceCatalogStore = Pick + +export function collectOrcadMigrationSourceCatalog( + store: OrcadSourceCatalogStore, + target: Pick +): OrcadMigrationCatalogPayload { + const allRepos = store.getRepos() + const allGroups = store.getProjectGroups() + const allFolders = store.getFolderWorkspaces() + const repositories = allRepos + .filter((repo) => repoBelongsToOrcadSource(repo, target.id)) + .map((repo) => structuredClone(repo)) + const ownedFolderIds = orcadSourceFolderWorkspaceIds( + { repos: allRepos, projectGroups: allGroups, folderWorkspaces: allFolders }, + target.id + ) + const folderWorkspaces = allFolders + .filter((workspace) => ownedFolderIds.has(workspace.id)) + .map((workspace) => structuredClone(workspace)) + const projectGroups = collectProjectGroups( + allGroups, + new Set([ + ...repositories.flatMap((repo) => (repo.projectGroupId ? [repo.projectGroupId] : [])), + ...folderWorkspaces.map((workspace) => workspace.projectGroupId), + ...allGroups.filter((group) => group.connectionId === target.id).map((group) => group.id) + ]) + ) + return { repositories, projectGroups, folderWorkspaces } +} + +function collectProjectGroups( + groups: ProjectGroup[], + initialIds: ReadonlySet +): ProjectGroup[] { + const byId = new Map(groups.map((group) => [group.id, group])) + const includedIds = new Set(initialIds) + const pending = [...initialIds] + while (pending.length > 0) { + const nextId = pending.pop() + if (!nextId) { + continue + } + const group = byId.get(nextId) + if (!group?.parentGroupId || includedIds.has(group.parentGroupId)) { + continue + } + includedIds.add(group.parentGroupId) + pending.push(group.parentGroupId) + } + return groups.filter((group) => includedIds.has(group.id)).map((group) => structuredClone(group)) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-browser-intents.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-browser-intents.ts new file mode 100644 index 00000000000..b139d31d17c --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-browser-intents.ts @@ -0,0 +1,73 @@ +import type { OrcadMigrationClientHostedBrowserCloseIntent } from '../../../shared/orcad-migration-client-state' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { + orcadMigrationOwnerMatchesScope, + unqualifyOrcadMigrationOwnerKey, + type OrcadMigrationSourceScope +} from './orcad-source-scope' + +export function collectCloseIntents( + state: PersistedState, + scope: OrcadMigrationSourceScope, + destinationEnvironmentId: string | undefined, + eligibleSession: WorkspaceSessionState | undefined, + onBlocked: () => void +): OrcadMigrationClientHostedBrowserCloseIntent[] | undefined { + const eligiblePages = new Set() + for (const [ownerKey, pages] of Object.entries( + eligibleSession?.clientHostedBrowserPagesByWorktree ?? {} + )) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + const worktreeId = unqualifyOrcadMigrationOwnerKey(ownerKey) + for (const page of pages) { + eligiblePages.add(`${worktreeId}\0${page.browserPageId}`) + } + } + const result: OrcadMigrationClientHostedBrowserCloseIntent[] = [] + const seen = new Set() + const sessions = [ + state.workspaceSession, + ...Object.values(state.workspaceSessionsByHostId ?? {}).flatMap((entry) => + entry ? [entry] : [] + ) + ].filter((entry): entry is WorkspaceSessionState => Boolean(entry)) + for (const session of sessions) { + for (const [sourceEnvironmentId, intents] of Object.entries( + session.clientHostedBrowserCloseIntentsByEnvironment ?? {} + )) { + // Intents already keyed to the destination were handled by a prior retry. + if (sourceEnvironmentId === destinationEnvironmentId) { + continue + } + for (const intent of intents) { + if (!orcadMigrationOwnerMatchesScope(intent.worktreeId, scope)) { + continue + } + const worktreeId = unqualifyOrcadMigrationOwnerKey(intent.worktreeId) + const key = `${sourceEnvironmentId}\0${intent.browserPageId}\0${worktreeId}` + if ( + !eligiblePages.has(`${worktreeId}\0${intent.browserPageId}`) || + !destinationEnvironmentId + ) { + onBlocked() + continue + } + if (seen.has(key)) { + onBlocked() + continue + } + seen.add(key) + result.push({ + sourceEnvironmentId, + browserPageId: intent.browserPageId, + worktreeId, + closedAt: intent.closedAt + }) + } + } + } + return result.length > 0 ? result : undefined +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-focus-census.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-focus-census.test.ts new file mode 100644 index 00000000000..e2929d1d64a --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-focus-census.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { ExecutionHostId } from '../../../shared/execution-host' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { Repo } from '../../../shared/repo-types' +import type { SshTarget } from '../../../shared/ssh-types' +import type { Tab } from '../../../shared/tab-types' +import { worktreeWorkspaceKey } from '../../../shared/workspace-scope' +import { collectOrcadMigrationUntransferredDependencyCensus } from './orcad-source-dependency-census' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' + +const TARGET: SshTarget = { id: 'ssh-prod', label: 'Prod', host: 'prod', port: 22, username: 'u' } +const SSH_HOST = `ssh:${TARGET.id}` as const +const REPO: Repo = { + id: 'repo-1', + path: '/srv/repo', + displayName: 'Repository', + badgeColor: '#737373', + addedAt: 1, + connectionId: TARGET.id, + executionHostId: SSH_HOST +} +const WORKTREE = `${REPO.id}::/srv/repo` + +function manifest(destinationEnvironmentId?: string): OrcadMigrationManifest { + return { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-10-03T12:00:00.000Z', + source: { sshTargetId: TARGET.id, sshTargetGeneration: null, targetLabel: TARGET.label }, + payload: { repositories: [REPO], projectGroups: [], folderWorkspaces: [] }, + ...(destinationEnvironmentId ? { destinationEnvironmentId } : {}), + manifestSha256: 'a'.repeat(64) + } +} + +function editorTab(executionHostId?: ExecutionHostId): Tab { + return { + id: 'tab-1', + entityId: '/srv/repo/README.md', + groupId: 'group-1', + worktreeId: WORKTREE, + ...(executionHostId ? { executionHostId } : {}), + contentType: 'editor', + label: 'README.md', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } +} + +function sourceState(): PersistedState { + const state = getDefaultPersistedState('/home/test') + state.sshTargets = [TARGET] + state.repos = [REPO] + return state +} + +function focusLocalOnSourceWorktree(state: PersistedState): void { + state.workspaceSession = { + ...state.workspaceSession, + activeRepoId: REPO.id, + activeWorktreeId: WORKTREE, + activeWorkspaceKey: worktreeWorkspaceKey(WORKTREE), + activeWorkspaceExecutionHostId: SSH_HOST, + activeTabId: 'tab-1' + } +} + +const sessionCount = (state: PersistedState): number => + collectOrcadMigrationUntransferredDependencyCensus(state, manifest('env-1')).counts[ + 'workspace-session' + ] + +describe('orcad migration census and client focus', () => { + it('never blocks a move on client focus aimed at a migrating worktree', () => { + const state = sourceState() + focusLocalOnSourceWorktree(state) + + expect(sessionCount(state)).toBe(0) + }) + + it('passes a v1.4.218 profile quit while focused on the source worktree', () => { + // v1.4.218 copied the global focus fields into 'local' and into every host partition it wrote. + const state = sourceState() + focusLocalOnSourceWorktree(state) + const focusCopy = { + ...state.workspaceSession, + unifiedTabs: {}, + tabsByWorktree: {} + } + state.workspaceSessionsByHostId = { + [SSH_HOST]: { ...focusCopy, unifiedTabs: { [WORKTREE]: [editorTab('local')] } }, + 'ssh:other-a': { ...focusCopy }, + 'ssh:other-b': { ...focusCopy } + } + expect(sessionCount(state)).toBe(0) + }) + + it('carries no client focus into the destination session', () => { + const state = sourceState() + focusLocalOnSourceWorktree(state) + state.workspaceSession.unifiedTabs = { [WORKTREE]: [editorTab(SSH_HOST)] } + + const session = collectOrcadMigrationSourceDormantState(state, manifest().source, { + repositories: [REPO], + projectGroups: [], + folderWorkspaces: [] + }).payload.workspaceSession + + expect(session?.unifiedTabs?.[WORKTREE]).toHaveLength(1) + expect(session?.activeWorktreeId ?? null).toBeNull() + expect(session?.activeWorkspaceKey ?? null).toBeNull() + }) + + it("counts a 'local'-stamped tab in a migrating SSH worktree as the SSH host's", () => { + const local = sourceState() + local.workspaceSession.unifiedTabs = { [WORKTREE]: [editorTab('local')] } + const partitioned = sourceState() + partitioned.workspaceSessionsByHostId = { + [SSH_HOST]: { + ...partitioned.workspaceSession, + unifiedTabs: { [WORKTREE]: [editorTab('local')] } + } + } + + expect(sessionCount(local)).toBe(0) + expect(sessionCount(partitioned)).toBe(0) + }) + + it('still blocks a tab another host owns inside a migrating worktree', () => { + const state = sourceState() + state.workspaceSession.unifiedTabs = { [WORKTREE]: [editorTab('ssh:other')] } + + expect(sessionCount(state)).toBe(1) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.test.ts new file mode 100644 index 00000000000..cf3f668e7b6 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, it } from 'vitest' +import type { OrcadMigrationCatalogPayload } from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../../../shared/constants' +import type { TerminalTab } from '../../../shared/terminal-tab-types' +import type { BrowserWorkspace } from '../../../shared/browser-workspace-types' +import { + CLIENT_HOSTED_BROWSER_PAGE_RECORD_VERSION, + type PersistedClientHostedBrowserPage +} from '../../../shared/client-hosted-browser-page-record' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { collectOrcadMigrationSourceClientState } from './orcad-source-client-state' + +const source = { + sshTargetId: 'target-1', + sshTargetGeneration: 3, + targetLabel: 'Build host' +} +const catalog: OrcadMigrationCatalogPayload = { + repositories: [ + { + id: 'repo-1', + path: '/srv/repo-1', + displayName: 'Repo', + badgeColor: '#737373', + addedAt: 1, + connectionId: source.sshTargetId, + executionHostId: 'ssh:target-1' + } + ], + projectGroups: [], + folderWorkspaces: [] +} + +function state(): PersistedState { + const persisted = getDefaultPersistedState('/home/test') + persisted.sshTargets = [ + { + id: source.sshTargetId, + label: source.targetLabel, + host: 'source.example.com', + port: 22, + username: 'deploy', + portForwards: [] + } + ] + return persisted +} + +function terminalTab(id: string, worktreeId: string): TerminalTab { + return { + id, + ptyId: null, + worktreeId, + title: id, + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } +} + +function session( + fields: Pick & Partial +): WorkspaceSessionState { + return { ...getDefaultWorkspaceSession(), tabGroups: {}, ...fields } +} + +function browserWorkspace(id: string, worktreeId: string): BrowserWorkspace { + return { + id, + worktreeId, + url: 'about:blank', + title: id, + loading: false, + faviconUrl: null, + canGoBack: false, + canGoForward: false, + loadError: null, + createdAt: 1 + } +} + +function hostedPage(browserPageId: string, workspaceId: string): PersistedClientHostedBrowserPage { + return { + v: CLIENT_HOSTED_BROWSER_PAGE_RECORD_VERSION, + browserPageId, + workspaceId, + browserProfileId: 'default', + url: 'about:blank', + title: browserPageId, + pairedDeviceId: 'device-1', + savedAt: 1 + } +} + +describe('source client-state migration', () => { + it('rekeys representable mobile selections and desktop routing', () => { + const current = state() + current.mobileClientTabSelectionsByDeviceId = { + phone: { + 'ssh:target-1|repo-1::/srv/worktree': { + activeTabId: 'tab-1', + activeGroupId: null, + activeTabIdByGroupId: {} + } + } + } + current.ui.lastActiveRepoId = 'repo-1' + current.ui.lastActiveWorktreeId = 'ssh:target-1|repo-1::/srv/worktree' + current.ui.workspaceHostScope = 'ssh:target-1' + current.ui.showDotfilesByWorktree = { + 'ssh:target-1|repo-1::/srv/worktree': false + } + const scoped = session({ + tabsByWorktree: { + 'ssh:target-1|repo-1::/srv/worktree': [ + terminalTab('tab-1', 'ssh:target-1|repo-1::/srv/worktree') + ] + }, + tabGroups: {} + }) + + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + 'environment-1', + scoped + ) + expect(result.blockedCounts).toEqual({ + 'mobile-tab-selection': 0, + 'ui-routing': 0, + 'saved-port-forward': 0 + }) + expect(result.payload?.mobileClientTabSelectionsByDeviceId).toMatchObject({ + phone: { 'repo-1::/srv/worktree': { activeTabId: 'tab-1' } } + }) + expect(result.payload?.uiRouting).toMatchObject({ + lastActiveRepoId: 'repo-1', + lastActiveWorktreeId: 'repo-1::/srv/worktree', + workspaceHostScope: 'local', + showDotfilesByWorktree: { 'repo-1::/srv/worktree': false } + }) + }) + + it('blocks a mobile selection that points at a group outside the migrated owner', () => { + const current = state() + const sourceOwner = 'ssh:target-1|repo-1::/srv/worktree' + const unrelatedOwner = 'ssh:target-1|repo-2::/srv/other' + current.mobileClientTabSelectionsByDeviceId = { + phone: { + [sourceOwner]: { + activeTabId: null, + activeGroupId: 'group-unrelated', + activeTabIdByGroupId: {} + } + } + } + const scoped = session({ + tabsByWorktree: {}, + tabGroups: { + [unrelatedOwner]: [ + { + id: 'group-unrelated', + worktreeId: unrelatedOwner, + activeTabId: null, + tabOrder: [] + } + ] + } + }) + + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + 'environment-1', + scoped + ) + + expect(result.payload?.mobileClientTabSelectionsByDeviceId).toBeUndefined() + expect(result.blockedCounts['mobile-tab-selection']).toBe(1) + }) + + it('blocks a per-group tab selection whose group is outside the migrated owner', () => { + const current = state() + const sourceOwner = 'ssh:target-1|repo-1::/srv/worktree' + const unrelatedOwner = 'ssh:target-1|repo-2::/srv/other' + current.mobileClientTabSelectionsByDeviceId = { + phone: { + [sourceOwner]: { + activeTabId: null, + activeGroupId: null, + activeTabIdByGroupId: { 'group-unrelated': 'tab-source' } + } + } + } + const scoped = session({ + tabsByWorktree: { + [sourceOwner]: [terminalTab('tab-source', sourceOwner)] + }, + tabGroups: { + [unrelatedOwner]: [ + { + id: 'group-unrelated', + worktreeId: unrelatedOwner, + activeTabId: 'tab-source', + tabOrder: ['tab-source'] + } + ] + } + }) + + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + 'environment-1', + scoped + ) + + expect(result.payload?.mobileClientTabSelectionsByDeviceId).toBeUndefined() + expect(result.blockedCounts['mobile-tab-selection']).toBe(1) + }) + + it('captures saved forwards and reports duplicate local ports', () => { + const current = state() + current.sshTargets[0].portForwards = [ + { localPort: 9000, remoteHost: '127.0.0.1', remotePort: 6768 }, + { localPort: 9000, remoteHost: '127.0.0.1', remotePort: 6769 } + ] + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + undefined, + undefined + ) + expect(result.payload?.savedPortForwards).toHaveLength(2) + expect(result.blockedCounts['saved-port-forward']).toBe(1) + }) + + it('captures only close intents for transferred client-hosted pages', () => { + const current = state() + const worktreeId = 'ssh:target-1|repo-1::/srv/worktree' + current.workspaceSession = session({ + tabsByWorktree: {}, + terminalLayoutsByTabId: {}, + browserTabsByWorktree: { + [worktreeId]: [browserWorkspace('browser-1', worktreeId)] + }, + clientHostedBrowserPagesByWorktree: { + [worktreeId]: [hostedPage('page-1', 'browser-1')] + }, + clientHostedBrowserCloseIntentsByEnvironment: { + 'old-environment': [ + { browserPageId: 'page-1', worktreeId, closedAt: 42 }, + { browserPageId: 'unrelated-page', worktreeId, closedAt: 43 } + ], + 'environment-1': [{ browserPageId: 'destination-page', worktreeId, closedAt: 44 }] + } + }) + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + 'environment-1', + current.workspaceSession + ) + + expect(result.blockedCount).toBe(1) + expect(result.payload?.clientHostedBrowserCloseIntents).toEqual([ + { + sourceEnvironmentId: 'old-environment', + browserPageId: 'page-1', + worktreeId: 'repo-1::/srv/worktree', + closedAt: 42 + } + ]) + }) + + it("ignores another host's agent acknowledgements and blocks only the source's own", () => { + const current = state() + current.workspaceSession = session({ + tabsByWorktree: { '/local/repo::/local/repo': [terminalTab('local-tab', '/local/repo')] } + }) + current.workspaceSessionsByHostId = { + 'ssh:target-2': session({ tabsByWorktree: { 'other::/x': [terminalTab('other-tab', 'x')] } }), + 'ssh:target-1': session({ + tabsByWorktree: { 'unmoved::/y': [terminalTab('source-tab', 'unmoved::/y')] } + }) + } + current.ui.acknowledgedAgentsByPaneKey = { + 'local-tab:leaf': 1, + 'other-tab:leaf': 2, + 'source-tab:leaf': 3 + } + const result = collectOrcadMigrationSourceClientState( + current, + source, + catalog, + 'environment-1', + undefined + ) + expect(result.blockedCounts['ui-routing']).toBe(1) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.ts new file mode 100644 index 00000000000..2c3acd8011d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-state.ts @@ -0,0 +1,281 @@ +import { hostStableKey } from '../../../shared/automation-owner-key' +import { parsePersistedAutomationHostFilter } from '../../../shared/automation-host-filter' +import type { + OrcadMigrationClientStatePayload, + OrcadMigrationUiRoutingState +} from '../../../shared/orcad-migration-client-state' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import type { + PersistedMobileClientTabSelection, + PersistedState +} from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { sessionPartitions } from './orcad-source-workspace-session-fragments' +import { collectCloseIntents } from './orcad-source-client-browser-intents' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + orcadMigrationOwnsRepoId, + unqualifyOrcadMigrationOwnerKey +} from './orcad-source-scope' +import { paneBelongsToTabs } from '../../../shared/workspace-session-pane-ownership' + +export type OrcadMigrationSourceClientStateInspection = { + payload: OrcadMigrationClientStatePayload | undefined + /** Invalid or unmatched closes are folded into the existing workspace-session blocker. */ + blockedCount: number + blockedCounts: { + 'mobile-tab-selection': number + 'ui-routing': number + 'saved-port-forward': number + } +} + +export function collectOrcadMigrationSourceClientState( + state: PersistedState, + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + destinationEnvironmentId: string | undefined, + eligibleSession: WorkspaceSessionState | undefined +): OrcadMigrationSourceClientStateInspection { + const scope = createOrcadMigrationSourceScope({ source, catalog, repos: state.repos }) + const blockedCounts = { + 'mobile-tab-selection': 0, + 'ui-routing': 0, + 'saved-port-forward': 0 + } + let blockedCount = 0 + const mobile = collectMobileSelections(state, scope, eligibleSession, blockedCounts) + const uiRouting = collectUiRouting( + state, + scope, + destinationEnvironmentId, + eligibleSession, + blockedCounts + ) + const target = state.sshTargets.find((entry) => entry.id === scope.targetId) + const savedPortForwards = target?.portForwards ? structuredClone(target.portForwards) : undefined + if (savedPortForwards) { + const ports = new Set() + for (const forward of savedPortForwards) { + if (ports.has(forward.localPort)) { + blockedCounts['saved-port-forward'] += 1 + } + ports.add(forward.localPort) + } + } + const closeIntents = collectCloseIntents( + state, + scope, + destinationEnvironmentId, + eligibleSession, + () => { + blockedCount += 1 + } + ) + const clientState: OrcadMigrationClientStatePayload = { + ...(mobile && Object.keys(mobile).length > 0 + ? { mobileClientTabSelectionsByDeviceId: mobile } + : {}), + ...(uiRouting && Object.keys(uiRouting).length > 0 ? { uiRouting } : {}), + ...(savedPortForwards && savedPortForwards.length > 0 ? { savedPortForwards } : {}), + ...(closeIntents && closeIntents.length > 0 + ? { clientHostedBrowserCloseIntents: closeIntents } + : {}) + } + return { + payload: Object.keys(clientState).length > 0 ? clientState : undefined, + blockedCount, + blockedCounts + } +} + +function collectMobileSelections( + state: PersistedState, + scope: ReturnType, + session: WorkspaceSessionState | undefined, + blockedCounts: OrcadMigrationSourceClientStateInspection['blockedCounts'] +): + | NonNullable + | undefined { + const eligible = session ? collectEligibleSessionIdentity(session, scope) : null + const result: NonNullable< + OrcadMigrationClientStatePayload['mobileClientTabSelectionsByDeviceId'] + > = {} + const destinationKeys = new Set() + for (const [deviceId, selections] of Object.entries( + state.mobileClientTabSelectionsByDeviceId ?? {} + )) { + const projected: Record = {} + for (const [ownerKey, selection] of Object.entries(selections)) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + const destinationKey = unqualifyOrcadMigrationOwnerKey(ownerKey) + if (destinationKeys.has(`${deviceId}\0${destinationKey}`)) { + blockedCounts['mobile-tab-selection'] += 1 + continue + } + if (!mobileSelectionIsRepresentable(selection, eligible)) { + blockedCounts['mobile-tab-selection'] += 1 + continue + } + destinationKeys.add(`${deviceId}\0${destinationKey}`) + projected[destinationKey] = structuredClone(selection) + } + if (Object.keys(projected).length > 0) { + result[deviceId] = projected + } + } + return Object.keys(result).length > 0 ? result : undefined +} + +function collectUiRouting( + state: PersistedState, + scope: ReturnType, + destinationEnvironmentId: string | undefined, + session: WorkspaceSessionState | undefined, + blockedCounts: OrcadMigrationSourceClientStateInspection['blockedCounts'] +): OrcadMigrationUiRoutingState | undefined { + const ui = state.ui + const result: OrcadMigrationUiRoutingState = {} + if (ui.lastActiveRepoId && orcadMigrationOwnsRepoId(scope, ui.lastActiveRepoId)) { + result.lastActiveRepoId = ui.lastActiveRepoId + } + if (ui.lastActiveWorktreeId && orcadMigrationOwnerMatchesScope(ui.lastActiveWorktreeId, scope)) { + result.lastActiveWorktreeId = unqualifyOrcadMigrationOwnerKey(ui.lastActiveWorktreeId) + } + const filterRepoIds = ui.filterRepoIds.filter((repoId) => orcadMigrationOwnsRepoId(scope, repoId)) + if (filterRepoIds.length > 0) { + result.filterRepoIds = filterRepoIds + } + const dotfiles = Object.entries(ui.showDotfilesByWorktree ?? {}) + .filter(([ownerKey]) => orcadMigrationOwnerMatchesScope(ownerKey, scope)) + .map(([ownerKey, enabled]) => [unqualifyOrcadMigrationOwnerKey(ownerKey), enabled] as const) + if (dotfiles.length > 0) { + result.showDotfilesByWorktree = Object.fromEntries(dotfiles) + } + const dismissed = (ui.setupScriptPromptDismissedRepoIds ?? []).filter((repoId) => + orcadMigrationOwnsRepoId(scope, repoId) + ) + if (dismissed.length > 0) { + result.setupScriptPromptDismissedRepoIds = dismissed + } + const manualOrder = (ui.manualRepoOrder ?? []) + .filter((entry) => entry.hostId === scope.hostId && scope.repoIds.has(entry.repoId)) + .map((entry) => ({ hostId: 'local' as const, repoId: entry.repoId })) + if (manualOrder.length > 0) { + result.manualRepoOrder = manualOrder + } + if (ui.workspaceHostScope === scope.hostId) { + result.workspaceHostScope = 'local' + } + const visibleHosts = (ui.visibleWorkspaceHostIds ?? []).filter( + (hostId) => hostId === scope.hostId + ) + if (visibleHosts.length > 0) { + result.visibleWorkspaceHostIds = ['local'] + } + const hostOrder = (ui.workspaceHostOrder ?? []).filter((hostId) => hostId === scope.hostId) + if (hostOrder.length > 0) { + result.workspaceHostOrder = ['local'] + } + const filter = parsePersistedAutomationHostFilter(ui.automationHostFilter) + if ( + filter.kind === 'host' && + hostStableKey(filter.host) === `host:desktop:ssh:${encodeURIComponent(scope.targetId)}` + ) { + result.automationHostFilter = destinationEnvironmentId + ? { + kind: 'host', + hostKey: hostStableKey({ + authority: { kind: 'runtime', environmentId: destinationEnvironmentId }, + selector: { kind: 'self' } + }) + } + : { kind: 'host', hostKey: 'host:desktop:self' } + } + const eligible = session ? collectEligibleSessionIdentity(session, scope) : null + const sourceTabIds = collectSourceOwnedTabIds(state, scope) + const acknowledgements = Object.entries(ui.acknowledgedAgentsByPaneKey ?? {}).filter( + ([paneKey]) => { + // Another host's acknowledgement is not this source's state to move or block on. + if (!paneBelongsToTabs(paneKey, sourceTabIds)) { + return false + } + const allowed = eligible ? paneBelongsToTabs(paneKey, eligible.tabIds) : false + if (!allowed) { + blockedCounts['ui-routing'] += 1 + } + return allowed + } + ) + if (acknowledgements.length > 0) { + result.acknowledgedAgentsByPaneKey = Object.fromEntries(acknowledgements) + } + return Object.keys(result).length > 0 ? result : undefined +} + +/** Every tab the source owns: its own partition's, and those keyed to its projects anywhere. */ +function collectSourceOwnedTabIds( + state: PersistedState, + scope: ReturnType +): Set { + const tabIds = new Set() + for (const [hostId, session] of sessionPartitions(state, 'local')) { + for (const [ownerKey, tabs] of Object.entries(session.tabsByWorktree ?? {})) { + if (hostId === scope.hostId || orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + tabs.forEach((tab) => tabIds.add(tab.id)) + } + } + } + return tabIds +} + +type EligibleSessionIdentity = { tabIds: ReadonlySet; groupIds: ReadonlySet } + +function collectEligibleSessionIdentity( + session: WorkspaceSessionState, + scope: ReturnType +): EligibleSessionIdentity { + const tabIds = new Set() + const groupIds = new Set() + for (const [ownerKey, tabs] of Object.entries(session.tabsByWorktree ?? {})) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + tabs.forEach((tab) => tabIds.add(tab.id)) + } + for (const [ownerKey, groups] of Object.entries(session.tabGroups ?? {})) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + groups.forEach((group) => groupIds.add(group.id)) + } + return { tabIds, groupIds } +} + +function mobileSelectionIsRepresentable( + selection: PersistedMobileClientTabSelection, + eligible: EligibleSessionIdentity | null +): boolean { + if (!eligible) { + return ( + selection.activeTabId === null && + selection.activeGroupId === null && + Object.keys(selection.activeTabIdByGroupId).length === 0 + ) + } + if (selection.activeTabId !== null && !eligible.tabIds.has(selection.activeTabId)) { + return false + } + if (selection.activeGroupId !== null && !eligible.groupIds.has(selection.activeGroupId)) { + return false + } + return Object.entries(selection.activeTabIdByGroupId).every( + ([groupId, tabId]) => eligible.groupIds.has(groupId) && eligible.tabIds.has(tabId) + ) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.test.ts new file mode 100644 index 00000000000..ad02faddceb --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { orcadMigrationCutoverFixture } from '../../ssh/orcad-migration-cutover-fixture' +import { subtractOrcadMigrationClientState } from './orcad-source-client-subtraction' + +const worktreeId = 'repo-1::/srv/worktree' + +function manifest(): OrcadMigrationManifest { + const base = orcadMigrationCutoverFixture('m-1', 'target-1', { + environmentId: 'environment-1' + }).manifest + return { + ...base, + payload: { + ...base.payload, + dormantState: { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [], + clientState: { + clientHostedBrowserCloseIntents: [ + { sourceEnvironmentId: 'old', browserPageId: 'page-1', worktreeId, closedAt: 42 }, + { sourceEnvironmentId: 'old', browserPageId: 'page-2', worktreeId, closedAt: 43 } + ] + } + } + } + } +} + +describe('retiring client-hosted browser close intents', () => { + it('finishes a retry after the moving write already flushed', () => { + const state = getDefaultPersistedState('/home/test') + state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment = { + old: [ + { browserPageId: 'page-1', worktreeId, closedAt: 42 }, + { browserPageId: 'page-2', worktreeId, closedAt: 43 } + ] + } + subtractOrcadMigrationClientState(state, manifest()) + const moved = structuredClone( + state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment + ) + expect(() => subtractOrcadMigrationClientState(state, manifest())).not.toThrow() + expect(state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment).toEqual(moved) + }) + + it('still refuses a source intent that is gone without reaching the destination', () => { + const state = getDefaultPersistedState('/home/test') + state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment = { + old: [{ browserPageId: 'page-2', worktreeId, closedAt: 43 }], + 'environment-1': [{ browserPageId: 'page-1', worktreeId, closedAt: 99 }] + } + expect(() => subtractOrcadMigrationClientState(state, manifest())).toThrow( + 'orcad_migration_source_close_intent_changed' + ) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.ts new file mode 100644 index 00000000000..d195255fe8d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-client-subtraction.ts @@ -0,0 +1,184 @@ +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' +import { parsePersistedAutomationHostFilter } from '../../../shared/automation-host-filter' +import { hostStableKey } from '../../../shared/automation-owner-key' +import { toRuntimeExecutionHostId } from '../../../shared/execution-host' +import { composeWorktreeHostIdentity } from '../../../shared/worktree/host-qualified-identity' +import type { OrcadMigrationClientStatePayload } from '../../../shared/orcad-migration-client-state' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { + MAX_CLIENT_HOSTED_BROWSER_CLOSE_INTENTS, + type ClientHostedBrowserCloseIntent +} from '../../../shared/client-hosted-browser-close-intent' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + unqualifyOrcadMigrationOwnerKey +} from './orcad-source-scope' + +/** On a copy of the profile only: the client state an earlier migration moved. */ +export function subtractOrcadMigrationClientState( + state: PersistedState, + manifest: OrcadMigrationManifest +): void { + const clientState = manifest.payload.dormantState?.clientState + if (!clientState) { + return + } + const source = manifest.source + const scope = createOrcadMigrationSourceScope({ + source, + catalog: manifest.payload, + repos: state.repos + }) + if (clientState.mobileClientTabSelectionsByDeviceId) { + for (const [deviceId, captured] of Object.entries( + clientState.mobileClientTabSelectionsByDeviceId + )) { + const selections = state.mobileClientTabSelectionsByDeviceId?.[deviceId] + if (!selections) { + continue + } + for (const ownerKey of Object.keys(captured)) { + delete selections[ownerKey] + } + if (Object.keys(selections).length === 0) { + delete state.mobileClientTabSelectionsByDeviceId?.[deviceId] + } + } + } + const target = state.sshTargets.find((entry) => entry.id === source.sshTargetId) + if (target && clientState.savedPortForwards) { + target.portForwards = structuredClone(clientState.savedPortForwards) + } + subtractCloseIntents(state, clientState.clientHostedBrowserCloseIntents ?? [], manifest) + rewriteDesktopUiForDestination(state, manifest, scope) +} + +function subtractCloseIntents( + state: PersistedState, + captured: readonly NonNullable< + OrcadMigrationClientStatePayload['clientHostedBrowserCloseIntents'] + >[number][], + manifest: OrcadMigrationManifest +): void { + if (captured.length === 0) { + return + } + const current = state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment ?? {} + const next: Record = Object.fromEntries( + Object.entries(current).map(([key, entries]) => [key, structuredClone(entries)]) + ) + const destinationEnvironmentId = manifest.destinationEnvironmentId + if (!destinationEnvironmentId) { + throw new Error('orcad_migration_source_close_intent_destination_invalid') + } + for (const intent of captured) { + const expected = serializeOrcadMigrationValue({ + browserPageId: intent.browserPageId, + worktreeId: intent.worktreeId, + closedAt: intent.closedAt + }) + const samePage = (entry: ClientHostedBrowserCloseIntent): boolean => + entry.browserPageId === intent.browserPageId && entry.worktreeId === intent.worktreeId + const sourceEntries = next[intent.sourceEnvironmentId] ?? [] + const sourceIndex = sourceEntries.findIndex(samePage) + const destinationEntries = next[destinationEnvironmentId] ?? [] + const existing = destinationEntries.find(samePage) + if (sourceIndex === -1) { + // A retry after the moving write flushed: the identical intent already sits at the destination. + if (existing && serializeOrcadMigrationValue(existing) === expected) { + continue + } + throw new Error('orcad_migration_source_close_intent_changed') + } + if (serializeOrcadMigrationValue(sourceEntries[sourceIndex]) !== expected) { + throw new Error('orcad_migration_source_close_intent_changed') + } + sourceEntries.splice(sourceIndex, 1) + if (sourceEntries.length === 0) { + delete next[intent.sourceEnvironmentId] + } else { + next[intent.sourceEnvironmentId] = sourceEntries + } + if (existing) { + if (serializeOrcadMigrationValue(existing) !== expected) { + throw new Error('orcad_migration_close_intent_destination_conflict') + } + continue + } + if (destinationEntries.length >= MAX_CLIENT_HOSTED_BROWSER_CLOSE_INTENTS) { + throw new Error('orcad_migration_close_intent_destination_capacity_exceeded') + } + destinationEntries.push({ + browserPageId: intent.browserPageId, + worktreeId: intent.worktreeId, + closedAt: intent.closedAt + }) + next[destinationEnvironmentId] = destinationEntries + } + state.workspaceSession.clientHostedBrowserCloseIntentsByEnvironment = next +} + +function rewriteDesktopUiForDestination( + state: PersistedState, + manifest: OrcadMigrationManifest, + scope: ReturnType +): void { + const route = manifest.payload.dormantState?.clientState?.uiRouting + const destinationEnvironmentId = manifest.destinationEnvironmentId + if (!route || !destinationEnvironmentId) { + return + } + const destinationHostId = toRuntimeExecutionHostId(destinationEnvironmentId) + if ( + route.lastActiveWorktreeId && + orcadMigrationOwnerMatchesScope(state.ui.lastActiveWorktreeId, scope) + ) { + state.ui.lastActiveWorktreeId = composeWorktreeHostIdentity( + destinationHostId, + route.lastActiveWorktreeId + ) + } + if (route.workspaceHostScope === 'local' && state.ui.workspaceHostScope === scope.hostId) { + state.ui.workspaceHostScope = destinationHostId + } + if (route.visibleWorkspaceHostIds?.includes('local') && state.ui.visibleWorkspaceHostIds) { + state.ui.visibleWorkspaceHostIds = state.ui.visibleWorkspaceHostIds.map((hostId) => + hostId === scope.hostId ? destinationHostId : hostId + ) + } + if (route.workspaceHostOrder?.includes('local')) { + state.ui.workspaceHostOrder = (state.ui.workspaceHostOrder ?? []).map((hostId) => + hostId === scope.hostId ? destinationHostId : hostId + ) + } + if (route.manualRepoOrder) { + const repoIds = new Set(route.manualRepoOrder.map((entry) => entry.repoId)) + state.ui.manualRepoOrder = (state.ui.manualRepoOrder ?? []).map((entry) => + entry.hostId === scope.hostId && repoIds.has(entry.repoId) + ? { ...entry, hostId: destinationHostId } + : entry + ) + } + if (route.showDotfilesByWorktree) { + const next = { ...state.ui.showDotfilesByWorktree } + for (const [ownerKey, enabled] of Object.entries(next)) { + if (orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + delete next[ownerKey] + next[`${destinationHostId}|${unqualifyOrcadMigrationOwnerKey(ownerKey)}`] = enabled + } + } + state.ui.showDotfilesByWorktree = next + } + if (route.automationHostFilter?.kind === 'host') { + const current = parsePersistedAutomationHostFilter(state.ui.automationHostFilter) + const sourceKey = hostStableKey({ + authority: { kind: 'desktop' }, + selector: { kind: 'ssh', targetId: scope.targetId } + }) + if (current.kind === 'host' && hostStableKey(current.host) === sourceKey) { + state.ui.automationHostFilter = structuredClone(route.automationHostFilter) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-delta-view.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-delta-view.ts new file mode 100644 index 00000000000..4247dd13f71 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-delta-view.ts @@ -0,0 +1,65 @@ +/** + * The source as a delta move sees it: a copy of the profile with everything earlier migrations of + * the host already moved subtracted from it. What is left is what an older build added, and only + * that is exported, censused and later committed. + */ +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationDormantStatePayload, + OrcadMigrationManifest, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { Repo } from '../../../shared/repo-types' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import { subtractOrcadSourceCatalogState } from './orcad-source-catalog-subtraction' +import { + collectOrcadMigrationUntransferredDependencyCensus, + type OrcadMigrationSourceDependencyCensus +} from './orcad-source-dependency-census' +import { subtractOrcadMigrationSourceDormantState } from './orcad-source-dormant-subtraction' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' +import { subtractOrcadMigrationScopeWorkspaceSession } from './orcad-source-workspace-session-subtraction' + +export type OrcadMigrationDeltaView = { + getRepos: () => Repo[] + getFolderWorkspaces: () => FolderWorkspace[] + getProjectGroups: () => ProjectGroup[] + collectOrcadMigrationSourceDormantState: ( + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + destinationEnvironmentId?: string + ) => OrcadMigrationDormantStatePayload + inspectOrcadMigrationUntransferredDependencies: ( + manifest: OrcadMigrationManifest + ) => OrcadMigrationSourceDependencyCensus +} + +/** `moved` re-exports what the earlier migrations own today, including later edits to it. */ +export function createOrcadMigrationDeltaView( + runtime: Pick, + moved: OrcadMigrationManifest +): OrcadMigrationDeltaView { + const state = structuredClone(runtime.state) + subtractOrcadSourceCatalogState(state, moved) + subtractOrcadMigrationSourceDormantState(state, moved) + // The server owns the moved projects' tabs now, even ones the older build left unmovable. + subtractOrcadMigrationScopeWorkspaceSession(state, moved) + const storage = runtime.terminalScrollbackSnapshotStorage + return { + getRepos: () => state.repos, + getFolderWorkspaces: () => state.folderWorkspaces, + getProjectGroups: () => state.projectGroups, + collectOrcadMigrationSourceDormantState: (source, catalog, destinationEnvironmentId) => + collectOrcadMigrationSourceDormantState( + state, + source, + catalog, + storage, + destinationEnvironmentId + ).payload, + inspectOrcadMigrationUntransferredDependencies: (manifest) => + collectOrcadMigrationUntransferredDependencyCensus(state, manifest, storage) + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.test.ts new file mode 100644 index 00000000000..b20f4b14e7c --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { Repo } from '../../../shared/repo-types' +import type { SshTarget } from '../../../shared/ssh-types' +import { worktreeWorkspaceKey } from '../../../shared/workspace-scope' +import { collectOrcadMigrationUntransferredDependencyCensus } from './orcad-source-dependency-census' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 8 +} + +const REPO: Repo = { + id: 'repo-1', + path: '/srv/repo', + displayName: 'Repository', + badgeColor: '#737373', + addedAt: 1, + connectionId: TARGET.id, + executionHostId: `ssh:${TARGET.id}`, + projectGroupId: 'group-1' +} + +const FOLDER: FolderWorkspace = { + id: 'folder-1', + projectGroupId: 'group-1', + name: 'Folder', + folderPath: '/srv/folder', + connectionId: TARGET.id, + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 1, + createdAt: 1, + updatedAt: 1 +} + +const MANIFEST: OrcadMigrationManifest = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-08-30T12:00:00.000Z', + source: { + sshTargetId: TARGET.id, + sshTargetGeneration: TARGET.generation ?? null, + targetLabel: TARGET.label + }, + payload: { + repositories: [REPO], + projectGroups: [], + folderWorkspaces: [FOLDER] + }, + manifestSha256: 'a'.repeat(64) +} + +describe('orcad migration source dependency census', () => { + it('ignores client focus on another host copied into the source host partition', () => { + const state = getDefaultPersistedState('/home/test') + state.sshTargets = [TARGET] + state.repos = [REPO] + state.folderWorkspaces = [FOLDER] + const localWorktreeId = 'local-repo::/home/test/local' + state.workspaceSession = { ...state.workspaceSession, activeWorktreeId: localWorktreeId } + state.workspaceSessionsByHostId = { + [`ssh:${TARGET.id}`]: { + ...state.workspaceSession, + activeWorktreeId: localWorktreeId, + activeWorkspaceKey: worktreeWorkspaceKey(localWorktreeId) + } + } + + expect(collectOrcadMigrationUntransferredDependencyCensus(state, MANIFEST)).toMatchObject({ + totalCount: 0 + }) + }) + + it('allows a static catalog with no unrepresented dependent state', () => { + const state = getDefaultPersistedState('/home/test') + state.sshTargets = [TARGET] + state.repos = [REPO] + state.folderWorkspaces = [FOLDER] + + expect(collectOrcadMigrationUntransferredDependencyCensus(state, MANIFEST)).toMatchObject({ + totalCount: 0 + }) + }) + + it('keeps another SSH target and its host partition outside the source fence', () => { + const state = getDefaultPersistedState('/home/test') + state.sshTargets = [TARGET, { ...TARGET, id: 'ssh-other', generation: 9 }] + state.workspaceSessionsByHostId = { + 'ssh:ssh-other': { + ...state.workspaceSession, + tabsByWorktree: { + 'other-repo::/srv/worktree': [ + { + id: 'tab-other', + ptyId: 'pty-other', + worktreeId: 'other-repo::/srv/worktree', + title: 'Other', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + } + } + } + + expect(collectOrcadMigrationUntransferredDependencyCensus(state, MANIFEST).totalCount).toBe(0) + }) + + it('counts a lease that is not terminated as live terminal work', () => { + const state = getDefaultPersistedState('/home/test') + state.sshTargets = [TARGET] + state.repos = [REPO] + state.sshRemotePtyLeases = [ + { targetId: TARGET.id, ptyId: 'pty-1', state: 'detached', createdAt: 1, updatedAt: 1 }, + { targetId: TARGET.id, ptyId: 'pty-2', state: 'terminated', createdAt: 1, updatedAt: 2 } + ] + + expect(collectOrcadMigrationUntransferredDependencyCensus(state, MANIFEST)).toMatchObject({ + totalCount: 1, + counts: { 'terminal-lease': 1 } + }) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.ts new file mode 100644 index 00000000000..ec5293b3c60 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-dependency-census.ts @@ -0,0 +1,84 @@ +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { + ORCAD_MIGRATION_DEPENDENCY_KINDS, + type OrcadMigrationDependencyKind +} from '../../../shared/orcad-migration-preflight' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { TerminalScrollbackSnapshotStorage } from '../../terminal-scrollback-snapshots' +import { + inspectOrcadMigrationSourceSessions, + type OrcadMigrationSourceSessionInspection +} from './orcad-source-session-dependencies' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + orcadMigrationPartitionScope, + type OrcadMigrationSourceScope +} from './orcad-source-scope' +import { paneBelongsToTabs } from '../../../shared/workspace-session-pane-ownership' + +export type OrcadMigrationSourceDependencyCensus = { + totalCount: number + counts: Record +} + +/** What still references the target that this manifest cannot carry. */ +export function collectOrcadMigrationUntransferredDependencyCensus( + state: PersistedState, + manifest: OrcadMigrationManifest, + storage?: TerminalScrollbackSnapshotStorage +): OrcadMigrationSourceDependencyCensus { + const scope = createOrcadMigrationSourceScope({ + source: manifest.source, + catalog: manifest.payload, + repos: state.repos + }) + const sessions = inspectOrcadMigrationSourceSessions(state, { + hostId: scope.hostId, + ownerMatches: (ownerKey, partitionHostId) => + orcadMigrationOwnerMatchesScope( + ownerKey, + orcadMigrationPartitionScope(scope, partitionHostId) + ) + }) + const dormant = collectOrcadMigrationSourceDormantState( + state, + manifest.source, + manifest.payload, + storage, + manifest.destinationEnvironmentId + ) + const counts: Record = { + ...dormant.blockedCounts, + 'terminal-lease': state.sshRemotePtyLeases.filter( + (lease) => lease.targetId === scope.targetId && lease.state !== 'terminated' + ).length, + 'terminal-recovery': countTerminalRecoveryState(state, scope, sessions) + } + return { + counts, + totalCount: ORCAD_MIGRATION_DEPENDENCY_KINDS.reduce((total, kind) => total + counts[kind], 0) + } +} + +// Consumer recoveries never block: the terminal gate proves before every move that their leases exited. +function countTerminalRecoveryState( + state: PersistedState, + scope: OrcadMigrationSourceScope, + sessions: OrcadMigrationSourceSessionInspection +): number { + const unsupported = state.migrationUnsupportedPtyEntries.filter( + (entry) => + orcadMigrationOwnerMatchesScope(entry.worktreeId, scope) || + (entry.tabId ? sessions.tabIds.has(entry.tabId) : false) || + sessions.ptyIds.has(entry.ptyId) || + (entry.paneKey ? paneBelongsToTabs(entry.paneKey, sessions.tabIds) : false) + ).length + const aliases = state.legacyPaneKeyAliasEntries.filter( + (entry) => + paneBelongsToTabs(entry.legacyPaneKey, sessions.tabIds) || + paneBelongsToTabs(entry.stablePaneKey, sessions.tabIds) + ).length + return unsupported + aliases +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-state.ts new file mode 100644 index 00000000000..f650035903d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-state.ts @@ -0,0 +1,252 @@ +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationDormantStatePayload, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import type { OrcadMigrationDependencyKind } from '../../../shared/orcad-migration-preflight' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { TerminalScrollbackSnapshotStorage } from '../../terminal-scrollback-snapshots' +import { projectHostSetupProjectionFromRepos } from '../../../shared/project-host-setup-projection' +import { isEmptyRetiredNameRegistry } from '../../../shared/worktree/retired-name-registry' +import { getRepoIdFromWorktreeId } from '../../../shared/worktree/id' +import { toOrcadDestinationRepository } from './orcad-destination-catalog-projection' +import { collectOrcadMigrationRetiredWorktreeNamespaces } from './orcad-source-retired-worktree-names' +import { collectOrcadMigrationSourceAutomationState } from './orcad-source-automation-state' +import { collectOrcadMigrationSourceWorkspaceSession } from './orcad-source-workspace-session' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + unqualifyOrcadMigrationOwnerKey +} from './orcad-source-scope' +import { collectOrcadMigrationSourceClientState } from './orcad-source-client-state' +import { inspectOrcadSourceWorktreeMetadata } from './orcad-source-worktree-metadata' +import { compareKeys } from './orcad-source-key-order' + +export const ORCAD_MIGRATION_TRANSFERRED_DORMANT_KINDS = [ + 'worktree-metadata', + 'worktree-lineage', + 'workspace-lineage', + 'workspace-session', + 'automation', + 'automation-run', + 'sparse-preset', + 'retired-worktree-name', + 'mobile-tab-selection', + 'ui-routing', + 'saved-port-forward' +] as const satisfies readonly OrcadMigrationDependencyKind[] + +type TransferredDormantKind = (typeof ORCAD_MIGRATION_TRANSFERRED_DORMANT_KINDS)[number] + +export type OrcadMigrationSourceDormantInspection = { + payload: OrcadMigrationDormantStatePayload + blockedCounts: Record +} + +export function collectOrcadMigrationSourceDormantState( + state: PersistedState, + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + storage?: TerminalScrollbackSnapshotStorage, + destinationEnvironmentId?: string +): OrcadMigrationSourceDormantInspection { + const scope = createOrcadMigrationSourceScope({ source, catalog, repos: state.repos }) + const blockedCounts = emptyBlockedCounts() + const destinationRepos = catalog.repositories.map(toOrcadDestinationRepository) + const setupByRepoId = new Map( + projectHostSetupProjectionFromRepos(destinationRepos).setups.flatMap((setup) => + setup.repoId ? [[setup.repoId, setup] as const] : [] + ) + ) + const metadata = inspectOrcadSourceWorktreeMetadata(state, scope) + blockedCounts['worktree-metadata'] = metadata.blockedCount + const worktreeMeta = uniqueDestinationRows( + metadata.rows.flatMap(({ sourceKey, meta }) => { + const worktreeId = unqualifyOrcadMigrationOwnerKey(sourceKey) + const setup = setupByRepoId.get(getRepoIdFromWorktreeId(worktreeId)) + return [ + { + sourceKey, + worktreeId, + meta: { + ...structuredClone(meta), + ...(setup ? { projectId: setup.projectId, projectHostSetupId: setup.id } : {}), + hostId: 'local' as const + } + } + ] + }), + (entry) => entry.worktreeId, + () => (blockedCounts['worktree-metadata'] += 1) + ) + const worktreeLineage = uniqueDestinationRows( + Object.entries(state.worktreeLineageById).flatMap(([sourceKey, lineage]) => { + const touches = [sourceKey, lineage.worktreeId, lineage.parentWorktreeId].some((value) => + orcadMigrationOwnerMatchesScope(value, scope) + ) + if (!touches) { + return [] + } + const worktreeId = unqualifyOrcadMigrationOwnerKey(sourceKey) + if ( + worktreeId !== lineage.worktreeId || + !orcadMigrationOwnerMatchesScope(lineage.worktreeId, scope) || + !orcadMigrationOwnerMatchesScope(lineage.parentWorktreeId, scope) + ) { + blockedCounts['worktree-lineage'] += 1 + return [] + } + return [{ sourceKey, worktreeId, lineage: structuredClone(lineage) }] + }), + (entry) => entry.worktreeId, + () => (blockedCounts['worktree-lineage'] += 1) + ) + const workspaceLineage = uniqueDestinationRows( + Object.entries(state.workspaceLineageByChildKey).flatMap(([sourceKey, lineage]) => { + const touches = [sourceKey, lineage.childWorkspaceKey, lineage.parentWorkspaceKey].some( + (value) => orcadMigrationOwnerMatchesScope(value, scope) + ) + if (!touches) { + return [] + } + const childWorkspaceKey = unqualifyOrcadMigrationOwnerKey(sourceKey) + if ( + childWorkspaceKey !== lineage.childWorkspaceKey || + !orcadMigrationOwnerMatchesScope(lineage.childWorkspaceKey, scope) || + !orcadMigrationOwnerMatchesScope(lineage.parentWorkspaceKey, scope) + ) { + blockedCounts['workspace-lineage'] += 1 + return [] + } + return [ + { + sourceKey, + childWorkspaceKey, + lineage: { + ...structuredClone(lineage), + childInstanceId: lineage.childInstanceId ?? null, + parentInstanceId: lineage.parentInstanceId ?? null + } + } + ] + }), + (entry) => entry.childWorkspaceKey, + () => (blockedCounts['workspace-lineage'] += 1) + ) + // A repo id another host shares keys one registry for both hosts: it stays where it is. + const ownedRepoIds = [...scope.repoIds].filter((repoId) => !scope.sharedRepoIds.has(repoId)) + const sparsePresets = ownedRepoIds + .flatMap((repoId) => state.sparsePresetsByRepo[repoId] ?? []) + .map((preset) => structuredClone(preset)) + .sort((left, right) => + compareKeys(`${left.repoId}\0${left.id}`, `${right.repoId}\0${right.id}`) + ) + const retiredWorktreeNames = ownedRepoIds + .flatMap((repoId) => { + const registry = state.retiredWorktreeNamesByRepo?.[repoId] + return registry && !isEmptyRetiredNameRegistry(registry) + ? [{ repoId, registry: structuredClone(registry) }] + : [] + }) + .sort((left, right) => compareKeys(left.repoId, right.repoId)) + const workspaceSession = collectOrcadMigrationSourceWorkspaceSession( + state, + source, + catalog, + storage + ) + blockedCounts['workspace-session'] = workspaceSession.blockedCount + const automationState = collectOrcadMigrationSourceAutomationState(state, source, catalog) + blockedCounts.automation = automationState.blockedAutomationCount + blockedCounts['automation-run'] = automationState.blockedRunCount + const clientState = collectOrcadMigrationSourceClientState( + state, + source, + catalog, + destinationEnvironmentId, + workspaceSession.payload + ) + blockedCounts['mobile-tab-selection'] = clientState.blockedCounts['mobile-tab-selection'] + blockedCounts['ui-routing'] = clientState.blockedCounts['ui-routing'] + blockedCounts['saved-port-forward'] = clientState.blockedCounts['saved-port-forward'] + blockedCounts['workspace-session'] += clientState.blockedCount + return { + payload: { + version: 1, + worktreeMeta: worktreeMeta.sort((left, right) => + compareKeys(left.worktreeId, right.worktreeId) + ), + worktreeLineage: worktreeLineage.sort((left, right) => + compareKeys(left.worktreeId, right.worktreeId) + ), + workspaceLineage: workspaceLineage.sort((left, right) => + compareKeys(left.childWorkspaceKey, right.childWorkspaceKey) + ), + sparsePresets, + retiredWorktreeNames, + retiredWorktreeNamespaces: collectOrcadMigrationRetiredWorktreeNamespaces(state, catalog), + ...(workspaceSession.payload ? { workspaceSession: workspaceSession.payload } : {}), + ...(workspaceSession.snapshots.length > 0 + ? { terminalScrollbackSnapshots: workspaceSession.snapshots } + : {}), + ...(automationState.automations.length > 0 + ? { automations: automationState.automations } + : {}), + ...(automationState.automationRuns.length > 0 + ? { automationRuns: automationState.automationRuns } + : {}), + ...(clientState.payload ? { clientState: clientState.payload } : {}) + }, + blockedCounts + } +} + +export function emptyDormantPayload(): OrcadMigrationDormantStatePayload { + return { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [] + } +} + +function uniqueDestinationRows( + rows: T[], + key: (row: T) => string, + onDuplicate: () => void +): T[] { + const unique = new Map() + const duplicates = new Set() + for (const row of rows) { + const rowKey = key(row) + if (duplicates.has(rowKey)) { + onDuplicate() + } else if (unique.has(rowKey)) { + unique.delete(rowKey) + duplicates.add(rowKey) + onDuplicate() + } else { + unique.set(rowKey, row) + } + } + return [...unique.values()] +} + +function emptyBlockedCounts(): Record { + return { + 'worktree-metadata': 0, + 'worktree-lineage': 0, + 'workspace-lineage': 0, + 'workspace-session': 0, + automation: 0, + 'automation-run': 0, + 'sparse-preset': 0, + 'retired-worktree-name': 0, + 'mobile-tab-selection': 0, + 'ui-routing': 0, + 'saved-port-forward': 0 + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-subtraction.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-subtraction.ts new file mode 100644 index 00000000000..131e3f2fc64 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-dormant-subtraction.ts @@ -0,0 +1,40 @@ +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { subtractOrcadSourceWorktreeMetadata } from './orcad-source-worktree-metadata' +import { subtractOrcadMigrationSourceAutomationState } from './orcad-source-automation-subtraction' +import { subtractOrcadMigrationSourceWorkspaceSession } from './orcad-source-workspace-session-subtraction' +import { subtractOrcadMigrationClientState } from './orcad-source-client-subtraction' + +/** On a copy of the profile only: what an earlier migration moved, gone from the delta view. */ +export function subtractOrcadMigrationSourceDormantState( + state: PersistedState, + manifest: OrcadMigrationManifest +): void { + // By scope, even with no dormant state: a downgraded build may have added some since. + subtractOrcadSourceWorktreeMetadata(state, manifest) + subtractOrcadMigrationSourceAutomationState(state, manifest) + const dormant = manifest.payload.dormantState + if (!dormant) { + return + } + dormant.worktreeLineage.forEach((entry) => delete state.worktreeLineageById[entry.sourceKey]) + dormant.workspaceLineage.forEach( + (entry) => delete state.workspaceLineageByChildKey[entry.sourceKey] + ) + for (const preset of dormant.sparsePresets) { + const remaining = (state.sparsePresetsByRepo[preset.repoId] ?? []).filter( + (entry) => entry.id !== preset.id + ) + if (remaining.length > 0) { + state.sparsePresetsByRepo[preset.repoId] = remaining + } else { + delete state.sparsePresetsByRepo[preset.repoId] + } + } + for (const entry of dormant.retiredWorktreeNames) { + delete state.retiredWorktreeNamesByRepo?.[entry.repoId] + } + subtractOrcadMigrationSourceWorkspaceSession(state, manifest) + subtractOrcadMigrationClientState(state, manifest) + // Retain snapshot files: the prior durable profile and rollback evidence can still reference them. +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-export.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-export.test.ts new file mode 100644 index 00000000000..6ad39576d5f --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-export.test.ts @@ -0,0 +1,181 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../../shared/constants' +import { toSshExecutionHostId } from '../../../shared/execution-host' +import type { SshTarget } from '../../../shared/ssh-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { closeTestStores, createSqliteTestStore } from '../../persistence-test-harness' +import { Store } from '../loading-store/store' +import { createOrcadMigrationManifest } from '../../ssh/orcad-migration-manifest-export' +import { + getProfileTerminalScrollbackSnapshotRoot, + readTerminalScrollbackStoredBytesSync, + writeTerminalScrollbackSnapshotSync +} from '../../terminal-scrollback-snapshots' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 3 +} +const REPO_ID = 'repo-1' +const WORKTREE_ID = `${REPO_ID}::/srv/app` + +const directories: string[] = [] +const dataFiles: string[] = [] +afterEach(async () => { + await closeTestStores() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function dormantSession(buffer: string): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [WORKTREE_ID]: [ + { + id: 'tab-1', + ptyId: null, + worktreeId: WORKTREE_ID, + title: 'Shell', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: 'leaf-1' }, + activeLeafId: 'leaf-1', + expandedLeafId: null, + buffersByLeafId: { 'leaf-1': buffer } + } + } + } +} + +function sourceStore(): Store { + const directory = mkdtempSync(join(tmpdir(), 'orcad-source-export-')) + directories.push(directory) + dataFiles.push(join(directory, 'orca-data.json')) + const store = createSqliteTestStore(Store, { dataFile: dataFiles.at(-1)! }) + store.addSshTarget(TARGET) + store.addRepo({ + id: REPO_ID, + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + store.setWorkspaceSession(dormantSession('dormant output\r\n'), toSshExecutionHostId(TARGET.id)) + return store +} + +describe('exporting a relay-hosted SSH target from the profile store', () => { + it('reads the target catalog and dormant scrollback without changing the source', () => { + const store = sourceStore() + const before = JSON.stringify({ + repos: store.getRepos(), + session: store.getWorkspaceSession(toSshExecutionHostId(TARGET.id)) + }) + + const manifest = createOrcadMigrationManifest(store, TARGET) + + expect(manifest.payload.repositories.map((repo) => repo.id)).toEqual([REPO_ID]) + const snapshots = manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + expect(snapshots).toHaveLength(1) + const chunk = store.readOrcadMigrationSourceSnapshotChunk(manifest, snapshots[0]!.ref, 0) + expect(Buffer.from(chunk.bytesBase64, 'base64').toString('utf8')).toBe('dormant output\r\n') + expect(chunk.eof).toBe(true) + expect( + JSON.stringify({ + repos: store.getRepos(), + session: store.getWorkspaceSession(toSshExecutionHostId(TARGET.id)) + }) + ).toBe(before) + }) + + it('refuses a chunk once the dormant buffer changed after export', () => { + const store = sourceStore() + const manifest = createOrcadMigrationManifest(store, TARGET) + const ref = manifest.payload.dormantState?.terminalScrollbackSnapshots?.[0]?.ref ?? '' + store.setWorkspaceSession(dormantSession('rewritten output'), toSshExecutionHostId(TARGET.id)) + expect(() => store.readOrcadMigrationSourceSnapshotChunk(manifest, ref, 0)).toThrow( + 'orcad_migration_source_snapshot_changed' + ) + }) + + it('finishes a retained transfer after the tab closes, then deletes the orphaned file', () => { + const store = sourceStore() + const hostId = toSshExecutionHostId(TARGET.id) + const storage = { snapshotRoot: getProfileTerminalScrollbackSnapshotRoot(dataFiles.at(-1)!) } + const stored = writeTerminalScrollbackSnapshotSync({ + tabId: 'tab-1', + leafId: 'leaf-1', + buffer: 'dormant output\r\n', + storage + })! + const session = dormantSession('') + session.terminalLayoutsByTabId['tab-1'] = { + ...session.terminalLayoutsByTabId['tab-1']!, + buffersByLeafId: {}, + scrollbackRefsByLeafId: { 'leaf-1': stored } + } + store.setWorkspaceSession(session, hostId) + const manifest = createOrcadMigrationManifest(store, TARGET) + const ref = manifest.payload.dormantState?.terminalScrollbackSnapshots?.[0]?.ref ?? '' + expect(ref).toBe(stored) + store.syncOrcadMigrationScrollbackRetention([manifest]) + store.setWorkspaceSession(getDefaultWorkspaceSession(), hostId) + + const chunk = store.readOrcadMigrationSourceSnapshotChunk(manifest, ref, 0) + expect(Buffer.from(chunk.bytesBase64, 'base64').toString('utf8')).toBe('dormant output\r\n') + store.syncOrcadMigrationScrollbackRetention([]) + expect(readTerminalScrollbackStoredBytesSync(ref, storage)).toBeNull() + }) + + it('serves an inline dormant buffer after its tab closes, across retries, until released', () => { + const store = sourceStore() + const hostId = toSshExecutionHostId(TARGET.id) + const storage = { snapshotRoot: getProfileTerminalScrollbackSnapshotRoot(dataFiles.at(-1)!) } + const manifest = createOrcadMigrationManifest(store, TARGET) + const ref = manifest.payload.dormantState?.terminalScrollbackSnapshots?.[0]?.ref ?? '' + store.syncOrcadMigrationScrollbackRetention([manifest]) + store.setWorkspaceSession(getDefaultWorkspaceSession(), hostId) + + for (let attempt = 0; attempt < 2; attempt += 1) { + store.syncOrcadMigrationScrollbackRetention([manifest]) + const chunk = store.readOrcadMigrationSourceSnapshotChunk(manifest, ref, 0) + expect(Buffer.from(chunk.bytesBase64, 'base64').toString('utf8')).toBe('dormant output\r\n') + } + store.syncOrcadMigrationScrollbackRetention([]) + expect(readTerminalScrollbackStoredBytesSync(ref, storage)).toBeNull() + }) + + it('refuses a chunk for a manifest whose digest does not match its contents', () => { + const store = sourceStore() + const manifest = createOrcadMigrationManifest(store, TARGET) + const ref = manifest.payload.dormantState?.terminalScrollbackSnapshots?.[0]?.ref ?? '' + expect(() => + store.readOrcadMigrationSourceSnapshotChunk({ ...manifest, migrationId: 'forged' }, ref, 0) + ).toThrow('orcad_migration_manifest_digest_mismatch') + }) + + it('names what still blocks: nothing, once the dormant state is exportable', () => { + const store = sourceStore() + const manifest = createOrcadMigrationManifest(store, TARGET) + const census = store.inspectOrcadMigrationUntransferredDependencies(manifest) + expect(census.counts['workspace-session']).toBe(0) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-export.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-export.ts new file mode 100644 index 00000000000..d5649036c2d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-export.ts @@ -0,0 +1,118 @@ +/** + * The Store's read-only view of a relay-hosted SSH target, for migrating it to a managed orcad. + * + * Everything here reads the profile-state store and returns copies; nothing writes or deletes + * source rows, and nothing else deletes them after an import either. + */ +import { + ORCAD_MIGRATION_SCROLLBACK_CHUNK_BYTES, + type OrcadMigrationTerminalScrollbackSnapshot +} from '../../../shared/orcad-migration-scrollback' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationDormantStatePayload, + OrcadMigrationManifest, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import { assertOrcadMigrationManifestDigest } from '../../orcad/orcad-migration-manifest-digest' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import { + collectOrcadMigrationUntransferredDependencyCensus, + type OrcadMigrationSourceDependencyCensus +} from './orcad-source-dependency-census' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' +import { readOrcadMigrationSourceScrollbackChunk } from './orcad-source-scrollback-state' +import { + createOrcadMigrationDeltaView, + type OrcadMigrationDeltaView +} from './orcad-source-delta-view' +import { syncOrcadMigrationScrollbackRetention } from './orcad-source-scrollback-retention' + +type OrcadSourceExportRuntime = Pick< + StoreRuntimeState, + 'state' | 'terminalScrollbackSnapshotStorage' | 'retainedScrollbackRefsByMigrationId' +> + +const orcadSourceExportContext = Symbol('OrcadSourceExportPersistence') + +export class OrcadSourceExportPersistence { + readonly [orcadSourceExportContext]: OrcadSourceExportRuntime + + constructor(runtime: OrcadSourceExportRuntime) { + this[orcadSourceExportContext] = runtime + } + + collectOrcadMigrationSourceDormantState( + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + destinationEnvironmentId?: string + ): OrcadMigrationDormantStatePayload { + const runtime = this[orcadSourceExportContext] + return collectOrcadMigrationSourceDormantState( + runtime.state, + source, + catalog, + runtime.terminalScrollbackSnapshotStorage, + destinationEnvironmentId + ).payload + } + + /** A copy of the source with `moved` subtracted from it, for a delta move. */ + createOrcadMigrationDeltaView(moved: OrcadMigrationManifest): OrcadMigrationDeltaView { + return createOrcadMigrationDeltaView(this[orcadSourceExportContext], moved) + } + + /** What still references the target that this manifest cannot carry. */ + inspectOrcadMigrationUntransferredDependencies( + manifest: OrcadMigrationManifest + ): OrcadMigrationSourceDependencyCensus { + const runtime = this[orcadSourceExportContext] + return collectOrcadMigrationUntransferredDependencyCensus( + runtime.state, + manifest, + runtime.terminalScrollbackSnapshotStorage + ) + } + + /** Holds the snapshots unfinished migrations name; see syncOrcadMigrationScrollbackRetention. */ + syncOrcadMigrationScrollbackRetention(pending: readonly OrcadMigrationManifest[]): void { + syncOrcadMigrationScrollbackRetention(this[orcadSourceExportContext], pending) + } + + /** + * One bounded chunk of a scrollback snapshot the signed manifest names. The bytes are checked + * against the manifest's length and digest, so a buffer that changed since export is refused. + */ + readOrcadMigrationSourceSnapshotChunk( + manifest: OrcadMigrationManifest, + ref: string, + offset: number + ): { bytesBase64: string; totalBytes: number; eof: boolean } { + assertOrcadMigrationManifestDigest(manifest) + const descriptor: OrcadMigrationTerminalScrollbackSnapshot | undefined = + manifest.payload.dormantState?.terminalScrollbackSnapshots?.find((entry) => entry.ref === ref) + if (!descriptor) { + throw new Error('orcad_migration_source_snapshot_unknown') + } + const runtime = this[orcadSourceExportContext] + return readOrcadMigrationSourceScrollbackChunk({ + state: runtime.state, + descriptor, + retained: [...runtime.retainedScrollbackRefsByMigrationId.values()].some((refs) => + refs.has(ref) + ), + offset, + length: ORCAD_MIGRATION_SCROLLBACK_CHUNK_BYTES, + storage: runtime.terminalScrollbackSnapshotStorage + }) + } +} + +export function installOrcadSourceExportPersistenceContext( + target: OrcadSourceExportPersistence, + source: OrcadSourceExportPersistence +): void { + Object.defineProperty(target, orcadSourceExportContext, { + value: source[orcadSourceExportContext] + }) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-host-qualified-scope.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-host-qualified-scope.test.ts new file mode 100644 index 00000000000..425b987d596 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-host-qualified-scope.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { Repo } from '../../../shared/repo-types' +import type { Tab } from '../../../shared/tab-types' +import { subtractOrcadMigrationClientState } from './orcad-source-client-subtraction' +import { collectOrcadMigrationUntransferredDependencyCensus } from './orcad-source-dependency-census' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' +import { subtractOrcadMigrationSourceDormantState } from './orcad-source-dormant-subtraction' +import { + orcadMigrationOwnerMatchesScope, + createOrcadMigrationSourceScope +} from './orcad-source-scope' + +// Two SSH hosts registered the same repository id; only host A converts. +const REPO: Repo = { + id: 'repo-1', + path: '/srv/repo', + displayName: 'Repository', + badgeColor: '#737373', + addedAt: 1, + connectionId: 'host-a' +} +const WORKTREE = `${REPO.id}::/srv/repo` +const HOST_B = 'ssh:host-b' + +function tab(id: string): Tab { + return { + id, + entityId: `/srv/repo/${id}.md`, + groupId: `group-${id}`, + worktreeId: WORKTREE, + contentType: 'editor', + label: id, + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } +} + +function manifest(state: PersistedState): OrcadMigrationManifest { + const source = { sshTargetId: 'host-a', sshTargetGeneration: null, targetLabel: 'A' } + const payload = { repositories: [REPO], projectGroups: [], folderWorkspaces: [] } + return { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-10-05T12:00:00.000Z', + source, + payload: { + ...payload, + dormantState: collectOrcadMigrationSourceDormantState( + state, + source, + payload, + undefined, + 'env-1' + ).payload + }, + destinationEnvironmentId: 'env-1', + manifestSha256: 'a'.repeat(64) + } +} + +function twoHostState(): PersistedState { + const state = getDefaultPersistedState('/home/test') + state.repos = [REPO, { ...REPO, connectionId: 'host-b' }] + state.workspaceSessionsByHostId = { + 'ssh:host-a': { ...state.workspaceSession, unifiedTabs: { [WORKTREE]: [tab('a')] } }, + [HOST_B]: { ...state.workspaceSession, unifiedTabs: { [WORKTREE]: [tab('b')] } } + } + // Host B's row, qualified, in the local partition. + state.workspaceSession.unifiedTabs = { [`${HOST_B}|${WORKTREE}`]: [tab('b-local')] } + state.worktreeMeta[`${HOST_B}|${WORKTREE}`] = { + displayName: 'B worktree', + comment: '', + isUnread: false, + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + isArchived: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 1, + hostId: HOST_B + } + return state +} + +describe('migration scope with host-qualified owners', () => { + const scope = createOrcadMigrationSourceScope({ + source: { sshTargetId: 'host-a', sshTargetGeneration: null, targetLabel: 'A' }, + catalog: { repositories: [REPO], projectGroups: [], folderWorkspaces: [] }, + repos: [REPO] + }) + + it('owns only keys qualified with its own host', () => { + expect(orcadMigrationOwnerMatchesScope(`ssh:host-a|${WORKTREE}`, scope)).toBe(true) + expect(orcadMigrationOwnerMatchesScope(`${HOST_B}|${WORKTREE}`, scope)).toBe(false) + expect(orcadMigrationOwnerMatchesScope(`runtime:env-1|${WORKTREE}`, scope)).toBe(false) + expect(orcadMigrationOwnerMatchesScope(WORKTREE, scope)).toBe(true) + }) + + it("never moves, counts or retires another host's state that shares a repo id", () => { + const state = twoHostState() + const moved = manifest(state) + const session = moved.payload.dormantState?.workspaceSession + expect( + Object.values(session?.unifiedTabs ?? {}) + .flat() + .map((entry) => entry.id) + ).toEqual(['a']) + expect( + collectOrcadMigrationUntransferredDependencyCensus(state, moved).counts['workspace-session'] + ).toBe(0) + + subtractOrcadMigrationSourceDormantState(state, moved) + + expect(state.workspaceSessionsByHostId?.[HOST_B]?.unifiedTabs?.[WORKTREE]).toHaveLength(1) + expect(state.workspaceSession.unifiedTabs?.[`${HOST_B}|${WORKTREE}`]).toHaveLength(1) + expect(state.worktreeMeta[`${HOST_B}|${WORKTREE}`]).toBeDefined() + expect(state.workspaceSessionsByHostId?.['ssh:host-a']?.unifiedTabs?.[WORKTREE]).toBeUndefined() + }) + + it('subtracts selected-worktree routing by retargeting it to the destination', () => { + const state = getDefaultPersistedState('/home/test') + state.repos = [REPO] + state.ui.lastActiveWorktreeId = WORKTREE + const moved = manifest(state) + expect(moved.payload.dormantState?.clientState?.uiRouting?.lastActiveWorktreeId).toBe(WORKTREE) + + subtractOrcadMigrationClientState(state, moved) + + expect(state.ui.lastActiveWorktreeId).toBe(`runtime:env-1|${WORKTREE}`) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-key-order.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-key-order.ts new file mode 100644 index 00000000000..e72a93b850c --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-key-order.ts @@ -0,0 +1,4 @@ +/** Code-unit key order, so every source export sorts identically on every host. */ +export function compareKeys(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-legacy-key-scope.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-legacy-key-scope.test.ts new file mode 100644 index 00000000000..f66bf207c9e --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-legacy-key-scope.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { Repo } from '../../../shared/repo-types' +import type { WorktreeMeta } from '../../../shared/worktree/meta-types' +import { DORMANT_AUTOMATION } from '../../persistence-orcad-migration-catalog-fixture' +import { subtractOrcadMigrationClientState } from './orcad-source-client-subtraction' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' +import { subtractOrcadMigrationSourceDormantState } from './orcad-source-dormant-subtraction' +import { createOrcadMigrationSourceScope } from './orcad-source-scope' +import { inspectOrcadSourceWorktreeMetadata } from './orcad-source-worktree-metadata' + +// Host A converts; host B registered the same repository id. Every key below is legacy, unqualified. +const REPO_A: Repo = { + id: 'repo-1', + path: '/srv/repo', + displayName: 'Repository', + badgeColor: '#737373', + addedAt: 1, + connectionId: 'host-a' +} +const REPO_B: Repo = { ...REPO_A, connectionId: 'host-b' } +const WORKTREE = `${REPO_A.id}::/srv/repo` +const SOURCE = { sshTargetId: 'host-a', sshTargetGeneration: 3, targetLabel: 'A' } + +function meta(hostId?: WorktreeMeta['hostId']): WorktreeMeta { + return { + displayName: 'worktree', + comment: '', + isUnread: false, + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + isArchived: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 1, + ...(hostId ? { hostId } : {}) + } +} + +function manifest(state: PersistedState): OrcadMigrationManifest { + const payload = { repositories: [REPO_A], projectGroups: [], folderWorkspaces: [] } + const dormant = collectOrcadMigrationSourceDormantState(state, SOURCE, payload, undefined, 'e1') + return { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-10-05T12:00:00.000Z', + source: SOURCE, + payload: { ...payload, dormantState: dormant.payload }, + destinationEnvironmentId: 'e1', + manifestSha256: 'a'.repeat(64) + } +} + +/** Host B's state in every store that is not a session partition, keyed by legacy repo ids. */ +function hostBLegacyState(): PersistedState { + const state = getDefaultPersistedState('/home/test') + state.repos = [REPO_A, REPO_B] + state.worktreeMeta[WORKTREE] = meta('ssh:host-b') + state.worktreeMeta[`${REPO_A.id}::/srv/repo-unmarked`] = meta() + state.automations = [ + { + ...DORMANT_AUTOMATION, + id: 'automation-b', + runContext: { ...DORMANT_AUTOMATION.runContext!, hostId: 'ssh:host-b', repoId: REPO_A.id }, + projectId: REPO_A.id, + executionTargetType: 'ssh', + executionTargetId: 'host-b', + executionTargetGeneration: SOURCE.sshTargetGeneration + } + ] + state.worktreeLineageById[WORKTREE] = { + worktreeId: WORKTREE, + worktreeInstanceId: 'i-1', + parentWorktreeId: `${REPO_A.id}::/srv/parent`, + parentWorktreeInstanceId: 'i-0', + origin: 'manual', + capture: { source: 'manual-action', confidence: 'explicit' }, + createdAt: 1 + } + state.sparsePresetsByRepo[REPO_A.id] = [ + { + id: 'preset-b', + repoId: REPO_A.id, + name: 'B', + directories: ['src'], + createdAt: 1, + updatedAt: 1 + } + ] + state.ui.lastActiveRepoId = REPO_A.id + state.ui.lastActiveWorktreeId = WORKTREE + state.ui.filterRepoIds = [REPO_A.id] + state.ui.showDotfilesByWorktree = { [WORKTREE]: true } + return state +} + +describe('legacy unqualified keys for a repo id two hosts share', () => { + it('cannot attribute them to the converting host', () => { + const state = hostBLegacyState() + const scope = createOrcadMigrationSourceScope({ + source: SOURCE, + catalog: { repositories: [REPO_A], projectGroups: [], folderWorkspaces: [] }, + repos: state.repos + }) + expect([...scope.sharedRepoIds]).toEqual([REPO_A.id]) + expect(inspectOrcadSourceWorktreeMetadata(state, scope)).toEqual({ rows: [], blockedCount: 0 }) + }) + + it("never moves, counts or retires host B's rows", () => { + const state = hostBLegacyState() + const before = structuredClone(state) + const moved = manifest(state) + const dormant = moved.payload.dormantState + expect(dormant?.worktreeMeta ?? []).toEqual([]) + expect(dormant?.worktreeLineage ?? []).toEqual([]) + expect(dormant?.sparsePresets ?? []).toEqual([]) + expect(dormant?.automations ?? []).toEqual([]) + expect(dormant?.clientState?.uiRouting ?? {}).toEqual({}) + expect( + Object.values( + collectOrcadMigrationSourceDormantState(state, SOURCE, moved.payload, undefined, 'e1') + .blockedCounts + ).every((count) => count === 0) + ).toBe(true) + + subtractOrcadMigrationSourceDormantState(state, moved) + subtractOrcadMigrationClientState(state, moved) + + expect(state.worktreeMeta).toEqual(before.worktreeMeta) + expect(state.automations).toEqual(before.automations) + expect(state.worktreeLineageById).toEqual(before.worktreeLineageById) + expect(state.sparsePresetsByRepo).toEqual(before.sparsePresetsByRepo) + expect(state.ui.lastActiveRepoId).toBe(REPO_A.id) + expect(state.ui.filterRepoIds).toEqual([REPO_A.id]) + expect(state.ui.showDotfilesByWorktree).toEqual({ [WORKTREE]: true }) + }) +}) + +describe('an identity alias whose metadata is gone (the B4 profile shape)', () => { + it('is nothing to move, and retirement drops it', () => { + const state = getDefaultPersistedState('/home/test') + state.repos = [REPO_A] + // As saved on B4: the legacy row carries the metadata; the alias names an identity that is gone. + state.worktreeMeta[WORKTREE] = { ...meta('ssh:host-a'), instanceId: 'instance-1' } + const alias = `ssh:host-a|${WORKTREE}` + state.worktreeIdentityAliases = { [alias]: ['wt2:ssh%3Ahost-a:instance-1'] } + state.worktreeMetaByIdentity = {} + const scope = createOrcadMigrationSourceScope({ + source: SOURCE, + catalog: { repositories: [REPO_A], projectGroups: [], folderWorkspaces: [] }, + repos: state.repos + }) + const inspection = inspectOrcadSourceWorktreeMetadata(state, scope) + expect(inspection.blockedCount).toBe(0) + expect(inspection.rows.map((row) => row.sourceKey)).toEqual([WORKTREE]) + + // A dangling alias with no legacy row beside it is dropped too. + state.worktreeIdentityAliases[`ssh:host-a|${REPO_A.id}::/srv/gone`] = ['wt2:ssh%3Ahost-a:x'] + expect(inspectOrcadSourceWorktreeMetadata(state, scope).blockedCount).toBe(0) + subtractOrcadMigrationSourceDormantState(state, manifest(state)) + expect(state.worktreeIdentityAliases).toEqual({}) + expect(state.worktreeMeta).toEqual({}) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.test.ts new file mode 100644 index 00000000000..c04be35e039 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { Repo } from '../../../shared/repo-types' +import { orcadMigrationCutoverFixture } from '../../ssh/orcad-migration-cutover-fixture' +import { collectOrcadMigrationSourceCatalog } from './orcad-source-catalog' +import { subtractOrcadSourceCatalogState } from './orcad-source-catalog-subtraction' +import { orcadSourceFolderWorkspaceIds, repoBelongsToOrcadSource } from './orcad-source-ownership' + +const TARGET = 'ssh-win' + +function repo(overrides: Partial & Pick): Repo { + return { displayName: overrides.id, badgeColor: '#000', addedAt: 0, ...overrides } +} + +function folder( + overrides: Partial & Pick +): FolderWorkspace { + return { + projectGroupId: 'group-folders', + name: overrides.id, + folderPath: 'C:\\Users\\Ann\\work', + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 0, + createdAt: 0, + updatedAt: 0, + ...overrides + } +} + +const group: ProjectGroup = { + id: 'group-folders', + name: 'Folders', + parentPath: null, + parentGroupId: null, + createdFrom: 'manual', + tabOrder: 0, + isCollapsed: false, + color: null, + createdAt: 0, + updatedAt: 0 +} + +/** A Windows relay host: drive-letter paths whose case differs between rows. */ +function windowsCatalog() { + return { + repos: [ + repo({ id: 'repo-app', path: 'C:\\Users\\Ann\\work\\app', connectionId: TARGET }), + repo({ id: 'repo-unified', path: 'D:/src/tool', executionHostId: `ssh:${TARGET}` }), + repo({ id: 'repo-local', path: 'C:\\Users\\Ann\\local' }) + ], + projectGroups: [group], + folderWorkspaces: [ + // No connection of its own: the repo inside it, matched case-insensitively, places it. + folder({ id: 'fw-inferred', folderPath: 'c:/users/ann/WORK' }), + folder({ id: 'fw-explicit', folderPath: 'C:\\elsewhere', connectionId: TARGET }), + folder({ id: 'fw-local', folderPath: 'C:\\Users\\Ann\\local' }), + folder({ id: 'fw-other', folderPath: 'C:\\Users\\Ann\\work', connectionId: 'ssh-other' }) + ] + } +} + +describe('orcad migration source ownership on a Windows SSH host', () => { + it('owns repos by either host spelling and folders the way the app attributes them', () => { + const catalog = windowsCatalog() + expect( + catalog.repos.filter((entry) => repoBelongsToOrcadSource(entry, TARGET)).map(({ id }) => id) + ).toEqual(['repo-app', 'repo-unified']) + expect([...orcadSourceFolderWorkspaceIds(catalog, TARGET)].sort()).toEqual([ + 'fw-explicit', + 'fw-inferred' + ]) + }) + + it('exports exactly those rows', () => { + const catalog = windowsCatalog() + const exported = collectOrcadMigrationSourceCatalog( + { + getRepos: () => catalog.repos, + getProjectGroups: () => catalog.projectGroups, + getFolderWorkspaces: () => catalog.folderWorkspaces + }, + { id: TARGET } + ) + expect(exported.repositories.map(({ id }) => id)).toEqual(['repo-app', 'repo-unified']) + expect(exported.folderWorkspaces.map(({ id }) => id).sort()).toEqual([ + 'fw-explicit', + 'fw-inferred' + ]) + }) + + it('retires a folder owned through its repo even though the repo goes first', () => { + const catalog = windowsCatalog() + const state = { ...getDefaultPersistedState('C:/Users/Ann'), ...catalog } + const fixture = orcadMigrationCutoverFixture('migration-1', TARGET) + subtractOrcadSourceCatalogState(state, { + ...fixture.manifest, + payload: { + repositories: catalog.repos.slice(0, 2), + projectGroups: [], + folderWorkspaces: catalog.folderWorkspaces.slice(0, 2) + } + }) + expect(state.repos.map(({ id }) => id)).toEqual(['repo-local']) + expect(state.folderWorkspaces.map(({ id }) => id)).toEqual(['fw-local', 'fw-other']) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.ts new file mode 100644 index 00000000000..d2c4023ab68 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-ownership.ts @@ -0,0 +1,56 @@ +/** + * Which catalog rows a migrating SSH target owns, answered the way the app attributes them. + * + * A repo names its host by the legacy `connectionId` or by `executionHostId: 'ssh:'`; a + * folder workspace by its own or its group's connection, or by the repos inside its folder, which + * main matches with `isPathInsideOrEqual` (drive letters and case folded on Windows paths). Export + * and subtraction must use one rule, or a row exported by one is left behind by the other. + */ +import { getRepoExecutionHostId, parseExecutionHostId } from '../../../shared/execution-host' +import { + resolveFolderWorkspaceHost, + type FolderWorkspaceHostState +} from '../../../shared/folder-workspace-execution-host' +import type { FolderWorkspace } from '../../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../../shared/project-group-types' +import type { Repo } from '../../../shared/repo-types' + +export function repoBelongsToOrcadSource(repo: Repo, targetId: string): boolean { + if (repo.connectionId === targetId) { + return true + } + const host = parseExecutionHostId(getRepoExecutionHostId(repo)) + return host?.kind === 'ssh' && host.targetId === targetId +} + +export function projectGroupBelongsToOrcadSource( + group: Pick, + targetId: string +): boolean { + if (group.connectionId === targetId) { + return true + } + const host = parseExecutionHostId(group.executionHostId) + return host?.kind === 'ssh' && host.targetId === targetId +} + +/** Folder workspace ids the target owns in `state`; read before any of its repos are removed. */ +export function orcadSourceFolderWorkspaceIds( + state: FolderWorkspaceHostState, + targetId: string +): Set { + const groupConnectionById = new Map( + state.projectGroups.map((group) => [group.id, group.connectionId]) + ) + const owned = (workspace: FolderWorkspace): boolean => { + if ( + (workspace.connectionId ?? groupConnectionById.get(workspace.projectGroupId) ?? null) === + targetId + ) { + return true + } + const host = resolveFolderWorkspaceHost(state, workspace.id) + return host.kind === 'ssh' && host.targetId === targetId + } + return new Set(state.folderWorkspaces.filter(owned).map((workspace) => workspace.id)) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-partial-partition.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-partial-partition.test.ts new file mode 100644 index 00000000000..49c49d1ba49 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-partial-partition.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultPersistedState } from '../../../shared/constants' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import type { Repo } from '../../../shared/repo-types' +import { collectOrcadMigrationSourceDormantState } from './orcad-source-dormant-state' + +const TARGET_ID = 'ssh-b4' +const WORKTREE = 'repo-1::/root/repo' +const REPO: Repo = { + id: 'repo-1', + path: '/root/repo', + displayName: 'repo', + badgeColor: '#737373', + addedAt: 1, + connectionId: TARGET_ID +} +const SOURCE = { sshTargetId: TARGET_ID, sshTargetGeneration: 2, targetLabel: 'B4' } +const CATALOG = { repositories: [REPO], projectGroups: [], folderWorkspaces: [] } + +/** The real-host shape: renderer snapshots split per host leave out maps a host had no rows in. */ +function partialPartitions() { + const state = getDefaultPersistedState('/tmp/orcad-partial-partition') + state.repos = [REPO] + state.workspaceSession = { + ...state.workspaceSession, + defaultTerminalTabsAppliedByWorktreeId: { [WORKTREE]: true } + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: reproduces partitions persisted without required maps. + const runtimePartition = { + activeRepoId: null, + activeWorktreeId: null, + activeTabId: null, + unifiedTabs: {}, + tabGroups: {} + } as unknown as WorkspaceSessionState + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: same, for the source host. + const hostPartition = { + activeRepoId: null, + activeWorktreeId: null, + activeTabId: null, + tabsByWorktree: { [WORKTREE]: [] }, + activeTabTypeByWorktree: { [WORKTREE]: 'terminal' }, + defaultTerminalTabsAppliedByWorktreeId: { [WORKTREE]: true }, + closedTerminalTabTombstonesByTabId: { + 'tab-closed': { worktreeId: WORKTREE, reason: 'user', closedAt: 1 } + }, + terminalLayoutsByTabId: {} + } as unknown as WorkspaceSessionState + state.workspaceSessionsByHostId = { + 'runtime:env-1': runtimePartition, + [`ssh:${TARGET_ID}`]: hostPartition + } + return state +} + +describe('collecting dormant state from partially written session partitions', () => { + it('neither throws on missing maps nor blocks on a marker both partitions agree on', () => { + const inspection = collectOrcadMigrationSourceDormantState(partialPartitions(), SOURCE, CATALOG) + expect(inspection.blockedCounts['workspace-session']).toBe(0) + expect(inspection.payload.workspaceSession?.defaultTerminalTabsAppliedByWorktreeId).toEqual({ + [WORKTREE]: true + }) + }) + + it('still blocks when two partitions disagree about the same worktree', () => { + const state = partialPartitions() + state.workspaceSession.activeTabTypeByWorktree = { [WORKTREE]: 'editor' } + expect( + collectOrcadMigrationSourceDormantState(state, SOURCE, CATALOG).blockedCounts[ + 'workspace-session' + ] + ).toBe(1) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-retired-worktree-names.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-retired-worktree-names.ts new file mode 100644 index 00000000000..f63c300a22d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-retired-worktree-names.ts @@ -0,0 +1,78 @@ +/** Retired worktree names (the name registry, not migration retirement) the target carries. */ +import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationDormantStatePayload +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { + isEmptyRetiredNameRegistry, + mergeRetiredNameRegistries, + type RetiredNameRegistry +} from '../../../shared/worktree/retired-name-registry' +import { getRemoteRetirementNamespaceKey } from '../../worktree-name-retirement' +import { + swapRetirementNamespaceHost, + retirementHostIdentity, + retirementNamespaceKeysToRead +} from '../../worktree-retirement-namespace' +import { compareKeys } from './orcad-source-key-order' + +const EMPTY_REGISTRY: RetiredNameRegistry = { exhaustedTiers: 0, names: [] } + +export function collectOrcadMigrationRetiredWorktreeNamespaces( + state: PersistedState, + catalog: OrcadMigrationCatalogPayload +): OrcadMigrationDormantStatePayload['retiredWorktreeNamespaces'] { + const lookup = (targetId: string) => state.sshTargets.find((target) => target.id === targetId) + const byDestination = new Map< + string, + { sourceNamespaceKeys: Set; registry: RetiredNameRegistry } + >() + for (const repo of catalog.repositories) { + const canonicalSource = getRemoteRetirementNamespaceKey(repo, state.settings, lookup) + if (!canonicalSource) { + continue + } + const sourceNamespaceKeys = retirementNamespaceKeysToRead(repo, canonicalSource, lookup).filter( + (key) => state.retiredWorktreeNamesByNamespace?.[key] !== undefined + ) + if (sourceNamespaceKeys.length === 0) { + continue + } + const registry = sourceNamespaceKeys.reduce( + (merged, key) => + mergeRetiredNameRegistries( + merged, + state.retiredWorktreeNamesByNamespace?.[key] ?? { exhaustedTiers: 0, names: [] } + ), + EMPTY_REGISTRY + ) + if (isEmptyRetiredNameRegistry(registry)) { + continue + } + const namespaceKey = swapRetirementNamespaceHost( + canonicalSource, + retirementHostIdentity(repo, lookup), + LOCAL_EXECUTION_HOST_ID + ) + if (!namespaceKey) { + continue + } + const existing = byDestination.get(namespaceKey) + byDestination.set(namespaceKey, { + sourceNamespaceKeys: new Set([ + ...(existing?.sourceNamespaceKeys ?? []), + ...sourceNamespaceKeys + ]), + registry: existing ? mergeRetiredNameRegistries(existing.registry, registry) : registry + }) + } + return [...byDestination.entries()] + .map(([namespaceKey, entry]) => ({ + namespaceKey, + sourceNamespaceKeys: [...entry.sourceNamespaceKeys].sort(compareKeys), + registry: entry.registry + })) + .sort((left, right) => compareKeys(left.namespaceKey, right.namespaceKey)) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.test.ts new file mode 100644 index 00000000000..ea0ae243648 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope +} from './orcad-source-scope' + +const scope = createOrcadMigrationSourceScope({ + source: { sshTargetId: 'ssh-prod', sshTargetGeneration: 1, targetLabel: 'Production' }, + catalog: { + repositories: [ + { + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: 'ssh-prod' + } + ], + projectGroups: [], + folderWorkspaces: [ + { + id: 'folder-1', + projectGroupId: 'group-1', + name: 'Notes', + folderPath: '/srv/notes', + connectionId: 'ssh-prod', + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 0, + createdAt: 1, + updatedAt: 1 + } + ] + }, + repos: [] +}) + +describe('migration source scope', () => { + it.each([ + ['a worktree of an exported repository', 'repo-1::/srv/app'], + ['an exported folder workspace', 'folder:folder-1'] + ])('owns %s', (_name, ownerKey) => { + expect(orcadMigrationOwnerMatchesScope(ownerKey, scope)).toBe(true) + }) + + it.each([ + ['another repository', 'repo-2::/srv/other'], + ['another folder workspace', 'folder:folder-2'], + ['no owner', null] + ])('does not own %s', (_name, ownerKey) => { + expect(orcadMigrationOwnerMatchesScope(ownerKey, scope)).toBe(false) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.ts new file mode 100644 index 00000000000..8af68c213e2 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scope.ts @@ -0,0 +1,108 @@ +import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../../../shared/execution-host' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import { parseWorkspaceKey } from '../../../shared/workspace-scope' +import { + getExecutionHostIdFromWorktreeHostIdentity, + getWorktreeIdFromHostIdentity, + isWorktreeHostIdentity +} from '../../../shared/worktree/host-qualified-identity' +import type { Repo } from '../../../shared/repo-types' +import { ownerKeyBelongsToRepo } from '../../orca-profiles/profile-project-worktree-identity' +import { repoBelongsToOrcadSource } from './orcad-source-ownership' + +export type OrcadMigrationSourceScope = { + targetId: string + targetGeneration: number | null + hostId: ReturnType + repoIds: ReadonlySet + folderWorkspaceKeys: ReadonlySet + /** + * Catalog repo ids another host registers too. A legacy unqualified key, or a bare repo id, for + * one of these cannot say whose it is, so it never matches: it is neither moved nor subtracted. + */ + sharedRepoIds: ReadonlySet + /** The session partition being read: an unqualified key there belongs to that partition's host. */ + partitionHostId?: string +} + +/** The scope as seen from one session partition. */ +export function orcadMigrationPartitionScope( + scope: OrcadMigrationSourceScope, + partitionHostId: string +): OrcadMigrationSourceScope { + return { ...scope, partitionHostId } +} + +export function createOrcadMigrationSourceScope(args: { + source: OrcadMigrationManifestSource + catalog: OrcadMigrationCatalogPayload + /** The profile's repo rows, read for ids another host shares. */ + repos: readonly Repo[] +}): OrcadMigrationSourceScope { + const repoIds = new Set(args.catalog.repositories.map((repo) => repo.id)) + return { + targetId: args.source.sshTargetId, + targetGeneration: args.source.sshTargetGeneration, + hostId: toSshExecutionHostId(args.source.sshTargetId), + repoIds, + folderWorkspaceKeys: new Set( + args.catalog.folderWorkspaces.map((workspace) => `folder:${workspace.id}`) + ), + sharedRepoIds: new Set( + args.repos + .filter( + (repo) => repoIds.has(repo.id) && !repoBelongsToOrcadSource(repo, args.source.sshTargetId) + ) + .map((repo) => repo.id) + ) + } +} + +/** A bare repo id the source owns alone; one another host shares cannot be attributed. */ +export function orcadMigrationOwnsRepoId( + scope: OrcadMigrationSourceScope, + repoId: string | null | undefined +): boolean { + return typeof repoId === 'string' && scope.repoIds.has(repoId) && !scope.sharedRepoIds.has(repoId) +} + +/** `rowHostId` is the row's own host evidence, such as worktree metadata's `hostId`. */ +export function orcadMigrationOwnerMatchesScope( + value: string | null | undefined, + scope: OrcadMigrationSourceScope, + rowHostId?: string +): boolean { + if (!value) { + return false + } + // Why: a repo id may repeat across hosts; only the qualifier, the partition or the row says whose. + const qualified = isWorktreeHostIdentity(value) + const ownerHost = qualified + ? getExecutionHostIdFromWorktreeHostIdentity(value) + : scope.partitionHostId !== undefined && scope.partitionHostId !== LOCAL_EXECUTION_HOST_ID + ? scope.partitionHostId + : rowHostId + if (ownerHost !== undefined && ownerHost !== scope.hostId) { + return false + } + const rawValue = qualified ? getWorktreeIdFromHostIdentity(value) : value + if (scope.folderWorkspaceKeys.has(rawValue)) { + return true + } + for (const repoId of scope.repoIds) { + if (ownerKeyBelongsToRepo(rawValue, repoId)) { + return ownerHost !== undefined || !scope.sharedRepoIds.has(repoId) + } + } + const parsed = parseWorkspaceKey(rawValue) + return ( + parsed?.type === 'folder' && scope.folderWorkspaceKeys.has(`folder:${parsed.folderWorkspaceId}`) + ) +} + +export function unqualifyOrcadMigrationOwnerKey(value: string): string { + return isWorktreeHostIdentity(value) ? getWorktreeIdFromHostIdentity(value) : value +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-cleanup.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-cleanup.ts new file mode 100644 index 00000000000..c7112608cbd --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-cleanup.ts @@ -0,0 +1,30 @@ +import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import { + collectTerminalScrollbackSnapshotRefs, + deleteTerminalScrollbackSnapshotSync +} from '../../terminal-scrollback-snapshots' +import { sessionPartitions } from './orcad-source-workspace-session-fragments' + +/** Deletes the given snapshot files unless a session or a pending export still names them. */ +export function deleteUnreferencedOrcadMigrationScrollback( + runtime: Pick< + StoreRuntimeState, + 'state' | 'terminalScrollbackSnapshotStorage' | 'retainedScrollbackRefsByMigrationId' + >, + refs: Iterable +): void { + const live = new Set([ + ...sessionPartitions(runtime.state, LOCAL_EXECUTION_HOST_ID).flatMap(([, session]) => [ + ...collectTerminalScrollbackSnapshotRefs(session) + ]), + ...[...runtime.retainedScrollbackRefsByMigrationId.values()].flatMap((retained) => [ + ...retained + ]) + ]) + for (const ref of refs) { + if (!live.has(ref)) { + deleteTerminalScrollbackSnapshotSync(ref, runtime.terminalScrollbackSnapshotStorage) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-retention.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-retention.ts new file mode 100644 index 00000000000..e27ed223c2e --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-retention.ts @@ -0,0 +1,66 @@ +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import type { StoreRuntimeState } from '../loading-store/store-runtime-state' +import { + readTerminalScrollbackStoredBytesSync, + writeTerminalScrollbackSnapshotSync +} from '../../terminal-scrollback-snapshots' +import { deleteUnreferencedOrcadMigrationScrollback } from './orcad-source-scrollback-cleanup' +import { findSnapshotBytes } from './orcad-source-scrollback-state' + +type RetentionRuntime = Pick< + StoreRuntimeState, + 'state' | 'terminalScrollbackSnapshotStorage' | 'retainedScrollbackRefsByMigrationId' +> + +/** + * Holds every snapshot an unfinished migration's manifest names, from the journaled export until + * the destination commits or the migration is abandoned, so a tab closed meanwhile still sends + * its bytes on any retry. An inline buffer has no file, so its bytes are written to its ref first. + * Released refs no session names are deleted. + */ +export function syncOrcadMigrationScrollbackRetention( + runtime: RetentionRuntime, + pending: readonly OrcadMigrationManifest[] +): void { + const held = runtime.retainedScrollbackRefsByMigrationId + const pendingIds = new Set(pending.map((manifest) => manifest.migrationId)) + const released = [...held].filter(([migrationId]) => !pendingIds.has(migrationId)) + for (const manifest of pending) { + if (held.has(manifest.migrationId)) { + continue + } + const snapshots = manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + for (const descriptor of snapshots) { + persistInlineSnapshot(runtime, descriptor) + } + held.set(manifest.migrationId, new Set(snapshots.map((snapshot) => snapshot.ref))) + } + for (const [migrationId] of released) { + held.delete(migrationId) + } + deleteUnreferencedOrcadMigrationScrollback( + runtime, + released.flatMap(([, refs]) => [...refs]) + ) +} + +function persistInlineSnapshot( + runtime: RetentionRuntime, + descriptor: NonNullable< + NonNullable['terminalScrollbackSnapshots'] + >[number] +): void { + const storage = runtime.terminalScrollbackSnapshotStorage + if (readTerminalScrollbackStoredBytesSync(descriptor.ref, storage)) { + return + } + const bytes = findSnapshotBytes(runtime.state, descriptor, storage) + if (bytes) { + writeTerminalScrollbackSnapshotSync({ + tabId: descriptor.tabId, + leafId: descriptor.leafId, + buffer: bytes.toString('utf8'), + storage + }) + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.test.ts new file mode 100644 index 00000000000..516365f8fa3 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import type { OrcadMigrationTerminalScrollbackSnapshot } from '../../../shared/orcad-migration-scrollback' +import { hasDuplicateOrcadMigrationScrollbackDescriptors } from './orcad-source-scrollback-state' + +const FIRST: OrcadMigrationTerminalScrollbackSnapshot = { + tabId: 'tab-1', + leafId: 'leaf-1', + ref: `v1-${'1'.repeat(32)}`, + sha256: 'a'.repeat(64), + byteLength: 1 +} + +describe('orcad source scrollback projection', () => { + it('refuses duplicate refs or tab/leaf identities across merged fragments', () => { + expect( + hasDuplicateOrcadMigrationScrollbackDescriptors([ + FIRST, + { ...FIRST, tabId: 'tab-2', leafId: 'leaf-2' } + ]) + ).toBe(true) + expect( + hasDuplicateOrcadMigrationScrollbackDescriptors([ + FIRST, + { ...FIRST, ref: `v1-${'2'.repeat(32)}` } + ]) + ).toBe(true) + expect( + hasDuplicateOrcadMigrationScrollbackDescriptors([ + FIRST, + { + ...FIRST, + tabId: 'tab-2', + leafId: 'leaf-2', + ref: `v1-${'2'.repeat(32)}` + } + ]) + ).toBe(false) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.ts new file mode 100644 index 00000000000..b5480507c2b --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-scrollback-state.ts @@ -0,0 +1,166 @@ +import { createHash } from 'node:crypto' +import type { OrcadMigrationTerminalScrollbackSnapshot } from '../../../shared/orcad-migration-scrollback' +import { + MAX_ORCAD_MIGRATION_SCROLLBACK_SNAPSHOTS, + MAX_ORCAD_MIGRATION_SCROLLBACK_TOTAL_BYTES +} from '../../../shared/orcad-migration-scrollback' +import type { PersistedState } from '../../../shared/persisted-state-types' +import { TERMINAL_SCROLLBACK_STORE_BYTE_LIMIT } from '../../../shared/terminal-scrollback-limits' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { + makeTerminalScrollbackSnapshotRef, + readTerminalScrollbackStoredBytesSync, + type TerminalScrollbackSnapshotStorage +} from '../../terminal-scrollback-snapshots' +import { compareKeys } from './orcad-source-key-order' + +export type ProjectedOrcadMigrationScrollback = { + session: WorkspaceSessionState + snapshots: OrcadMigrationTerminalScrollbackSnapshot[] + blockedCount: number +} + +export function hasDuplicateOrcadMigrationScrollbackDescriptors( + snapshots: readonly OrcadMigrationTerminalScrollbackSnapshot[] +): boolean { + const refs = new Set() + const leaves = new Set() + for (const snapshot of snapshots) { + const leaf = `${snapshot.tabId}\0${snapshot.leafId}` + if (refs.has(snapshot.ref) || leaves.has(leaf)) { + return true + } + refs.add(snapshot.ref) + leaves.add(leaf) + } + return false +} + +export function projectOrcadMigrationSessionScrollback( + session: WorkspaceSessionState, + storage?: TerminalScrollbackSnapshotStorage +): ProjectedOrcadMigrationScrollback { + const projected = structuredClone(session) + const snapshots: OrcadMigrationTerminalScrollbackSnapshot[] = [] + let blockedCount = 0 + let totalBytes = 0 + for (const [tabId, layout] of Object.entries(projected.terminalLayoutsByTabId ?? {})) { + const sourceLayout = session.terminalLayoutsByTabId[tabId] + const refs: Record = {} + const leafIds = new Set([ + ...Object.keys(sourceLayout.buffersByLeafId ?? {}), + ...Object.keys(sourceLayout.scrollbackRefsByLeafId ?? {}) + ]) + for (const leafId of [...leafIds].sort(compareKeys)) { + const buffer = sourceLayout.buffersByLeafId?.[leafId] + const sourceRef = sourceLayout.scrollbackRefsByLeafId?.[leafId] + const ref = buffer ? makeTerminalScrollbackSnapshotRef(tabId, leafId) : sourceRef + const bytes = buffer + ? Buffer.from(buffer, 'utf8') + : sourceRef + ? readTerminalScrollbackStoredBytesSync(sourceRef, storage) + : null + if ( + !ref || + !bytes || + bytes.length === 0 || + bytes.length > TERMINAL_SCROLLBACK_STORE_BYTE_LIMIT + ) { + blockedCount += 1 + continue + } + totalBytes += bytes.length + refs[leafId] = ref + snapshots.push({ + tabId, + leafId, + ref, + sha256: createHash('sha256').update(bytes).digest('hex'), + byteLength: bytes.length + }) + } + delete layout.buffersByLeafId + if (Object.keys(refs).length > 0) { + layout.scrollbackRefsByLeafId = refs + } else { + delete layout.scrollbackRefsByLeafId + } + } + if ( + snapshots.length > MAX_ORCAD_MIGRATION_SCROLLBACK_SNAPSHOTS || + totalBytes > MAX_ORCAD_MIGRATION_SCROLLBACK_TOTAL_BYTES + ) { + blockedCount += 1 + } + snapshots.sort((left, right) => + compareKeys(`${left.tabId}\0${left.leafId}`, `${right.tabId}\0${right.leafId}`) + ) + return { session: projected, snapshots, blockedCount } +} + +export function readOrcadMigrationSourceScrollbackChunk(args: { + state: PersistedState + descriptor: OrcadMigrationTerminalScrollbackSnapshot + offset: number + length: number + storage?: TerminalScrollbackSnapshotStorage + /** Kept on disk for an export in flight, so a tab closed mid-transfer still reads from storage. */ + retained?: boolean +}): { bytesBase64: string; totalBytes: number; eof: boolean } { + const bytes = findSnapshotBytes(args.state, args.descriptor, args.storage, args.retained) + if (!bytes) { + throw new Error('orcad_migration_source_snapshot_changed') + } + if (!Number.isSafeInteger(args.offset) || args.offset < 0 || args.offset > bytes.length) { + throw new Error('orcad_migration_source_snapshot_offset_invalid') + } + const end = Math.min(bytes.length, args.offset + args.length) + return { + bytesBase64: bytes.subarray(args.offset, end).toString('base64'), + totalBytes: bytes.length, + eof: end === bytes.length + } +} + +export function findSnapshotBytes( + state: PersistedState, + descriptor: OrcadMigrationTerminalScrollbackSnapshot, + storage?: TerminalScrollbackSnapshotStorage, + retained = false +): Buffer | null { + for (const session of sessionPartitions(state)) { + const layout = session.terminalLayoutsByTabId?.[descriptor.tabId] + if (!layout) { + continue + } + const buffer = layout.buffersByLeafId?.[descriptor.leafId] + const ref = layout.scrollbackRefsByLeafId?.[descriptor.leafId] + const bytes = buffer + ? Buffer.from(buffer, 'utf8') + : ref === descriptor.ref + ? readTerminalScrollbackStoredBytesSync(ref, storage) + : null + if (matchesDescriptor(bytes, descriptor)) { + return bytes + } + } + const stored = retained ? readTerminalScrollbackStoredBytesSync(descriptor.ref, storage) : null + return matchesDescriptor(stored, descriptor) ? stored : null +} + +function matchesDescriptor( + bytes: Buffer | null, + descriptor: OrcadMigrationTerminalScrollbackSnapshot +): bytes is Buffer { + return ( + bytes !== null && + bytes.length === descriptor.byteLength && + createHash('sha256').update(bytes).digest('hex') === descriptor.sha256 + ) +} + +function sessionPartitions(state: PersistedState): WorkspaceSessionState[] { + return [state.workspaceSession, ...Object.values(state.workspaceSessionsByHostId ?? {})].filter( + (session): session is WorkspaceSessionState => session !== undefined + ) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-session-dependencies.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-session-dependencies.ts new file mode 100644 index 00000000000..8e1131b6e30 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-session-dependencies.ts @@ -0,0 +1,67 @@ +import type { ExecutionHostId } from '../../../shared/execution-host' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { sessionPartitions } from './orcad-source-workspace-session-fragments' + +export type OrcadMigrationSourceSessionInspection = { + ptyIds: Set + tabIds: Set +} + +type SessionScope = { + hostId: ExecutionHostId + /** `partitionHostId` owns the partition's unqualified keys. */ + ownerMatches: (ownerKey: string, partitionHostId: string) => boolean +} + +/** The terminal tabs and PTYs the target's sessions own, across every partition. */ +export function inspectOrcadMigrationSourceSessions( + state: PersistedState, + scope: SessionScope +): OrcadMigrationSourceSessionInspection { + const result: OrcadMigrationSourceSessionInspection = { + ptyIds: new Set(), + tabIds: new Set() + } + for (const [hostId, session] of sessionPartitions(state, 'local')) { + inspectSession(session, scope, hostId, result) + } + return result +} + +function inspectSession( + session: WorkspaceSessionState, + scope: SessionScope, + partitionHostId: string, + result: OrcadMigrationSourceSessionInspection +): void { + const sourceHostPartition = partitionHostId === scope.hostId + const matchesOwner = (ownerKey: string): boolean => + Boolean(ownerKey) && (sourceHostPartition || scope.ownerMatches(ownerKey, partitionHostId)) + for (const [ownerKey, tabs] of Object.entries(session.tabsByWorktree ?? {})) { + if (!matchesOwner(ownerKey)) { + continue + } + for (const tab of tabs) { + result.tabIds.add(tab.id) + if (tab.ptyId) { + result.ptyIds.add(tab.ptyId) + } + } + } + for (const [ownerKey, tabs] of Object.entries(session.unifiedTabs ?? {})) { + if (!matchesOwner(ownerKey)) { + continue + } + for (const tab of tabs) { + if (tab.contentType === 'terminal') { + result.tabIds.add(tab.entityId) + } + } + } + for (const tombstone of Object.values(session.terminalSurfaceTombstonesByPaneKey ?? {})) { + if (matchesOwner(tombstone.worktreeId)) { + result.ptyIds.add(tombstone.ptyId) + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-eligibility.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-eligibility.ts new file mode 100644 index 00000000000..3076c0b85b0 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-eligibility.ts @@ -0,0 +1,168 @@ +import { isWorkspaceKey } from '../../../shared/workspace-scope' +import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' +import type { SleepingAgentSessionRecord } from '../../../shared/agent-session-resume' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { buildMarkdownFrontmatterIdMap } from '../../orca-profiles/profile-session-owner-transfer' +import { + orcadMigrationOwnerMatchesScope, + orcadMigrationOwnsRepoId, + unqualifyOrcadMigrationOwnerKey, + type OrcadMigrationSourceScope +} from './orcad-source-scope' +import { paneBelongsToTerminalLayout } from '../../../shared/workspace-session-pane-ownership' +import { collectSessionOwnerKeys } from './orcad-source-workspace-session-fragments' + +export function countUnrepresentableMarkdownState( + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope, + sourceHostPartition: boolean +): number { + const projection = { + mapOwnerKey: (ownerKey: string) => + sourceHostPartition || orcadMigrationOwnerMatchesScope(ownerKey, scope) + ? unqualifyOrcadMigrationOwnerKey(ownerKey) + : null, + mapWorktreeId: unqualifyOrcadMigrationOwnerKey + } + const mappings = buildMarkdownFrontmatterIdMap(session.openFilesByWorktree, projection) + return Object.keys(session.markdownFrontmatterVisible ?? {}).filter( + (fileId) => mappings.get(fileId) === null + ).length +} + +export function countUnsupportedSessionState( + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope, + sourceHostPartition: boolean, + terminalTabIds: ReadonlySet +): number { + const owns = (ownerKey: string): boolean => orcadMigrationOwnerMatchesScope(ownerKey, scope) + const matches = (ownerKey: string): boolean => + Boolean(ownerKey) && (sourceHostPartition || owns(ownerKey)) + let count = sourceHostPartition + ? [...collectSessionOwnerKeys(session)].filter((ownerKey) => !owns(ownerKey)).length + : 0 + // PTY bindings, remote session ids and shutdown markers are not counted: whether their terminals + // still run is the terminal gate's verdict, taken before every move; the move drops them. + count += Object.values(session.sleepingAgentSessionsByPaneKey ?? {}).filter((record) => { + const touchesSource = matches(record.worktreeId) || record.connectionId === scope.targetId + return ( + touchesSource && !transferableSleepingAgentSession(record, session, scope, terminalTabIds) + ) + }).length + count += Object.entries(session.clientHostedBrowserPagesByWorktree ?? {}) + .filter(([ownerKey]) => matches(ownerKey)) + .filter( + ([ownerKey, pages]) => !clientHostedPagesAreTransferable(session, ownerKey, pages) + ).length + count += + sourceHostPartition && + session.activeRepoId && + owns(session.activeRepoId) && + !scope.repoIds.has(session.activeRepoId) + ? 1 + : 0 + // Focus outside the source partition is client focus, not host state, so it never moves. + // activeConnectionIdsAtShutdown is not counted: it is the renderer's live "connected now" hint, and + // the remote work it can stand for (tab PTYs, remote session ids, leases) is the terminal gate's. + for (const [ownerKey, files] of Object.entries(session.openFilesByWorktree ?? {})) { + if (owns(ownerKey)) { + count += files.filter( + (file) => file.externalSshTargetId !== undefined || Boolean(file.runtimeEnvironmentId) + ).length + } + } + for (const [ownerKey, workspaces] of Object.entries(session.browserTabsByWorktree ?? {})) { + if (owns(ownerKey)) { + count += workspaces.filter((workspace) => + Boolean(workspace.sessionProfileId || workspace.sessionPartition) + ).length + } + } + for (const [ownerKey, tabs] of Object.entries(session.unifiedTabs ?? {})) { + if (owns(ownerKey)) { + count += tabs.filter( + (tab) => + tab.executionHostId !== undefined && !isOrcadSourceTabHost(tab.executionHostId, scope) + ).length + } + } + return count +} + +function clientHostedPagesAreTransferable( + session: WorkspaceSessionState, + ownerKey: string, + pages: NonNullable[string] +): boolean { + const browserWorkspaceIds = new Set( + (session.browserTabsByWorktree?.[ownerKey] ?? []).map((workspace) => workspace.id) + ) + return pages.every((page) => browserWorkspaceIds.has(page.workspaceId)) +} + +export function projectDormantSessionFocus( + source: WorkspaceSessionState, + transferred: WorkspaceSessionState, + scope: OrcadMigrationSourceScope, + terminalTabIds: ReadonlySet +): void { + // These scalars are UI focus, not execution ownership. They are safe to carry + // only from the source host partition and only when they point at an entity + // already proven dormant and included in the projected session. + if (orcadMigrationOwnsRepoId(scope, source.activeRepoId)) { + transferred.activeRepoId = source.activeRepoId + } + if (source.activeWorktreeId && orcadMigrationOwnerMatchesScope(source.activeWorktreeId, scope)) { + transferred.activeWorktreeId = unqualifyOrcadMigrationOwnerKey(source.activeWorktreeId) + } + const activeWorkspaceKey = + source.activeWorkspaceKey && orcadMigrationOwnerMatchesScope(source.activeWorkspaceKey, scope) + ? unqualifyOrcadMigrationOwnerKey(source.activeWorkspaceKey) + : null + if (activeWorkspaceKey && isWorkspaceKey(activeWorkspaceKey)) { + transferred.activeWorkspaceKey = activeWorkspaceKey + } + if (source.activeWorkspaceExecutionHostId === scope.hostId) { + transferred.activeWorkspaceExecutionHostId = LOCAL_EXECUTION_HOST_ID + } + if (source.activeTabId && terminalTabIds.has(source.activeTabId)) { + transferred.activeTabId = source.activeTabId + } +} + +/** Older builds stamped 'local' on tabs created in an SSH worktree; its owner key proves the host. */ +function isOrcadSourceTabHost(executionHostId: string, scope: OrcadMigrationSourceScope): boolean { + return executionHostId === scope.hostId || executionHostId === LOCAL_EXECUTION_HOST_ID +} + +export function projectSessionToDestination( + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope +): WorkspaceSessionState { + const projected = structuredClone(session) + for (const tabs of Object.values(projected.unifiedTabs ?? {})) { + for (const tab of tabs) { + if (tab.executionHostId === scope.hostId) { + tab.executionHostId = LOCAL_EXECUTION_HOST_ID + } + } + } + return projected +} + +export function transferableSleepingAgentSession( + record: SleepingAgentSessionRecord, + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope, + terminalTabIds: ReadonlySet +): boolean { + return ( + orcadMigrationOwnerMatchesScope(record.worktreeId, scope) && + (record.connectionId == null || record.connectionId === scope.targetId) && + // Older profiles can still contain a worker-resume fence; never discard its authority. + (!('automaticResumeBlockedBy' in record) || record.automaticResumeBlockedBy === undefined) && + ((record.origin ?? 'worktree-sleep') === 'worktree-sleep' || + paneBelongsToTerminalLayout(record, session, terminalTabIds)) + ) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-fragments.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-fragments.ts new file mode 100644 index 00000000000..5b926a7525d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-fragments.ts @@ -0,0 +1,142 @@ +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { mergeWorkspaceSessions } from '../../orca-profiles/profile-project-session-state' +import { SESSION_FIELDS_PRUNED_BY_OWNER_KEY } from '../../orca-profiles/profile-project-session-field-disposition' +import { withRequiredWorkspaceSessionMaps } from '../loading-store/session-host-partitions' + +export function sessionPartitions( + state: { + workspaceSession: WorkspaceSessionState + workspaceSessionsByHostId?: Record + }, + localHostId: string +): [string, WorkspaceSessionState][] { + return [ + [localHostId, withRequiredWorkspaceSessionMaps(state.workspaceSession)], + ...Object.entries(state.workspaceSessionsByHostId ?? {}).flatMap( + ([hostId, session]): [string, WorkspaceSessionState][] => + // A partition written before its maps were filled in on write still loads without them. + session ? [[hostId, withRequiredWorkspaceSessionMaps(session)]] : [] + ) + ] +} + +/** + * Null when two partitions disagree about the same worktree. The same marker in both (a renderer + * snapshot can copy one into the local and the host partition) is not a disagreement. + */ +export function mergeSessionFragments( + fragments: WorkspaceSessionState[] +): WorkspaceSessionState | null { + const ownerKeys = new Set() + const entityKeys = new Set() + const keyedValues = new Map() + let merged: WorkspaceSessionState | undefined + for (const fragment of fragments) { + const owners = collectSessionOwnerKeys(fragment) + if ( + [...owners].some((key) => ownerKeys.has(key) && !agreesOnOwner(fragment, key, keyedValues)) || + hasDuplicates(entityKeys, collectSessionEntityKeys(fragment)) + ) { + return null + } + owners.forEach((key) => ownerKeys.add(key)) + for (const field of OWNER_KEYED_FIELDS) { + for (const [key, value] of Object.entries(fragment[field] ?? {})) { + keyedValues.set(`${field}\0${key}`, serializeOrcadMigrationValue(value)) + } + } + merged = mergeWorkspaceSessions(merged, fragment) + } + return merged ?? null +} + +const OWNER_KEYED_FIELDS = [ + 'tabsByWorktree', + 'openFilesByWorktree', + 'browserTabsByWorktree', + 'unifiedTabs', + 'tabGroups', + ...SESSION_FIELDS_PRUNED_BY_OWNER_KEY +] as const + +function agreesOnOwner( + fragment: WorkspaceSessionState, + ownerKey: string, + keyedValues: ReadonlyMap +): boolean { + return OWNER_KEYED_FIELDS.every((field) => { + const value: unknown = Object.getOwnPropertyDescriptor(fragment[field] ?? {}, ownerKey)?.value + const seen = keyedValues.get(`${field}\0${ownerKey}`) + return value === undefined || seen === undefined || seen === serializeOrcadMigrationValue(value) + }) +} + +export function collectSessionOwnerKeys(session: WorkspaceSessionState): Set { + const keys = new Set() + for (const field of OWNER_KEYED_FIELDS) { + Object.keys(session[field] ?? {}).forEach((key) => keys.add(key)) + } + Object.values(session.tabsByWorktree ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.openFilesByWorktree ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.browserTabsByWorktree ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.browserPagesByWorkspace ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.unifiedTabs ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.tabGroups ?? {}) + .flat() + .forEach((entry) => keys.add(entry.worktreeId)) + Object.values(session.terminalSurfaceTombstonesByPaneKey ?? {}).forEach((entry) => + keys.add(entry.worktreeId) + ) + Object.values(session.sleepingAgentSessionsByPaneKey ?? {}).forEach((entry) => + keys.add(entry.worktreeId) + ) + return keys +} + +export function collectSessionEntityKeys(session: WorkspaceSessionState): string[] { + const keys: string[] = [] + Object.values(session.tabsByWorktree ?? {}) + .flat() + .forEach((tab) => keys.push(`terminal:${tab.id}`)) + Object.values(session.browserTabsByWorktree ?? {}) + .flat() + .forEach((tab) => keys.push(`browser:${tab.id}`)) + Object.values(session.clientHostedBrowserPagesByWorktree ?? {}) + .flat() + .forEach((page) => keys.push(`client-browser-page:${page.browserPageId}`)) + Object.values(session.unifiedTabs ?? {}) + .flat() + .forEach((tab) => keys.push(`tab:${tab.id}`)) + Object.values(session.tabGroups ?? {}) + .flat() + .forEach((group) => keys.push(`group:${group.id}`)) + Object.keys(session.terminalSurfaceTombstonesByPaneKey ?? {}).forEach((key) => + keys.push(`tombstone:${key}`) + ) + Object.keys(session.sleepingAgentSessionsByPaneKey ?? {}).forEach((key) => + keys.push(`sleeping-agent:${key}`) + ) + return keys +} + +function hasDuplicates(seen: Set, incoming: Iterable): boolean { + let duplicate = false + for (const key of incoming) { + if (seen.has(key)) { + duplicate = true + } + seen.add(key) + } + return duplicate +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-layout.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-layout.ts new file mode 100644 index 00000000000..e3f98d8809d --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-layout.ts @@ -0,0 +1,52 @@ +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { parseWorkspaceKey } from '../../../shared/workspace-scope' +import { + orcadMigrationOwnerMatchesScope, + type OrcadMigrationSourceScope +} from './orcad-source-scope' + +export function collectOwnedTerminalTabIds( + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope +): Set { + const tabIds = new Set( + Object.entries(session.tabsByWorktree ?? {}).flatMap(([ownerKey, tabs]) => + orcadMigrationOwnerMatchesScope(ownerKey, scope) ? tabs.map((tab) => tab.id) : [] + ) + ) + for (const [ownerKey, tabs] of Object.entries(session.unifiedTabs ?? {})) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + for (const tab of tabs) { + if (tab.contentType === 'terminal') { + tabIds.add(tab.id) + tabIds.add(tab.entityId) + } + } + } + return tabIds +} +/** + * Relay PTY handles never move: the terminal gate proves their terminals exited before any move + * commits, so the destination gets the tabs and layouts without them and starts fresh shells. + */ +export function dropOrcadMigrationTerminalBindings(fragment: WorkspaceSessionState): void { + for (const tabs of Object.values(fragment.tabsByWorktree ?? {})) { + for (const tab of tabs) { + tab.ptyId = null + } + } + for (const layout of Object.values(fragment.terminalLayoutsByTabId ?? {})) { + delete layout.ptyIdsByLeafId + } + delete fragment.remoteSessionIdsByTabId + delete fragment.terminalPtyIncarnationsByPaneKey + delete fragment.activeWorktreeIdsOnShutdown + // A folder's topology fence guarded its relay PTYs only; the manifest carries repo fences alone. + for (const key of Object.keys(fragment.terminalTopologyRevisionByRepoId ?? {})) { + if (parseWorkspaceKey(key)?.type === 'folder') { + delete fragment.terminalTopologyRevisionByRepoId?.[key] + } + } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-subtraction.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-subtraction.ts new file mode 100644 index 00000000000..146cf2ee8de --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session-subtraction.ts @@ -0,0 +1,120 @@ +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import { markdownFileIdCandidates } from '../../orca-profiles/profile-session-markdown-transfer' +import { removeWorkspaceSessionOwners } from '../restoring-sessions/session-owner-removal' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + orcadMigrationOwnsRepoId, + orcadMigrationPartitionScope, + type OrcadMigrationSourceScope +} from './orcad-source-scope' +import { LOCAL_EXECUTION_HOST_ID, parseExecutionHostId } from '../../../shared/execution-host' +import { collectSessionOwnerKeys } from './orcad-source-workspace-session-fragments' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' + +/** On a copy of the profile only, like every subtraction the delta view runs. */ +export function subtractOrcadMigrationSourceWorkspaceSession( + state: PersistedState, + manifest: OrcadMigrationManifest +): void { + if (!manifest.payload.dormantState?.workspaceSession) { + return + } + subtractOrcadMigrationScopeWorkspaceSession(state, manifest) +} + +/** Every partition's session state for the manifest's catalog, whether or not it could move. */ +export function subtractOrcadMigrationScopeWorkspaceSession( + state: PersistedState, + manifest: OrcadMigrationManifest +): void { + const scope = createOrcadMigrationSourceScope({ + source: manifest.source, + catalog: manifest.payload, + repos: state.repos + }) + state.workspaceSession = removeOwnedSessionState( + state.workspaceSession, + orcadMigrationPartitionScope(scope, LOCAL_EXECUTION_HOST_ID) + ) + const partitions = state.workspaceSessionsByHostId + if (partitions) { + state.workspaceSessionsByHostId = Object.fromEntries( + Object.entries(partitions).map(([hostId, session]) => [ + hostId, + session + ? removeOwnedSessionState(session, orcadMigrationPartitionScope(scope, hostId)) + : session + ]) + ) + } +} + +function removeOwnedSessionState( + session: WorkspaceSessionState, + scope: OrcadMigrationSourceScope +): WorkspaceSessionState { + const ownerKeys = new Set( + [...collectSessionOwnerKeys(session)].filter((ownerKey) => + orcadMigrationOwnerMatchesScope(ownerKey, scope) + ) + ) + const removedMarkdownFileIds = new Set() + for (const [ownerKey, files] of Object.entries(session.openFilesByWorktree ?? {})) { + if (!orcadMigrationOwnerMatchesScope(ownerKey, scope)) { + continue + } + for (const file of files) { + markdownFileIdCandidates(file.filePath, ownerKey, file.runtimeEnvironmentId).forEach((id) => + removedMarkdownFileIds.add(id) + ) + } + } + // Focus retargeted to the destination names the same ids on the managed host; keep it. + const retargetedFocus = isOrcadRuntimeHostFocus(session) + ? { + activeRepoId: session.activeRepoId, + activeWorktreeId: session.activeWorktreeId, + activeWorkspaceKey: session.activeWorkspaceKey, + activeTabId: session.activeTabId + } + : null + const next = removeWorkspaceSessionOwners(session, ownerKeys) ?? session + // Selection-only sessions and canonical workspace keys need the same scoped subtraction. + const retired = next === session ? structuredClone(session) : next + if (retargetedFocus) { + Object.assign(retired, retargetedFocus) + } else { + if (orcadMigrationOwnsRepoId(scope, retired.activeRepoId)) { + retired.activeRepoId = null + } + if (orcadMigrationOwnerMatchesScope(retired.activeWorktreeId, scope)) { + retired.activeWorktreeId = null + } + if (orcadMigrationOwnerMatchesScope(retired.activeWorkspaceKey, scope)) { + retired.activeWorkspaceKey = null + } + } + if (retired.activeWorkspaceExecutionHostId === scope.hostId) { + retired.activeWorkspaceExecutionHostId = null + } + if (retired.markdownFrontmatterVisible) { + retired.markdownFrontmatterVisible = Object.fromEntries( + Object.entries(retired.markdownFrontmatterVisible).filter( + ([fileId]) => !removedMarkdownFileIds.has(fileId) + ) + ) + } + for (const repoId of scope.repoIds) { + if (!scope.sharedRepoIds.has(repoId)) { + delete retired.terminalTopologyRevisionByRepoId?.[repoId] + } + } + return retired +} + +/** Focus already on a runtime host names that host's workspace, never the SSH source's. */ +function isOrcadRuntimeHostFocus(session: WorkspaceSessionState): boolean { + return parseExecutionHostId(session.activeWorkspaceExecutionHostId)?.kind === 'runtime' +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session.ts new file mode 100644 index 00000000000..5743c5f0148 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-workspace-session.ts @@ -0,0 +1,127 @@ +import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationManifestSource +} from '../../../shared/orcad-migration-manifest' +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' +import type { OrcadMigrationTerminalScrollbackSnapshot } from '../../../shared/orcad-migration-scrollback' +import { + extractSessionOwnersForTransfer, + hasTransferredSessionState +} from '../../orca-profiles/profile-session-owner-transfer' +import type { TerminalScrollbackSnapshotStorage } from '../../terminal-scrollback-snapshots' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + orcadMigrationPartitionScope, + unqualifyOrcadMigrationOwnerKey +} from './orcad-source-scope' +import { + countUnrepresentableMarkdownState, + countUnsupportedSessionState, + projectDormantSessionFocus, + projectSessionToDestination, + transferableSleepingAgentSession +} from './orcad-source-workspace-session-eligibility' +import { + collectOwnedTerminalTabIds, + dropOrcadMigrationTerminalBindings +} from './orcad-source-workspace-session-layout' +import { + mergeSessionFragments, + sessionPartitions +} from './orcad-source-workspace-session-fragments' +import { + hasDuplicateOrcadMigrationScrollbackDescriptors, + projectOrcadMigrationSessionScrollback +} from './orcad-source-scrollback-state' + +export type OrcadMigrationSourceWorkspaceSessionInspection = { + payload: WorkspaceSessionState | undefined + snapshots: OrcadMigrationTerminalScrollbackSnapshot[] + blockedCount: number +} + +/** Fails closed: a session shape no collector expects blocks the move instead of failing connect. */ +export function collectOrcadMigrationSourceWorkspaceSession( + state: PersistedState, + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + storage?: TerminalScrollbackSnapshotStorage +): OrcadMigrationSourceWorkspaceSessionInspection { + try { + return collectWorkspaceSession(state, source, catalog, storage) + } catch (error) { + console.warn('[migration] Unreadable workspace session blocks the move:', error) + return { payload: undefined, snapshots: [], blockedCount: 1 } + } +} + +function collectWorkspaceSession( + state: PersistedState, + source: OrcadMigrationManifestSource, + catalog: OrcadMigrationCatalogPayload, + storage?: TerminalScrollbackSnapshotStorage +): OrcadMigrationSourceWorkspaceSessionInspection { + const sourceScope = createOrcadMigrationSourceScope({ source, catalog, repos: state.repos }) + const fragments: WorkspaceSessionState[] = [] + const snapshots: OrcadMigrationTerminalScrollbackSnapshot[] = [] + let blockedCount = 0 + for (const [partitionId, session] of sessionPartitions(state, LOCAL_EXECUTION_HOST_ID)) { + const scope = orcadMigrationPartitionScope(sourceScope, partitionId) + const sourceHostPartition = partitionId === scope.hostId + const terminalTabIds = collectOwnedTerminalTabIds(session, scope) + blockedCount += countUnsupportedSessionState( + session, + scope, + sourceHostPartition, + terminalTabIds + ) + blockedCount += countUnrepresentableMarkdownState(session, scope, sourceHostPartition) + const fragment = extractSessionOwnersForTransfer(session, { + mapOwnerKey: (ownerKey) => + orcadMigrationOwnerMatchesScope(ownerKey, scope) + ? unqualifyOrcadMigrationOwnerKey(ownerKey) + : null, + mapWorktreeId: unqualifyOrcadMigrationOwnerKey, + projectSessionFocus: sourceHostPartition + ? ({ source, transferred, terminalTabIds }) => + projectDormantSessionFocus(source, transferred, scope, terminalTabIds) + : undefined, + projectSleepingAgentSession: (record) => + transferableSleepingAgentSession(record, session, scope, terminalTabIds) + ? { + ...structuredClone(record), + worktreeId: unqualifyOrcadMigrationOwnerKey(record.worktreeId), + connectionId: null + } + : null + }) + if (!sourceHostPartition) { + // Client focus is not host state, so the destination never carries it. + fragment.activeWorktreeId = null + delete fragment.activeWorkspaceKey + } + dropOrcadMigrationTerminalBindings(fragment) + if (hasTransferredSessionState(fragment)) { + const projected = projectOrcadMigrationSessionScrollback( + projectSessionToDestination(fragment, scope), + storage + ) + blockedCount += projected.blockedCount + snapshots.push(...projected.snapshots) + fragments.push(projected.session) + } + } + if (hasDuplicateOrcadMigrationScrollbackDescriptors(snapshots)) { + blockedCount += 1 + } + if (blockedCount > 0 || fragments.length === 0) { + return { payload: undefined, snapshots: [], blockedCount } + } + const merged = mergeSessionFragments(fragments) + return merged + ? { payload: merged, snapshots, blockedCount: 0 } + : { payload: undefined, snapshots: [], blockedCount: 1 } +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-source-worktree-metadata.ts b/src/main/persistence/migrating-orcad-catalog/orcad-source-worktree-metadata.ts new file mode 100644 index 00000000000..ba2bfc9b2d0 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-source-worktree-metadata.ts @@ -0,0 +1,127 @@ +import type { PersistedState } from '../../../shared/persisted-state-types' +import type { WorktreeMeta } from '../../../shared/worktree/meta-types' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { serializeOrcadMigrationValue } from '../../../shared/orcad-migration-manifest' +import { canonicalWorktreeIdentity } from '../../../shared/worktree/identity' +import { + getExecutionHostIdFromWorktreeHostIdentity, + isWorktreeHostIdentity +} from '../../../shared/worktree/host-qualified-identity' +import { pruneUnreferencedWorktreeIdentityMeta } from '../loading-store/worktree-identity-metadata' +import { + createOrcadMigrationSourceScope, + orcadMigrationOwnerMatchesScope, + unqualifyOrcadMigrationOwnerKey, + type OrcadMigrationSourceScope +} from './orcad-source-scope' + +export function inspectOrcadSourceWorktreeMetadata( + state: PersistedState, + scope: OrcadMigrationSourceScope +) { + const rows: { sourceKey: string; meta: WorktreeMeta }[] = [] + let blockedCount = 0 + for (const [sourceKey, meta] of Object.entries(state.worktreeMeta)) { + const host = getExecutionHostIdFromWorktreeHostIdentity(sourceKey) + if ((host && host !== scope.hostId) || (meta.hostId && meta.hostId !== scope.hostId)) { + if (host === scope.hostId || meta.hostId === scope.hostId) { + blockedCount++ + } + continue + } + if (orcadMigrationOwnerMatchesScope(sourceKey, scope, meta.hostId)) { + rows.push({ sourceKey, meta }) + } else if (meta.hostId === scope.hostId || host === scope.hostId) { + blockedCount++ + } + } + const referenced = new Set(Object.values(state.worktreeIdentityAliases ?? {}).flat()) + for (const [identity, meta] of Object.entries(state.worktreeMetaByIdentity ?? {})) { + if (meta.hostId === scope.hostId && !referenced.has(identity)) { + blockedCount++ + } + } + for (const [alias, identities] of Object.entries(state.worktreeIdentityAliases ?? {})) { + if (getExecutionHostIdFromWorktreeHostIdentity(alias) !== scope.hostId) { + continue + } + if (isDanglingAlias(state, identities) && orcadMigrationOwnerMatchesScope(alias, scope)) { + continue // Names no metadata (its legacy row, if any, moves instead): nothing to move. + } + const meta = identities.length === 1 ? state.worktreeMetaByIdentity?.[identities[0]] : undefined + const worktreeId = unqualifyOrcadMigrationOwnerKey(alias) + if ( + isWorktreeHostIdentity(worktreeId) || + !orcadMigrationOwnerMatchesScope(alias, scope) || + !meta || + !meta.instanceId || + (meta.hostId !== undefined && meta.hostId !== scope.hostId) || + identities[0] !== + canonicalWorktreeIdentity({ + worktreeId, + executionHostId: scope.hostId, + instanceId: meta.instanceId + }) + ) { + blockedCount++ + continue + } + const legacy = rows.filter( + (row) => unqualifyOrcadMigrationOwnerKey(row.sourceKey) === worktreeId + ) + if ( + legacy.length > 1 || + (legacy.length === 1 && + serializeOrcadMigrationValue({ ...legacy[0].meta, hostId: scope.hostId }) !== + serializeOrcadMigrationValue({ ...meta, hostId: scope.hostId })) + ) { + // Neither representation may silently discard data held only by its competing row. + blockedCount++ + continue + } + if (legacy.length === 0) { + rows.push({ sourceKey: alias, meta: { ...meta, hostId: scope.hostId } }) + } + } + return { rows, blockedCount } +} + +export function subtractOrcadSourceWorktreeMetadata( + state: PersistedState, + manifest: OrcadMigrationManifest +) { + const scope = createOrcadMigrationSourceScope({ + source: manifest.source, + catalog: manifest.payload, + repos: state.repos + }) + const entries = manifest.payload.dormantState?.worktreeMeta ?? [] + // Rows a downgraded build added in a moved project go too: the server's copy of it wins. + const added = inspectOrcadSourceWorktreeMetadata(state, scope).rows.map((row) => row.sourceKey) + const sourceKeys = new Set([...entries.map((entry) => entry.sourceKey), ...added]) + const worktreeIds = new Set([ + ...entries.map((entry) => entry.worktreeId), + ...added.map(unqualifyOrcadMigrationOwnerKey) + ]) + const removedIdentities = new Set() + for (const [alias, identities] of Object.entries(state.worktreeIdentityAliases ?? {})) { + if ( + getExecutionHostIdFromWorktreeHostIdentity(alias) === scope.hostId && + (worktreeIds.has(unqualifyOrcadMigrationOwnerKey(alias)) || + (isDanglingAlias(state, identities) && orcadMigrationOwnerMatchesScope(alias, scope))) + ) { + identities.forEach((identity) => removedIdentities.add(identity)) + delete state.worktreeIdentityAliases?.[alias] + } + } + sourceKeys.forEach((sourceKey) => delete state.worktreeMeta[sourceKey]) + pruneUnreferencedWorktreeIdentityMeta(state, removedIdentities) +} + +/** An alias whose identities hold no metadata, as a profile that lost them leaves behind. */ +function isDanglingAlias(state: PersistedState, identities: readonly string[]): boolean { + return identities.every((identity) => { + const meta = state.worktreeMetaByIdentity?.[identity] + return !meta || Object.keys(meta).length === 0 + }) +} diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.test.ts b/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.test.ts new file mode 100644 index 00000000000..80f270835a6 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../../shared/constants' +import type { OrcadMigrationManifest } from '../../../shared/orcad-migration-manifest' +import { assertOrcadMigrationStagedCatalogClaims } from './orcad-staged-catalog-claims' + +const LEAF = '11111111-1111-4111-8111-111111111111' + +function manifest(id: string): OrcadMigrationManifest { + return { + version: 1, + migrationId: id, + manifestSha256: id.repeat(64), + createdAt: '2026-09-06T00:00:00.000Z', + source: { sshTargetId: id, sshTargetGeneration: 1, targetLabel: id }, + payload: { + repositories: [{ id, path: `/srv/${id}`, displayName: id, badgeColor: '', addedAt: 1 }], + projectGroups: [], + folderWorkspaces: [], + dormantState: { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [], + workspaceSession: getDefaultWorkspaceSession() + } + } + } +} + +function assertClaims(next: OrcadMigrationManifest, prior: OrcadMigrationManifest) { + return assertOrcadMigrationStagedCatalogClaims(next, [ + { + version: 1, + manifest: prior, + stagedAt: prior.createdAt + } + ]) +} + +describe('staged migration catalog claims', () => { + it('allows disjoint stages and exact same-transaction retries without mutation', () => { + const first = manifest('a') + const second = manifest('b') + const before = structuredClone({ first, second }) + expect(() => assertClaims(second, first)).not.toThrow() + expect(() => assertClaims(structuredClone(first), first)).not.toThrow() + expect({ first, second }).toEqual(before) + }) + + it('rejects a reused transaction ID with a different digest', () => { + const first = manifest('a') + expect(() => assertClaims({ ...first, manifestSha256: 'b'.repeat(64) }, first)).toThrow( + 'orcad_migration_id_reused_with_different_manifest' + ) + }) + + it('reserves the repository path even when another migration changes its ID', () => { + const first = manifest('a') + const second = manifest('b') + second.payload.repositories[0].path = first.payload.repositories[0].path + expect(() => assertClaims(second, first)).toThrow( + 'orcad_migration_staged_claim_conflict:repository-path:/srv/a' + ) + }) + + it('reserves folder identity without treating a shared folder path as identity', () => { + const first = manifest('a') + const second = manifest('b') + const folder = { + id: 'folder', + projectGroupId: 'group', + name: 'Folder', + folderPath: '/srv/folder', + connectionId: null, + linkedTask: null, + linkedTaskSourceContext: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 1, + createdAt: 1, + updatedAt: 1 + } + first.payload.folderWorkspaces = [folder] + second.payload.folderWorkspaces = [{ ...folder, id: 'distinct-folder' }] + expect(() => assertClaims(second, first)).not.toThrow() + second.payload.folderWorkspaces = [{ ...folder }] + expect(() => assertClaims(second, first)).toThrow( + 'orcad_migration_staged_claim_conflict:folder-workspace:folder' + ) + }) + + it.each(['topology', 'scrollback'] as const)( + 'reserves a stable pane claimed by another tab through %s', + (claim) => { + const first = manifest('a') + const second = manifest('b') + const firstSession = first.payload.dormantState!.workspaceSession! + firstSession.terminalLayoutsByTabId.first = { + root: claim === 'topology' ? { type: 'leaf', leafId: LEAF } : null, + activeLeafId: null, + expandedLeafId: null, + ...(claim === 'scrollback' ? { scrollbackRefsByLeafId: { [LEAF]: 'snapshot' } } : {}) + } + second.payload.dormantState!.workspaceSession!.terminalLayoutsByTabId.second = { + root: { type: 'leaf', leafId: LEAF }, + activeLeafId: null, + expandedLeafId: null + } + const before = structuredClone({ first, second }) + expect(() => assertClaims(second, first)).toThrow( + `orcad_migration_staged_claim_conflict:session:terminal-leaf:${LEAF}` + ) + expect({ first, second }).toEqual(before) + } + ) + + it('reserves an empty tab layout before it contains any pane', () => { + const first = manifest('a') + const second = manifest('b') + for (const entry of [first, second]) { + entry.payload.dormantState!.workspaceSession!.terminalLayoutsByTabId.tab = { + root: null, + activeLeafId: null, + expandedLeafId: null + } + } + expect(() => assertClaims(second, first)).toThrow( + 'orcad_migration_staged_claim_conflict:session:terminal-layout:tab' + ) + }) +}) diff --git a/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.ts b/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.ts new file mode 100644 index 00000000000..631a7662157 --- /dev/null +++ b/src/main/persistence/migrating-orcad-catalog/orcad-staged-catalog-claims.ts @@ -0,0 +1,61 @@ +import { normalizeRuntimePathForComparison } from '../../../shared/cross-platform-path' +import type { + OrcadMigrationManifest, + OrcadMigrationStagedCatalog +} from '../../../shared/orcad-migration-manifest' +import { collectOrcadMigrationSessionEntityOwners } from './orcad-destination-workspace-session' + +export function assertOrcadMigrationStagedCatalogClaims( + manifest: OrcadMigrationManifest, + staged: readonly OrcadMigrationStagedCatalog[] +): void { + const requested = catalogClaims(manifest) + for (const entry of staged) { + if (entry.manifest.migrationId === manifest.migrationId) { + if (entry.manifest.manifestSha256 !== manifest.manifestSha256) { + throw new Error('orcad_migration_id_reused_with_different_manifest') + } + continue + } + for (const claim of catalogClaims(entry.manifest)) { + if (requested.has(claim)) { + throw new Error(`orcad_migration_staged_claim_conflict:${claim}`) + } + } + } +} + +function catalogClaims(manifest: OrcadMigrationManifest): Set { + const { payload } = manifest + const claims = new Set() + for (const repo of payload.repositories) { + claims.add(`repository:${repo.id}`) + claims.add(`repository-path:${normalizeRuntimePathForComparison(repo.path)}`) + } + payload.projectGroups.forEach((group) => claims.add(`project-group:${group.id}`)) + payload.folderWorkspaces.forEach((folder) => claims.add(`folder-workspace:${folder.id}`)) + const dormant = payload.dormantState + if (dormant) { + dormant.worktreeMeta.forEach((entry) => claims.add(`worktree-meta:${entry.worktreeId}`)) + dormant.worktreeLineage.forEach((entry) => claims.add(`worktree-lineage:${entry.worktreeId}`)) + dormant.workspaceLineage.forEach((entry) => + claims.add(`workspace-lineage:${entry.childWorkspaceKey}`) + ) + dormant.sparsePresets.forEach((preset) => + claims.add(`sparse-preset:${preset.repoId}:${preset.id}`) + ) + dormant.automations?.forEach((automation) => claims.add(`automation:${automation.id}`)) + dormant.automationRuns?.forEach((run) => claims.add(`automation-run:${run.id}`)) + if (dormant.workspaceSession) { + for (const tabId of Object.keys(dormant.workspaceSession.terminalLayoutsByTabId ?? {})) { + claims.add(`session:terminal-layout:${tabId}`) + } + for (const entity of collectOrcadMigrationSessionEntityOwners( + dormant.workspaceSession + ).keys()) { + claims.add(`session:${entity}`) + } + } + } + return claims +} diff --git a/src/main/persistence/terminal-topology/terminal-topology-boundary-ratchet.test.ts b/src/main/persistence/terminal-topology/terminal-topology-boundary-ratchet.test.ts index 27aa67c7d6e..575a6a31906 100644 --- a/src/main/persistence/terminal-topology/terminal-topology-boundary-ratchet.test.ts +++ b/src/main/persistence/terminal-topology/terminal-topology-boundary-ratchet.test.ts @@ -53,6 +53,8 @@ const ALLOWED_REFERENCES: Record = { patchWorkspaceSession: ['ipc/session.ts'], stageWorkspaceSessionBeforeUnload: ['ipc/renderer-shutdown-checkpoint.ts'], setWorkspaceSessionForWorktree: [ + // Headless editor-tab retirement, like the headless mobile-session tab writers below. + 'runtime/mobile-session-editor-projection.ts', 'runtime/orca-runtime-adopt-terminal-orphans-from-inventory.ts', 'runtime/orca-runtime-apply-mobile-session-tab-navigation.ts', 'runtime/orca-runtime-build-headless-mobile-session-browser-tabs.ts', diff --git a/src/main/persistence/tracking-repos/deregistered-repo-residue.ts b/src/main/persistence/tracking-repos/deregistered-repo-residue.ts index e26d43c6cc7..5e2a4de40a4 100644 --- a/src/main/persistence/tracking-repos/deregistered-repo-residue.ts +++ b/src/main/persistence/tracking-repos/deregistered-repo-residue.ts @@ -56,6 +56,12 @@ export function collectDeregisteredRepoIds(state: PersistedState): Set { retainOwner(getWorktreeIdFromHostIdentity(alias)) } } + // A staged catalog owns recovery rows before its repositories become registered. + for (const staged of state.orcadMigrationStagedCatalogs ?? []) { + for (const repo of staged.manifest.payload.repositories) { + liveRepoIds.add(repo.id) + } + } const orphanRepoIds = new Set() // Only a full `::` locator seeds the set. A bare key -- a folder workspace id, a // repo-keyed topology revision, a test-shaped locator -- cannot be told apart from a repo id, and diff --git a/src/main/providers/ssh-pty-errors.ts b/src/main/providers/ssh-pty-errors.ts index 4d312caa8b1..24f0467263e 100644 --- a/src/main/providers/ssh-pty-errors.ts +++ b/src/main/providers/ssh-pty-errors.ts @@ -8,6 +8,9 @@ export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH' * reply is host evidence of the opposite). */ export const SSH_PTY_SOURCE_RESTORE_REQUIRED_ERROR = 'SSH_PTY_SOURCE_RESTORE_REQUIRED' +/** The id is unknown to this relay while an older build's relay for the target is still live. + * Deliberately not `SSH_SESSION_EXPIRED`, so the pane keeps its binding instead of respawning. */ +export const SSH_PTY_HELD_BY_PREVIOUS_RELAY_ERROR = 'SSH_PTY_HELD_BY_PREVIOUS_RELAY' export function isSshPtyNotFoundError(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error) @@ -65,3 +68,12 @@ export class SshPtyProvenExitedOnRelayError extends SshPtyAbsentFromRelayError { export function isSshPtyProvenExitedOnRelayError(error: unknown): boolean { return error instanceof SshPtyProvenExitedOnRelayError } + +/** Raised in place of {@link SshPtyAbsentFromRelayError} while an older build's relay for the + * target may still run the PTY, so neither the lease nor the pane binding is retired. */ +export class SshPtyHeldByPreviousRelayError extends Error { + constructor(relayPtyId: string) { + super(`${SSH_PTY_HELD_BY_PREVIOUS_RELAY_ERROR}: ${relayPtyId}`) + this.name = 'SshPtyHeldByPreviousRelayError' + } +} diff --git a/src/main/providers/ssh-pty-legacy-relay-delegation.test.ts b/src/main/providers/ssh-pty-legacy-relay-delegation.test.ts new file mode 100644 index 00000000000..abcbc7ce045 --- /dev/null +++ b/src/main/providers/ssh-pty-legacy-relay-delegation.test.ts @@ -0,0 +1,337 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { SshPtyHeldByPreviousRelayError } from './ssh-pty-errors' +import { SshPtyProvider } from './ssh-pty-provider' +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import { createMockMux, type MockMultiplexer } from './ssh-pty-provider-mock-multiplexer' +import { + attachHeldPtyThroughPreviousRelay, + type SshPtyLegacyRelayRouting +} from './ssh-pty-legacy-relay-delegation' +import { SshLegacyRelayRouter } from '../ssh/ssh-legacy-relay-router' + +const HELD = 'ssh:target-1@@pty2:old:1' +const CURRENT = 'ssh:target-1@@pty2:new:1' + +function asMux(mock: MockMultiplexer): SshChannelMultiplexer { + mock.onNotification.mockReturnValue(() => {}) + // The provider reaches only the stubbed members on these paths. + return Object.assign(Object.create(null), mock) +} + +function currentRelayHoldsTheId(): void { + // Before setup: the delegation captures the provider's own spawn when it is installed. + vi.spyOn(SshPtyProvider.prototype, 'spawn').mockImplementationOnce(async () => { + throw new SshPtyHeldByPreviousRelayError('pty2:old:1') + }) +} + +function setup() { + const currentMux = createMockMux() + const legacyMux = createMockMux() + const provider = new SshPtyProvider('target-1', asMux(currentMux), undefined, 7) + const legacy = new SshPtyProvider('target-1', asMux(legacyMux), undefined, 8) + const served = new Set() + const release = vi.fn(() => served.clear()) + const routing: SshPtyLegacyRelayRouting = { + attach: vi.fn(async (id: string) => { + served.add(id) + return { provider: legacy, release } + }), + providerFor: (id) => (served.has(id) ? legacy : undefined), + track: (id, request) => (served.has(id) ? request(legacy) : undefined), + // The old relay holds HELD; a short-lived route stops it there. + stopHeld: vi.fn(async (id: string, stop: (provider: SshPtyProvider) => Promise) => { + if (id !== HELD) { + return { stopped: false as const, reachable: true } + } + await stop(legacy) + return { stopped: true as const } + }), + onExit: () => () => {}, + servedProviders: () => (served.size > 0 ? [legacy] : []), + dispose: vi.fn() + } + provider.setLegacyRelayRouting(routing) + return { provider, legacy, currentMux, legacyMux, routing, release, served } +} + +describe('SshPtyProvider delegation to an earlier build relay', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('reattaches a PTY the current relay holds for an older relay through that relay', async () => { + currentRelayHoldsTheId() + const { provider, legacy, routing } = setup() + const legacySpawn = vi.spyOn(legacy, 'spawn').mockResolvedValue({ id: HELD, isReattach: true }) + + await expect(provider.spawn({ sessionId: HELD, cols: 80, rows: 24 })).resolves.toEqual({ + id: HELD, + isReattach: true + }) + expect(routing.attach).toHaveBeenCalledWith(HELD) + expect(legacySpawn).toHaveBeenCalledWith({ sessionId: HELD, cols: 80, rows: 24 }) + }) + + it('keeps the held refusal when no older relay holds the PTY', async () => { + currentRelayHoldsTheId() + const { provider, routing } = setup() + vi.mocked(routing.attach).mockResolvedValueOnce(null) + + await expect(provider.spawn({ sessionId: HELD, cols: 80, rows: 24 })).rejects.toBeInstanceOf( + SshPtyHeldByPreviousRelayError + ) + }) + + it('releases the route when the attach through it fails', async () => { + currentRelayHoldsTheId() + const { provider, legacy, release } = setup() + vi.spyOn(legacy, 'spawn').mockRejectedValue(new Error('PTY "pty2:old:1" not found')) + + await expect(provider.spawn({ sessionId: HELD, cols: 80, rows: 24 })).rejects.toThrow( + 'not found' + ) + expect(release).toHaveBeenCalled() + }) + + it('sends a served PTY’s input, size and acks to the older relay only', () => { + const { provider, currentMux, legacyMux, served } = setup() + served.add(HELD) + + provider.resize(HELD, 100, 30) + provider.acknowledgeDataEvent(HELD, 64) + provider.resize(CURRENT, 90, 20) + + expect(legacyMux.notify).toHaveBeenCalledWith('pty.resize', { + id: 'pty2:old:1', + cols: 100, + rows: 30 + }) + expect(legacyMux.notify).toHaveBeenCalledWith('pty.ackData', { + id: 'pty2:old:1', + charCount: 64 + }) + expect(currentMux.notify).toHaveBeenCalledWith('pty.resize', { + id: 'pty2:new:1', + cols: 90, + rows: 20 + }) + expect(currentMux.notify).not.toHaveBeenCalledWith( + 'pty.resize', + expect.objectContaining({ id: 'pty2:old:1' }) + ) + expect(provider.hasPty(HELD)).toBe(true) + }) + + it('lists served PTYs beside the current relay’s own', async () => { + const { provider, currentMux, legacyMux, served } = setup() + served.add(HELD) + currentMux.request.mockResolvedValueOnce([{ id: 'pty2:new:1', cwd: '/', title: 'sh' }]) + legacyMux.request.mockResolvedValueOnce([ + { id: 'pty2:old:1', cwd: '/', title: 'sh' }, + { id: 'pty2:old:2', cwd: '/', title: 'sh' } + ]) + + const listed = await provider.listProcesses() + + expect(listed.map((row) => row.id)).toEqual([CURRENT, HELD]) + }) + + it('answers owner listings from the serving relay and never serializes its PTYs here', async () => { + const { provider, legacy, currentMux, served } = setup() + served.add(HELD) + const legacyListings = vi + .spyOn(legacy, 'providesAgentSessionOwnerListings') + .mockReturnValue(true) + currentMux.request.mockResolvedValueOnce('[]') + + expect(provider.providesAgentSessionOwnerListings(HELD)).toBe(true) + expect(legacyListings).toHaveBeenCalledWith(HELD) + await provider.serialize([HELD, CURRENT]) + expect(currentMux.request).toHaveBeenCalledWith('pty.serialize', { ids: ['pty2:new:1'] }) + }) + + it('refuses a second routing install', () => { + const { provider, routing } = setup() + expect(() => provider.setLegacyRelayRouting(routing)).toThrow( + 'ssh_pty_legacy_relay_routing_already_installed' + ) + }) + + it('fails the listing when an older relay cannot answer, so its PTYs read unverifiable', async () => { + const { provider, currentMux, legacyMux, served } = setup() + served.add(HELD) + currentMux.request.mockResolvedValueOnce([{ id: 'pty2:new:1', cwd: '/', title: 'sh' }]) + legacyMux.request.mockRejectedValueOnce(new Error('relay timed out')) + + await expect(provider.listProcesses()).rejects.toThrow('relay timed out') + }) + + it('closes its routes with the provider', () => { + const { provider, routing } = setup() + provider.dispose() + expect(routing.dispose).toHaveBeenCalled() + }) + + it('reads a class router through its own instance, as the session installs it', () => { + const currentMux = createMockMux() + const provider = new SshPtyProvider('target-1', asMux(currentMux), undefined, 7) + provider.setLegacyRelayRouting( + new SshLegacyRelayRouter({ + targetId: 'target-1', + endpoints: async () => [], + openRoute: async () => null + }) + ) + + provider.acknowledgeDataEvent(CURRENT, 32) + + expect(currentMux.notify).toHaveBeenCalledWith('pty.ackData', { + id: 'pty2:new:1', + charCount: 32 + }) + }) + + it('refuses input to a held PTY no older relay serves, instead of dropping it silently', async () => { + currentRelayHoldsTheId() + const { provider, currentMux, routing } = setup() + vi.mocked(routing.attach).mockResolvedValueOnce(null) + await expect(provider.spawn({ sessionId: HELD, cols: 80, rows: 24 })).rejects.toBeInstanceOf( + SshPtyHeldByPreviousRelayError + ) + + expect(provider.write(HELD, 'ls\n')).toBe(false) + expect(provider.write('pty2:old:1', 'ls\n')).toBe(false) + await expect(provider.writeWithSettlement(HELD, 'ls\n')).resolves.toMatchObject({ + outcome: 'refused', + reason: 'endpoint_awaiting_recovery' + }) + expect(currentMux.notify).not.toHaveBeenCalledWith('pty.data', expect.anything()) + expect(provider.write(CURRENT, 'ls\n')).toBe(true) + }) + + it('delivers input once a reconnect routes the held PTY to the older relay', async () => { + const { provider, legacyMux, routing } = setup() + vi.mocked(routing.attach).mockResolvedValueOnce(null) + await expect(attachHeldPtyThroughPreviousRelay(provider, HELD)).resolves.toBeNull() + expect(provider.write(HELD, 'ls\n')).toBe(false) + + legacyMux.request.mockResolvedValueOnce({ incarnationId: 'inc-old' }) + await expect(attachHeldPtyThroughPreviousRelay(provider, HELD)).resolves.toMatchObject({ + incarnationId: 'inc-old' + }) + expect(provider.write(HELD, 'ls\n')).toBe(true) + expect(legacyMux.notify).toHaveBeenCalledWith('pty.data', { id: 'pty2:old:1', data: 'ls\n' }) + }) + + it('stops a served PTY on the older relay, never the current one', async () => { + const { provider, currentMux, legacyMux, served } = setup() + served.add(HELD) + + await provider.shutdown(HELD, { immediate: true }) + + expect(legacyMux.request).toHaveBeenCalledWith( + 'pty.shutdown', + expect.objectContaining({ id: 'pty2:old:1', immediate: true }), + undefined + ) + expect(currentMux.request).not.toHaveBeenCalledWith( + 'pty.shutdown', + expect.anything(), + undefined + ) + }) + + it('finds the older relay for a stop on a PTY no pane resumed this connection', async () => { + const { provider, legacyMux, routing } = setup() + + await provider.shutdown(HELD, { immediate: false }) + + expect(routing.stopHeld).toHaveBeenCalledWith(HELD, expect.any(Function)) + expect(routing.attach).not.toHaveBeenCalled() + expect(legacyMux.request).toHaveBeenCalledWith( + 'pty.shutdown', + expect.objectContaining({ id: 'pty2:old:1' }), + undefined + ) + }) + + it('refuses a stop for a held PTY no older relay can serve, instead of reporting it stopped', async () => { + const { provider, currentMux, routing } = setup() + vi.mocked(routing.attach).mockResolvedValue(null) + vi.mocked(routing.stopHeld).mockResolvedValue({ stopped: false, reachable: true }) + await expect(attachHeldPtyThroughPreviousRelay(provider, HELD)).resolves.toBeNull() + + await expect(provider.shutdown(HELD, { immediate: true })).rejects.toBeInstanceOf( + SshPtyHeldByPreviousRelayError + ) + expect(currentMux.request).not.toHaveBeenCalledWith( + 'pty.shutdown', + expect.anything(), + undefined + ) + }) + + it('hears exits the older relays report, so a stop can confirm them', () => { + const currentMux = createMockMux() + const provider = new SshPtyProvider('target-1', asMux(currentMux), undefined, 7) + const exitListeners: Parameters[0][] = [] + provider.setLegacyRelayRouting({ + attach: async () => null, + providerFor: () => undefined, + track: () => undefined, + stopHeld: async () => ({ stopped: false, reachable: true }), + onExit: (listener) => { + exitListeners.push(listener) + return () => {} + }, + servedProviders: () => [], + dispose: () => {} + }) + const heard = vi.fn() + provider.onExit(heard) + + const exit = { id: HELD, code: 0, providerGeneration: 8, ptyIncarnation: 'inc-old' } + exitListeners.forEach((listener) => listener(exit)) + + expect(heard).toHaveBeenCalledWith(exit) + }) + + it('refuses a stop when an older relay that may hold the PTY cannot be asked', async () => { + const { provider, currentMux, routing } = setup() + vi.mocked(routing.stopHeld).mockResolvedValue({ stopped: false, reachable: false }) + + await expect(provider.shutdown(CURRENT, { immediate: true })).rejects.toBeInstanceOf( + SshPtyHeldByPreviousRelayError + ) + expect(currentMux.request).not.toHaveBeenCalledWith( + 'pty.shutdown', + expect.anything(), + undefined + ) + }) + + it('stops an id no relay holds on the current relay', async () => { + const { provider, currentMux } = setup() + + await provider.shutdown(CURRENT, { immediate: true }) + + expect(currentMux.request).toHaveBeenCalledWith( + 'pty.shutdown', + expect.objectContaining({ id: 'pty2:new:1' }), + undefined + ) + }) + + it('reads a stop whose bridge dropped mid-request as unverifiable, not failed', async () => { + const { provider, legacyMux, served } = setup() + served.add(HELD) + legacyMux.request.mockRejectedValueOnce( + Object.assign(new Error('Multiplexer disposed'), { code: 'DISPOSED' }) + ) + + await expect(provider.shutdown(HELD, { immediate: true })).rejects.toBeInstanceOf( + SshPtyHeldByPreviousRelayError + ) + }) +}) diff --git a/src/main/providers/ssh-pty-legacy-relay-delegation.ts b/src/main/providers/ssh-pty-legacy-relay-delegation.ts new file mode 100644 index 00000000000..55224e44051 --- /dev/null +++ b/src/main/providers/ssh-pty-legacy-relay-delegation.ts @@ -0,0 +1,255 @@ +/** + * Lets the target's provider hand a PTY to an earlier build's relay that still runs it. + * + * Every per-PTY caller resolves the target's one registered provider, so routing lives here rather + * than at each call site: a reattach the current relay answered with + * {@link SshPtyHeldByPreviousRelayError} is retried through the old relay, and once that pane is + * served there, every later operation on its id goes to the same relay. + */ +import { SshPtyHeldByPreviousRelayError } from './ssh-pty-errors' +import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' +import type { SshPtyProvider } from './ssh-pty-provider' +import type { SshPtyAttachResult } from './ssh-pty-session-reattach' +import type { PtySpawnOptions, PtySpawnResult } from './types' +import { writeRefused } from '../../shared/pty-write-settlement' + +export type SshPtyLegacyRelayRouting = { + /** The served route for a held PTY, or null when no older relay holds it. */ + attach: (appPtyId: string) => Promise<{ provider: SshPtyProvider; release: () => void } | null> + providerFor: (appPtyId: string) => SshPtyProvider | undefined + /** Runs a request for a served PTY on its route, keeping the route open until it settles. */ + track: ( + appPtyId: string, + request: (provider: SshPtyProvider) => Promise + ) => Promise | undefined + /** Stops a PTY an older relay holds through a short-lived route; see SshLegacyRelayRouter. */ + stopHeld: ( + appPtyId: string, + stop: (provider: SshPtyProvider) => Promise + ) => Promise<{ stopped: true } | { stopped: false; reachable: boolean }> + /** Exits the older relays report for served PTYs. */ + onExit: (listener: Parameters[0]) => () => void + servedProviders: () => SshPtyProvider[] + dispose: () => void +} + +const routingByProvider = new WeakMap() +/** + * App PTY ids an older relay may run but no route serves. The current relay drops input for an id it + * never minted without a word, so writes to these are refused instead of reported as delivered. + */ +const heldByProvider = new WeakMap>() + +function recordHeld(provider: SshPtyProvider, appPtyId: string, held: boolean): void { + const ids = heldByProvider.get(provider) + if (held) { + ids?.add(appPtyId) + } else { + ids?.delete(appPtyId) + } +} + +/** + * The reconnect path's counterpart to the delegated spawn: a reattach the current relay disowned is + * retried through the older relay that holds it. Null when no older relay serves the PTY. + */ +export async function attachHeldPtyThroughPreviousRelay( + provider: SshPtyProvider, + appPtyId: string, + expected?: { paneKey?: string; tabId?: string } +): Promise { + const served = await routingByProvider.get(provider)?.attach(appPtyId) + recordHeld(provider, appPtyId, !served) + if (!served) { + return null + } + try { + return await served.provider.attachForReconnect(appPtyId, expected) + } catch (error) { + served.release() + recordHeld(provider, appPtyId, true) + throw error + } +} + +/** A stop whose bridge dropped mid-request may or may not have landed: unverifiable, not failed. */ +async function unverifiableIfBridgeLost(stop: Promise, relayPtyId: string): Promise { + try { + return await stop + } catch (error) { + const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined + if (code === 'DISPOSED' || code === 'CONNECTION_LOST') { + throw new SshPtyHeldByPreviousRelayError(relayPtyId) + } + throw error + } +} + +export function installSshPtyLegacyRelayDelegation( + provider: SshPtyProvider, + routing: SshPtyLegacyRelayRouting +): void { + // Why: a second install would wrap the wrappers and route through two routing tables. + if (routingByProvider.has(provider)) { + throw new Error('ssh_pty_legacy_relay_routing_already_installed') + } + routingByProvider.set(provider, routing) + const held = new Set() + heldByProvider.set(provider, held) + const own = { + dispose: provider.dispose.bind(provider), + spawn: provider.spawn.bind(provider), + attach: provider.attach.bind(provider), + attachForReconnect: provider.attachForReconnect.bind(provider), + shutdown: provider.shutdown.bind(provider), + onExit: provider.onExit, + pauseProducer: provider.pauseProducer.bind(provider), + resumeProducer: provider.resumeProducer.bind(provider), + listProcesses: provider.listProcesses, + write: provider.write, + writeWithSettlement: provider.writeWithSettlement, + resize: provider.resize, + sendSignal: provider.sendSignal, + getCwd: provider.getCwd, + getInitialCwd: provider.getInitialCwd, + clearBuffer: provider.clearBuffer, + resetInputModes: provider.resetInputModes, + closeStartupQueryAuthority: provider.closeStartupQueryAuthority, + acknowledgeDataEvent: provider.acknowledgeDataEvent, + hasChildProcesses: provider.hasChildProcesses, + getForegroundProcess: provider.getForegroundProcess, + inspectProcess: provider.inspectProcess, + hasPty: provider.hasPty, + getAppliedSize: provider.getAppliedSize, + serialize: provider.serialize, + providesAgentSessionOwnerListings: provider.providesAgentSessionOwnerListings.bind(provider) + } + // Why normalized: the reconnect path names a PTY in relay form, panes in app form. + const routed = (id: string): SshPtyProvider | undefined => { + try { + return routing.providerFor(toAppSshPtyId(provider.getConnectionId(), id)) + } catch { + return undefined + } + } + + provider.dispose = () => { + routing.dispose() + own.dispose() + } + + provider.spawn = async (opts: PtySpawnOptions): Promise => { + try { + return await own.spawn(opts) + } catch (error) { + if (!(error instanceof SshPtyHeldByPreviousRelayError) || !opts.sessionId) { + throw error + } + const served = await routing.attach(opts.sessionId) + recordHeld(provider, opts.sessionId, !served) + if (!served) { + throw error + } + try { + return await served.provider.spawn(opts) + } catch (legacyError) { + served.release() + recordHeld(provider, opts.sessionId, true) + throw legacyError + } + } + } + provider.attach = (id) => routed(id)?.attach(id) ?? own.attach(id) + provider.attachForReconnect = (id, expected, recovery) => + routed(id)?.attachForReconnect(id, expected, recovery) ?? + own.attachForReconnect(id, expected, recovery) + // Why the stop finds its route first: the current relay answers a stop for an id it never minted + // as done, so a held PTY sent there was reported stopped while its shell kept running. + provider.shutdown = async (id, opts) => { + const appPtyId = toAppSshPtyId(provider.getConnectionId(), id) + const relayPtyId = toRelaySshPtyId(provider.getConnectionId(), id) + const stopOnOldRelay = (legacy: SshPtyProvider): Promise => legacy.shutdown(id, opts) + const servedStop = routing.track(appPtyId, stopOnOldRelay) + if (servedStop) { + return await unverifiableIfBridgeLost(servedStop, relayPtyId) + } + if (own.hasPty(appPtyId) || own.hasPty(relayPtyId)) { + return await own.shutdown(id, opts) + } + // A PTY no pane resumed this connection: a short-lived route stops it on the relay that runs it. + const held = await unverifiableIfBridgeLost( + routing.stopHeld(appPtyId, stopOnOldRelay), + relayPtyId + ) + if (held.stopped) { + return + } + if (!held.reachable || isHeldUnserved(id)) { + throw new SshPtyHeldByPreviousRelayError(relayPtyId) + } + return await own.shutdown(id, opts) + } + // Why merged: a stop waits for the exit of the PTY it stopped, which an older relay reports. + provider.onExit = (callback) => { + const stopOwn = own.onExit(callback) + const stopRouted = routing.onExit(callback) + return () => { + stopOwn() + stopRouted() + } + } + provider.pauseProducer = (id) => (routed(id) ?? own).pauseProducer(id) + provider.resumeProducer = (id) => (routed(id) ?? own).resumeProducer(id) + const isHeldUnserved = (id: string): boolean => { + try { + return held.has(toAppSshPtyId(provider.getConnectionId(), id)) + } catch { + return false + } + } + provider.write = (id, data) => + routed(id)?.write(id, data) ?? (isHeldUnserved(id) ? false : own.write(id, data)) + provider.writeWithSettlement = (id, data) => + routed(id)?.writeWithSettlement(id, data) ?? + (isHeldUnserved(id) + ? Promise.resolve(writeRefused('endpoint_awaiting_recovery')) + : own.writeWithSettlement(id, data)) + provider.resize = (id, cols, rows) => (routed(id) ?? own).resize(id, cols, rows) + provider.sendSignal = (id, signal) => + routed(id)?.sendSignal(id, signal) ?? own.sendSignal(id, signal) + provider.getCwd = (id) => routed(id)?.getCwd(id) ?? own.getCwd(id) + provider.getInitialCwd = (id) => routed(id)?.getInitialCwd(id) ?? own.getInitialCwd(id) + provider.clearBuffer = (id) => routed(id)?.clearBuffer(id) ?? own.clearBuffer(id) + provider.resetInputModes = (id) => routed(id)?.resetInputModes(id) ?? own.resetInputModes(id) + provider.closeStartupQueryAuthority = (id) => + routed(id)?.closeStartupQueryAuthority(id) ?? own.closeStartupQueryAuthority(id) + provider.acknowledgeDataEvent = (id, charCount) => + (routed(id) ?? own).acknowledgeDataEvent(id, charCount) + provider.hasChildProcesses = (id) => + routed(id)?.hasChildProcesses(id) ?? own.hasChildProcesses(id) + provider.getForegroundProcess = (id) => + routed(id)?.getForegroundProcess(id) ?? own.getForegroundProcess(id) + provider.inspectProcess = (id, options) => + routed(id)?.inspectProcess(id, options) ?? own.inspectProcess(id, options) + provider.hasPty = (id) => routed(id) !== undefined || own.hasPty(id) + provider.getAppliedSize = (id) => (routed(id) ?? own).getAppliedSize(id) + provider.providesAgentSessionOwnerListings = (id) => + (routed(id) ?? own).providesAgentSessionOwnerListings(id) + // Why not routed: revive replays onto this relay and would respawn a PTY the older one still runs. + provider.serialize = (ids) => own.serialize(ids.filter((id) => routed(id) === undefined)) + // Why merged, and rejecting when an older relay cannot answer: a served PTY missing from the + // listing reads as exited to inventory, and a relay we could not ask proves nothing. + provider.listProcesses = async (options) => { + const [current, ...previous] = await Promise.all([ + own.listProcesses(options), + ...routing + .servedProviders() + .map((legacy) => + legacy + .listProcesses(options) + .then((rows) => rows.filter((row) => routed(row.id) === legacy)) + ) + ]) + return [...current, ...previous.flat()] + } +} diff --git a/src/main/providers/ssh-pty-notification-recovery-activation.test.ts b/src/main/providers/ssh-pty-notification-recovery-activation.test.ts new file mode 100644 index 00000000000..e2198f77027 --- /dev/null +++ b/src/main/providers/ssh-pty-notification-recovery-activation.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it, vi } from 'vitest' +import { createSubscription, sourceActivation } from './ssh-pty-notification-routing-test-fixture' + +describe('subscribeSshPtyNotifications', () => { + it('accepts non-empty recovery from the activation checkpoint', () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + ) + + handler('pty.data', { + id: 'pty-1', + data: 'next', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + expect(onData).not.toHaveBeenCalled() + lease.commit() + expect(onData).toHaveBeenCalledWith( + expect.objectContaining({ + data: 'next', + source: expect.objectContaining({ sourceStartSu: 4, sourceEndSu: 8 }) + }) + ) + }) + + it('routes held and later recovery frames only to the private sink until commit', () => { + const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 }) + ) + const publishSource = (data: string, sourceEndSu: number): void => { + handler('pty.data', { + id: 'pty-1', + data, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu, + sourceLengthSu: 4 + }) + } + + publishSource('held', 8) + const recoveryLease = lease.transferToRecovery(onRecoveryData) + publishSource('next', 12) + + expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next']) + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + + recoveryLease.commit() + expect(onData).not.toHaveBeenCalled() + publishSource('live', 16) + + expect(onRecoveryData).toHaveBeenCalledTimes(2) + expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' })) + expect(livePtyIds).toContain('ssh:conn@@pty-1') + }) + + it('retires an exited private recovery when its activation commits', () => { + const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 })) + handler('pty.data', { + id: 'pty-1', + data: 'held', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 4, + sourceLengthSu: 4 + }) + const recoveryLease = lease.transferToRecovery(onRecoveryData) + + handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' }) + recoveryLease.commit() + handler('pty.data', { + id: 'pty-1', + data: 'late', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + expect(onRecoveryData).toHaveBeenCalledOnce() + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + expect(mux.request).toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' }) + ) + }) + + it('retires private recovery locally and restores the exact predecessor', () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + installReceivingActivation( + 'pty-1', + sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 }) + ).commit() + handler('pty.data', { + id: 'pty-1', + data: 'pre', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const replacement = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 3, + recoveryEndSu: 6 + }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + replacement.transferToRecovery(onRecoveryData).retire() + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) + expect(mux.request).not.toHaveBeenCalled() + }) + + it('rejects a stale activation without disturbing current continuity', () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit() + + expect(() => + installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 1, + ownerGeneration: 4, + deliveryToken: 'token-stale', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + ).toThrow('ssh_source_receiving_activation_stale') + + handler('pty.data', { + id: 'pty-1', + data: 'one', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + expect(onData).toHaveBeenCalledOnce() + }) + + it('drops provisional frames and settles cancellation before rollback completes', async () => { + const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'next', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + await expect(lease.rollback()).resolves.toBe(true) + + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + }) + + it('restores the exact prior cursor when a replacement rolls back after frames', async () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit() + handler('pty.data', { + id: 'pty-1', + data: 'pre', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const replacement = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + await replacement.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) + }) + + it('does not let an older lease rollback replace a newer activation', async () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const older = installReceivingActivation('pty-1', sourceActivation()) + const newer = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new' + }) + ) + + await older.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + newer.commit() + + expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + expect(mux.request).not.toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ deliveryToken: 'token-new' }) + ) + }) +}) diff --git a/src/main/providers/ssh-pty-notification-routing-recovery.test.ts b/src/main/providers/ssh-pty-notification-routing-recovery.test.ts new file mode 100644 index 00000000000..2ad66f62b18 --- /dev/null +++ b/src/main/providers/ssh-pty-notification-routing-recovery.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, it, vi } from 'vitest' +import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' + +type MockMux = { + onNotification: ReturnType + request: ReturnType +} + +function createSubscription() { + const mux: MockMux = { + onNotification: vi.fn(), + request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 })) + } + const dataListeners = new Set<(payload: { id: string; data: string }) => void>() + const replayListeners = new Set<(payload: { id: string; data: string }) => void>() + const exitListeners = new Set<(payload: { id: string; code: number }) => void>() + const livePtyIds = new Set() + const recordExit = vi.fn() + const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`) + const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`) + + const subscription = subscribeSshPtyNotifications({ + mux: mux as never, + toAppPtyId, + dataListeners: dataListeners as never, + replayListeners: replayListeners as never, + exitListeners: exitListeners as never, + livePtyIds, + recordExit, + providerGeneration: 7, + resolvePtyIncarnation, + peekPtyIncarnation: () => undefined + }) + const handler = mux.onNotification.mock.calls[0]?.[0] as ( + method: string, + params: Record + ) => void + if (!handler) { + throw new Error('notification handler was not registered') + } + return { + handler, + mux, + toAppPtyId, + dataListeners, + replayListeners, + exitListeners, + livePtyIds, + recordExit, + resolvePtyIncarnation, + installReceivingActivation: subscription.installReceivingActivation + } +} + +function sourceActivation( + overrides: Partial = {} +): PtySourceReceivingActivation { + return Object.freeze({ + status: 'pending', + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + checkpointSourceEndSu: 0, + recoveryEndSu: 0, + ...overrides + }) +} + +describe('SSH PTY notification recovery routing', () => { + it('rejects a stale activation without disturbing current continuity', () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit() + + expect(() => + installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 1, + ownerGeneration: 4, + deliveryToken: 'token-stale', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + ).toThrow('ssh_source_receiving_activation_stale') + + handler('pty.data', { + id: 'pty-1', + data: 'one', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + expect(onData).toHaveBeenCalledOnce() + }) + + it('drops provisional frames and settles cancellation before rollback completes', async () => { + const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'next', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + await expect(lease.rollback()).resolves.toBe(true) + + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + }) + + it('restores the exact prior cursor when a replacement rolls back after frames', async () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit() + handler('pty.data', { + id: 'pty-1', + data: 'pre', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const replacement = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + await replacement.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) + }) + + it('does not let an older lease rollback replace a newer activation', async () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const older = installReceivingActivation('pty-1', sourceActivation()) + const newer = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new' + }) + ) + + await older.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + newer.commit() + + expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + expect(mux.request).not.toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ deliveryToken: 'token-new' }) + ) + }) + + it('ignores PTY methods with missing ids', () => { + const { handler, toAppPtyId, dataListeners } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + + expect(() => handler('pty.data', { data: 'orphan' })).not.toThrow() + expect(toAppPtyId).not.toHaveBeenCalled() + expect(onData).not.toHaveBeenCalled() + }) + + it('leaves recovery and cancellation control methods to their dedicated handlers', () => { + const { + handler, + mux, + toAppPtyId, + dataListeners, + replayListeners, + exitListeners, + livePtyIds, + recordExit, + resolvePtyIncarnation + } = createSubscription() + const onData = vi.fn() + const onReplay = vi.fn() + const onExit = vi.fn() + dataListeners.add(onData) + replayListeners.add(onReplay) + exitListeners.add(onExit) + livePtyIds.add('ssh:conn@@unrelated') + + for (const method of [ + 'pty.recoveryData', + 'pty.recoveryComplete', + 'pty.restoreRequired', + 'pty.deliveryCanceled' + ]) { + handler(method, { + id: 'pty-1', + data: 'control', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3 + }) + } + + expect(toAppPtyId).not.toHaveBeenCalled() + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(recordExit).not.toHaveBeenCalled() + expect(onData).not.toHaveBeenCalled() + expect(onReplay).not.toHaveBeenCalled() + expect(onExit).not.toHaveBeenCalled() + expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated'])) + expect(mux.request).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/providers/ssh-pty-notification-routing-test-fixture.ts b/src/main/providers/ssh-pty-notification-routing-test-fixture.ts new file mode 100644 index 00000000000..6d86d673037 --- /dev/null +++ b/src/main/providers/ssh-pty-notification-routing-test-fixture.ts @@ -0,0 +1,94 @@ +import { vi, type Mock } from 'vitest' +import { + subscribeSshPtyNotifications, + type SshPtyNotificationSubscription +} from './ssh-pty-notification-routing' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' + +type NotificationHandler = (method: string, params: Record) => void +type CancellationRequest = ( + method: string, + params: Record +) => Promise<{ canceled: boolean; sentEndSu: number; creditedEndSu: number }> + +type MockMux = { + onNotification: Mock<(handler: NotificationHandler) => void> + request: Mock +} + +export type SshPtyNotificationTestSubscription = { + handler: NotificationHandler + mux: MockMux + toAppPtyId: Mock<(id: string) => string> + dataListeners: Set<(payload: { id: string; data: string }) => void> + replayListeners: Set<(payload: { id: string; data: string }) => void> + exitListeners: Set<(payload: { id: string; code: number }) => void> + livePtyIds: Set + recordExit: Mock<(relayPtyId: string, incarnationId: unknown) => void> + resolvePtyIncarnation: Mock<(id: string) => string> + installReceivingActivation: SshPtyNotificationSubscription['installReceivingActivation'] +} + +export function createSubscription(): SshPtyNotificationTestSubscription { + const mux: MockMux = { + onNotification: vi.fn<(handler: NotificationHandler) => void>(), + request: vi.fn(async () => ({ + canceled: true, + sentEndSu: 0, + creditedEndSu: 0 + })) + } + const dataListeners = new Set<(payload: { id: string; data: string }) => void>() + const replayListeners = new Set<(payload: { id: string; data: string }) => void>() + const exitListeners = new Set<(payload: { id: string; code: number }) => void>() + const livePtyIds = new Set() + const recordExit = vi.fn<(relayPtyId: string, incarnationId: unknown) => void>() + const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`) + const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`) + + const subscription = subscribeSshPtyNotifications({ + mux: mux as never, + toAppPtyId, + dataListeners: dataListeners as never, + replayListeners: replayListeners as never, + exitListeners: exitListeners as never, + livePtyIds, + recordExit, + providerGeneration: 7, + resolvePtyIncarnation, + peekPtyIncarnation: () => undefined + }) + + const handler = mux.onNotification.mock.calls[0]?.[0] + if (!handler) { + throw new Error('notification handler was not registered') + } + + return { + handler, + mux, + toAppPtyId, + dataListeners, + replayListeners, + exitListeners, + livePtyIds, + recordExit, + resolvePtyIncarnation, + installReceivingActivation: subscription.installReceivingActivation + } +} + +export function sourceActivation( + overrides: Partial = {} +): PtySourceReceivingActivation { + return Object.freeze({ + status: 'pending', + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + checkpointSourceEndSu: 0, + recoveryEndSu: 0, + ...overrides + }) +} diff --git a/src/main/providers/ssh-pty-notification-routing.test.ts b/src/main/providers/ssh-pty-notification-routing.test.ts index fdb6776bf36..332e8c3cdd6 100644 --- a/src/main/providers/ssh-pty-notification-routing.test.ts +++ b/src/main/providers/ssh-pty-notification-routing.test.ts @@ -1,74 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing' -import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' - -type MockMux = { - onNotification: ReturnType - request: ReturnType -} - -function createSubscription() { - const mux: MockMux = { - onNotification: vi.fn(), - request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 })) - } - const dataListeners = new Set<(payload: { id: string; data: string }) => void>() - const replayListeners = new Set<(payload: { id: string; data: string }) => void>() - const exitListeners = new Set<(payload: { id: string; code: number }) => void>() - const livePtyIds = new Set() - const recordExit = vi.fn() - const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`) - const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`) - - const subscription = subscribeSshPtyNotifications({ - mux: mux as never, - toAppPtyId, - dataListeners: dataListeners as never, - replayListeners: replayListeners as never, - exitListeners: exitListeners as never, - livePtyIds, - recordExit, - providerGeneration: 7, - resolvePtyIncarnation, - peekPtyIncarnation: () => undefined - }) - - const handler = mux.onNotification.mock.calls[0]?.[0] as ( - method: string, - params: Record - ) => void - if (!handler) { - throw new Error('notification handler was not registered') - } - - return { - handler, - mux, - toAppPtyId, - dataListeners, - replayListeners, - exitListeners, - livePtyIds, - recordExit, - resolvePtyIncarnation, - installReceivingActivation: subscription.installReceivingActivation - } -} - -function sourceActivation( - overrides: Partial = {} -): PtySourceReceivingActivation { - return Object.freeze({ - status: 'pending', - clientGeneration: 2, - ownerGeneration: 3, - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - checkpointSourceEndSu: 0, - recoveryEndSu: 0, - ...overrides - }) -} +import { createSubscription, sourceActivation } from './ssh-pty-notification-routing-test-fixture' describe('subscribeSshPtyNotifications', () => { it('ignores non-PTY notifications without mapping params.id', () => { @@ -480,328 +411,6 @@ describe('subscribeSshPtyNotifications', () => { expect(mux.request).not.toHaveBeenCalled() }) - it('accepts non-empty recovery from the activation checkpoint', () => { - const { handler, dataListeners, installReceivingActivation } = createSubscription() - const onData = vi.fn() - dataListeners.add(onData) - const lease = installReceivingActivation( - 'pty-1', - sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) - ) - - handler('pty.data', { - id: 'pty-1', - data: 'next', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 8, - sourceLengthSu: 4 - }) - - expect(onData).not.toHaveBeenCalled() - lease.commit() - expect(onData).toHaveBeenCalledWith( - expect.objectContaining({ - data: 'next', - source: expect.objectContaining({ sourceStartSu: 4, sourceEndSu: 8 }) - }) - ) - }) - - it('routes held and later recovery frames only to the private sink until commit', () => { - const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription() - const onData = vi.fn() - const onRecoveryData = vi.fn() - dataListeners.add(onData) - const lease = installReceivingActivation( - 'pty-1', - sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 }) - ) - const publishSource = (data: string, sourceEndSu: number): void => { - handler('pty.data', { - id: 'pty-1', - data, - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu, - sourceLengthSu: 4 - }) - } - - publishSource('held', 8) - const recoveryLease = lease.transferToRecovery(onRecoveryData) - publishSource('next', 12) - - expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next']) - expect(onData).not.toHaveBeenCalled() - expect(livePtyIds).not.toContain('ssh:conn@@pty-1') - - recoveryLease.commit() - expect(onData).not.toHaveBeenCalled() - publishSource('live', 16) - - expect(onRecoveryData).toHaveBeenCalledTimes(2) - expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' })) - expect(livePtyIds).toContain('ssh:conn@@pty-1') - }) - - it('retires an exited private recovery when its activation commits', () => { - const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = - createSubscription() - const onData = vi.fn() - const onRecoveryData = vi.fn() - dataListeners.add(onData) - const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 })) - handler('pty.data', { - id: 'pty-1', - data: 'held', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 4, - sourceLengthSu: 4 - }) - const recoveryLease = lease.transferToRecovery(onRecoveryData) - - handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' }) - recoveryLease.commit() - handler('pty.data', { - id: 'pty-1', - data: 'late', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 8, - sourceLengthSu: 4 - }) - - expect(onRecoveryData).toHaveBeenCalledOnce() - expect(onData).not.toHaveBeenCalled() - expect(livePtyIds).not.toContain('ssh:conn@@pty-1') - expect(mux.request).toHaveBeenCalledWith( - 'pty.cancelDelivery', - expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' }) - ) - }) - - it('retires private recovery locally and restores the exact predecessor', () => { - const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() - const onData = vi.fn() - const onRecoveryData = vi.fn() - dataListeners.add(onData) - installReceivingActivation( - 'pty-1', - sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 }) - ).commit() - handler('pty.data', { - id: 'pty-1', - data: 'pre', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-old', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 3, - sourceLengthSu: 3 - }) - const replacement = installReceivingActivation( - 'pty-1', - sourceActivation({ - clientGeneration: 3, - ownerGeneration: 4, - deliveryToken: 'token-new', - checkpointSourceEndSu: 3, - recoveryEndSu: 6 - }) - ) - handler('pty.data', { - id: 'pty-1', - data: 'new', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-new', - clientGeneration: 3, - ownerGeneration: 4, - sourceEndSu: 6, - sourceLengthSu: 3 - }) - - replacement.transferToRecovery(onRecoveryData).retire() - handler('pty.data', { - id: 'pty-1', - data: 'old', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-old', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 6, - sourceLengthSu: 3 - }) - - expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) - expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) - expect(mux.request).not.toHaveBeenCalled() - }) - - it('rejects a stale activation without disturbing current continuity', () => { - const { handler, dataListeners, installReceivingActivation } = createSubscription() - const onData = vi.fn() - dataListeners.add(onData) - installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit() - - expect(() => - installReceivingActivation( - 'pty-1', - sourceActivation({ - clientGeneration: 1, - ownerGeneration: 4, - deliveryToken: 'token-stale', - checkpointSourceEndSu: 3, - recoveryEndSu: 3 - }) - ) - ).toThrow('ssh_source_receiving_activation_stale') - - handler('pty.data', { - id: 'pty-1', - data: 'one', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 3, - sourceLengthSu: 3 - }) - expect(onData).toHaveBeenCalledOnce() - }) - - it('drops provisional frames and settles cancellation before rollback completes', async () => { - const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = - createSubscription() - const onData = vi.fn() - dataListeners.add(onData) - const lease = installReceivingActivation( - 'pty-1', - sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) - ) - handler('pty.data', { - id: 'pty-1', - data: 'next', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-1', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 8, - sourceLengthSu: 4 - }) - - await expect(lease.rollback()).resolves.toBe(true) - - expect(onData).not.toHaveBeenCalled() - expect(livePtyIds).not.toContain('ssh:conn@@pty-1') - expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { - id: 'pty-1', - clientGeneration: 2, - ownerGeneration: 3, - deliveryToken: 'token-1' - }) - }) - - it('restores the exact prior cursor when a replacement rolls back after frames', async () => { - const { handler, dataListeners, installReceivingActivation } = createSubscription() - const onData = vi.fn() - dataListeners.add(onData) - installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit() - handler('pty.data', { - id: 'pty-1', - data: 'pre', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-old', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 3, - sourceLengthSu: 3 - }) - const replacement = installReceivingActivation( - 'pty-1', - sourceActivation({ - clientGeneration: 3, - ownerGeneration: 4, - deliveryToken: 'token-new', - checkpointSourceEndSu: 3, - recoveryEndSu: 3 - }) - ) - handler('pty.data', { - id: 'pty-1', - data: 'new', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-new', - clientGeneration: 3, - ownerGeneration: 4, - sourceEndSu: 6, - sourceLengthSu: 3 - }) - - await replacement.rollback() - handler('pty.data', { - id: 'pty-1', - data: 'old', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-old', - clientGeneration: 2, - ownerGeneration: 3, - sourceEndSu: 6, - sourceLengthSu: 3 - }) - - expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) - }) - - it('does not let an older lease rollback replace a newer activation', async () => { - const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() - const onData = vi.fn() - dataListeners.add(onData) - const older = installReceivingActivation('pty-1', sourceActivation()) - const newer = installReceivingActivation( - 'pty-1', - sourceActivation({ - clientGeneration: 3, - ownerGeneration: 4, - deliveryToken: 'token-new' - }) - ) - - await older.rollback() - handler('pty.data', { - id: 'pty-1', - data: 'new', - ptyIncarnation: 'incarnation-1', - deliveryToken: 'token-new', - clientGeneration: 3, - ownerGeneration: 4, - sourceEndSu: 3, - sourceLengthSu: 3 - }) - newer.commit() - - expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) - expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { - id: 'pty-1', - clientGeneration: 2, - ownerGeneration: 3, - deliveryToken: 'token-1' - }) - expect(mux.request).not.toHaveBeenCalledWith( - 'pty.cancelDelivery', - expect.objectContaining({ deliveryToken: 'token-new' }) - ) - }) - it('ignores PTY methods with missing ids', () => { const { handler, toAppPtyId, dataListeners } = createSubscription() const onData = vi.fn() diff --git a/src/main/providers/ssh-pty-process-list.ts b/src/main/providers/ssh-pty-process-list.ts new file mode 100644 index 00000000000..5c9c2fa8248 --- /dev/null +++ b/src/main/providers/ssh-pty-process-list.ts @@ -0,0 +1,52 @@ +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import type { IPtyProvider, PtyProcessInfo } from './types' +import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' +import { mapSshPtyProcessList } from './ssh-agent-session-process-list' +import type { SshPtyProviderOutputState } from './ssh-pty-provider-output-state' + +export function createSshPtyProcessLister( + args: Pick< + Parameters[0], + 'mux' | 'connectionId' | 'livePtyIds' | 'outputState' + > +): IPtyProvider['listProcesses'] { + return (options) => + listSshPtyProcesses({ + ...args, + includeForegroundProcessEvidence: options?.includeForegroundProcessEvidence, + deadlineMs: options?.deadlineMs + }) +} + +export async function listSshPtyProcesses( + args: Readonly<{ + mux: SshChannelMultiplexer + connectionId: string + livePtyIds: Set + outputState: SshPtyProviderOutputState + includeForegroundProcessEvidence?: boolean + deadlineMs?: number + }> +): Promise { + const result = await args.mux.request( + 'pty.listProcesses', + args.includeForegroundProcessEvidence === undefined + ? undefined + : { includeForegroundProcessEvidence: args.includeForegroundProcessEvidence }, + args.deadlineMs === undefined + ? undefined + : { timeoutMs: Math.max(1, args.deadlineMs - Date.now()) } + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the relay answers pty.listProcesses with PtyProcessInfo rows; the mapper rejects unproven ownership. + const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => + toAppSshPtyId(args.connectionId, id) + ) + for (const process of processes) { + args.livePtyIds.add(process.id) + args.outputState.rememberPtyIncarnation( + toRelaySshPtyId(args.connectionId, process.id), + process.incarnationId + ) + } + return processes +} diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 59088247e18..b304adf8176 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -1,5 +1,5 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' -import type { IPtyProvider, PtyProcessInfo, PtySpawnOptions, PtySpawnResult } from './types' +import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from './types' import type { WriteSettlement } from '../../shared/pty-write-settlement' import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc-color-reply' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' @@ -13,7 +13,6 @@ import type { } from './ssh-pty-provider-contract' import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state' import { spawnFreshSshPty } from './ssh-agent-session-create-operation' -import { mapSshPtyProcessList } from './ssh-agent-session-process-list' import { requestSshPtyAttach, reattachSshPtySessionForSpawn, @@ -26,6 +25,11 @@ import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' import type { PtyProcessInspection } from './pty-process-inspection' import { spawnWithTerminalRuntimeRepair, type TerminalRepairHook } from './ssh-pty-spawn-repair' import { createSshPtyProviderRpcOperations } from './ssh-pty-provider-rpc-operations' +import { createSshPtyProcessLister } from './ssh-pty-process-list' +import { + installSshPtyLegacyRelayDelegation, + type SshPtyLegacyRelayRouting +} from './ssh-pty-legacy-relay-delegation' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined { @@ -37,7 +41,8 @@ export class SshPtyProvider implements IPtyProvider { private mux: SshChannelMultiplexer private connectionId: string private livePtyIds = new Set() - readonly getAppliedSize: NonNullable + getAppliedSize: NonNullable + listProcesses: IPtyProvider['listProcesses'] private readonly agentSessionCapabilities: SshAgentSessionCapabilities private spawnExitRaces = new SshPtySpawnExitRaceTracker() private readonly outputState: SshPtyProviderOutputState @@ -101,6 +106,12 @@ export class SshPtyProvider implements IPtyProvider { this.spawnExitRaces.recordExit(relayPtyId, incarnationId) } }) + this.listProcesses = createSshPtyProcessLister({ + mux, + connectionId, + livePtyIds: this.livePtyIds, + outputState: this.outputState + }) } dispose(): void { @@ -116,6 +127,11 @@ export class SshPtyProvider implements IPtyProvider { private toAppPtyId = (id: string): string => toAppSshPtyId(this.connectionId, id) + /** Installed by SshRelaySession once per provider, for PTYs an earlier build's relay still runs. */ + setLegacyRelayRouting(routing: SshPtyLegacyRelayRouting): void { + installSshPtyLegacyRelayDelegation(this, routing) + } + /** Installed by SshRelaySession, which owns the connection, the repair lock and the reconnect. */ setTerminalUnavailableRecovery(recover: TerminalRepairHook): void { this.recoverFromTerminalUnavailable = recover @@ -270,26 +286,6 @@ export class SshPtyProvider implements IPtyProvider { this.livePtyIds.delete(id) } - async listProcesses(opts?: { - deadlineMs?: number - includeForegroundProcessEvidence?: boolean - }): Promise { - const result = await this.mux.request( - 'pty.listProcesses', - opts?.includeForegroundProcessEvidence === undefined - ? undefined - : { includeForegroundProcessEvidence: opts.includeForegroundProcessEvidence }, - relayTimeoutOptions(opts?.deadlineMs) - ) - const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => this.toAppPtyId(id)) - for (const process of processes) { - this.livePtyIds.add(process.id) - const relayPtyId = this.toRelayPtyId(process.id) - this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId) - } - return processes - } - hasPty = (id: string): boolean => this.livePtyIds.has(id) onData = (callback: SshPtyDataCallback): (() => void) => this.outputState.onData(callback) diff --git a/src/main/providers/ssh-pty-reattach-previous-relay-hold.test.ts b/src/main/providers/ssh-pty-reattach-previous-relay-hold.test.ts new file mode 100644 index 00000000000..6c2c2c93b08 --- /dev/null +++ b/src/main/providers/ssh-pty-reattach-previous-relay-hold.test.ts @@ -0,0 +1,89 @@ +// After an app update the previous build's relay can still run a pane's PTY, and the new relay +// answers "not found" for an id it never minted. That answer must not reach the pane as +// `SSH_SESSION_EXPIRED`: the renderer cold-restores on that token, and spawn-execute expires the +// lease, so the user's running terminal would be silently replaced by an empty shell. +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import { PTY_ATTACH_PROVEN_EXITED_MARKER } from '../../shared/pty-attach-absence-evidence' +import { + isSshPtyAbsentFromRelayError, + SSH_PTY_HELD_BY_PREVIOUS_RELAY_ERROR, + SSH_SESSION_EXPIRED_ERROR, + SshPtyHeldByPreviousRelayError +} from './ssh-pty-errors' + +const { previousRelayMayHoldTerminals } = vi.hoisted(() => ({ + previousRelayMayHoldTerminals: vi.fn() +})) +vi.mock('../ssh/ssh-previous-relay-terminals', () => ({ previousRelayMayHoldTerminals })) + +import { reattachSshPtySessionForSpawn } from './ssh-pty-session-reattach' +import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' + +const CONNECTION = 'conn-1' +const SESSION = 'pty2:old-epoch:1' + +function refusingMux(message: string): SshChannelMultiplexer { + // Only `request` is reached on this path; the untyped base keeps the stub free of a cast. + return Object.assign(Object.create(null), { + request: vi.fn(async () => { + throw new Error(message) + }) + }) +} + +async function refusalFrom(message: string): Promise { + try { + await reattachSshPtySessionForSpawn({ + mux: refusingMux(message), + connectionId: CONNECTION, + sessionId: SESSION, + options: { cols: 80, rows: 24 }, + exitRaceTracker: new SshPtySpawnExitRaceTracker(), + acceptLivePty: () => {} + }) + } catch (error) { + if (error instanceof Error) { + return error + } + throw error + } + throw new Error('expected the reattach to be refused') +} + +describe('a not-found reattach while an older Orca relay is live on the host', () => { + beforeEach(() => { + previousRelayMayHoldTerminals.mockReset() + }) + + it('keeps the pane bound instead of reporting the session expired', async () => { + previousRelayMayHoldTerminals.mockResolvedValue(true) + + const error = await refusalFrom(`PTY "${SESSION}" not found`) + + expect(error).toBeInstanceOf(SshPtyHeldByPreviousRelayError) + expect(error.message).toContain(SSH_PTY_HELD_BY_PREVIOUS_RELAY_ERROR) + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(isSshPtyAbsentFromRelayError(error)).toBe(false) + expect(previousRelayMayHoldTerminals).toHaveBeenCalledWith(CONNECTION) + }) + + it('reports absence as before when no older relay may hold it', async () => { + previousRelayMayHoldTerminals.mockResolvedValue(false) + + const error = await refusalFrom(`PTY "${SESSION}" not found`) + + expect(isSshPtyAbsentFromRelayError(error)).toBe(true) + }) + + it('does not hold an id the current relay proved exited', async () => { + previousRelayMayHoldTerminals.mockResolvedValue(true) + + const error = await refusalFrom( + `PTY "${SESSION}" not found (${PTY_ATTACH_PROVEN_EXITED_MARKER})` + ) + + expect(isSshPtyAbsentFromRelayError(error)).toBe(true) + expect(previousRelayMayHoldTerminals).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index f05373a03dc..3a978e1978f 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -5,11 +5,13 @@ import { SSH_PTY_SOURCE_RESTORE_REQUIRED_ERROR, SSH_SESSION_EXPIRED_ERROR, SshPtyAbsentFromRelayError, + SshPtyHeldByPreviousRelayError, SshPtyProvenExitedOnRelayError, isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from './ssh-pty-errors' import { isProvenExitedPtyAttachRefusal } from '../../shared/pty-attach-absence-evidence' +import { previousRelayMayHoldTerminals } from '../ssh/ssh-previous-relay-terminals' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import type { PtySpawnOptions, PtySpawnResult } from './types' import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' @@ -247,6 +249,9 @@ export async function reattachSshPtySession(args: { if (isProvenExitedPtyAttachRefusal(error)) { throw new SshPtyProvenExitedOnRelayError(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`) } + if (await previousRelayMayHoldTerminals(args.connectionId)) { + throw new SshPtyHeldByPreviousRelayError(relaySessionId) + } throw new SshPtyAbsentFromRelayError(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`) } throw error diff --git a/src/main/providers/ssh-pty-spawn-env.test.ts b/src/main/providers/ssh-pty-spawn-env.test.ts index b1fba08b0ca..e2397d0ba5a 100644 --- a/src/main/providers/ssh-pty-spawn-env.test.ts +++ b/src/main/providers/ssh-pty-spawn-env.test.ts @@ -2,6 +2,29 @@ import { describe, expect, it } from 'vitest' import { buildSshPtySpawnEnv } from './ssh-pty-spawn-env' import type { RemoteCliBridgeEnv } from './ssh-pty-provider-contract' +describe('buildSshPtySpawnEnv relay bridge', () => { + it('prepends the CLI bin dir once and publishes the Node relay bridge', () => { + const env = buildSshPtySpawnEnv({ + env: { PATH: '/home/me/.orca-relay/bin:/usr/bin' }, + remoteCliBridgeEnv: { + binDir: '/home/me/.orca-relay/bin', + relayDir: '/home/me/.orca-relay/relay-v1', + nodePath: '/usr/bin/node', + sockPath: '/home/me/.orca-relay/relay.sock' + } + }) + + expect(env).toMatchObject({ + PATH: '/home/me/.orca-relay/bin:/usr/bin', + ORCA_REMOTE_CLI_BIN_DIR: '/home/me/.orca-relay/bin', + ORCA_RELAY_DIR: '/home/me/.orca-relay/relay-v1', + ORCA_RELAY_NODE_PATH: '/usr/bin/node', + ORCA_RELAY_SOCKET_PATH: '/home/me/.orca-relay/relay.sock' + }) + expect(env).not.toHaveProperty('ORCA_RELAY_CREDENTIAL_FILE') + }) +}) + const bridge: RemoteCliBridgeEnv = { binDir: '/remote/orca/bin', relayDir: '/remote/orca', diff --git a/src/main/runtime/__fixtures__/claude-ready-task-wakeup.meta.json b/src/main/runtime/__fixtures__/claude-ready-task-wakeup.meta.json index 5d0c9b7bfc8..2a2bfcc398c 100644 --- a/src/main/runtime/__fixtures__/claude-ready-task-wakeup.meta.json +++ b/src/main/runtime/__fixtures__/claude-ready-task-wakeup.meta.json @@ -1,9 +1,7 @@ { "capturedAt": "2026-10-06T04:06:33.762Z", "platform": "darwin", - "command": [ - "claude" - ], + "command": ["claude"], "cols": 120, "rows": 40, "note": "Claude Code 2.1.291; native ready-title capture in existing workspace; no prompt submitted", diff --git a/src/main/runtime/headless-runtime-graph.test.ts b/src/main/runtime/headless-runtime-graph.test.ts new file mode 100644 index 00000000000..a33778eb35f --- /dev/null +++ b/src/main/runtime/headless-runtime-graph.test.ts @@ -0,0 +1,27 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { publishHeadlessRuntimeGraph } from './headless-runtime-graph' + +describe('headless runtime graph', () => { + it('lets a session tab inventory settle on a host no renderer publishes for', async () => { + const runtime = new OrcaRuntimeService() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the inventory census reads only listProcesses. + runtime.setPtyController({ listProcesses: vi.fn(async () => []) } as never) + publishHeadlessRuntimeGraph(runtime) + + await expect(runtime.listAllMobileSessionTabsInventory()).resolves.toEqual({ + snapshots: [], + authoritative: true + }) + }) + + it('is published by orcad before its RPC server accepts a client', () => { + // Why a source check: orcad's startup needs a real profile store and pty host to run. + const entry = readFileSync(join(import.meta.dirname, '../orcad/orcad-entry.ts'), 'utf8') + const published = entry.indexOf('publishHeadlessRuntimeGraph(runtime)') + expect(published).toBeGreaterThan(-1) + expect(published).toBeLessThan(entry.indexOf('await rpc.start()')) + }) +}) diff --git a/src/main/runtime/headless-runtime-graph.ts b/src/main/runtime/headless-runtime-graph.ts new file mode 100644 index 00000000000..7715c6907c2 --- /dev/null +++ b/src/main/runtime/headless-runtime-graph.ts @@ -0,0 +1,13 @@ +import { HEADLESS_RUNTIME_WINDOW_ID } from '../../shared/runtime-types' +import type { OrcaRuntimeService } from './orca-runtime' + +/** + * A headless host has no renderer to publish a graph. Without this empty one the graph never + * reads as ready, so status clients see an unready server and `session.tabs.listAll` waits on + * an inventory publication that never comes. + */ +export function publishHeadlessRuntimeGraph( + runtime: Pick +): void { + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) +} diff --git a/src/main/runtime/headless-terminal-split-layout.test.ts b/src/main/runtime/headless-terminal-split-layout.test.ts index bb745d49c61..7d9c3483190 100644 --- a/src/main/runtime/headless-terminal-split-layout.test.ts +++ b/src/main/runtime/headless-terminal-split-layout.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from 'vitest' import type { TerminalLayoutSnapshot } from '../../shared/terminal-tab-types' +import { terminalLayoutContainsLeaf } from '../../shared/workspace-session-pane-ownership' import { buildHeadlessTerminalSplitLayout, - countTerminalLayoutLeaves, - terminalLayoutContainsLeaf + countTerminalLayoutLeaves } from './headless-terminal-split-layout' describe('terminalLayoutContainsLeaf', () => { diff --git a/src/main/runtime/headless-terminal-split-layout.ts b/src/main/runtime/headless-terminal-split-layout.ts index 681d70451e8..8e326750f19 100644 --- a/src/main/runtime/headless-terminal-split-layout.ts +++ b/src/main/runtime/headless-terminal-split-layout.ts @@ -2,20 +2,6 @@ import type { TerminalLayoutSnapshot, TerminalPaneLayoutNode } from '../../shared/terminal-tab-types' - -export function terminalLayoutContainsLeaf( - node: TerminalPaneLayoutNode | null | undefined, - leafId: string -): boolean { - if (!node) { - return false - } - return node.type === 'leaf' - ? node.leafId === leafId - : terminalLayoutContainsLeaf(node.first, leafId) || - terminalLayoutContainsLeaf(node.second, leafId) -} - /** * Insert a newly split-off leaf into a terminal tab's persisted layout tree. * diff --git a/src/main/runtime/managed-server-actions-registry.ts b/src/main/runtime/managed-server-actions-registry.ts new file mode 100644 index 00000000000..3bb3368f3ee --- /dev/null +++ b/src/main/runtime/managed-server-actions-registry.ts @@ -0,0 +1,33 @@ +/** + * The managed-server actions the Managed servers settings run, published by the desktop main + * process so runtime RPC can offer the same ones. A runtime with none registered (headless + * `orca serve`) has no SSH registry to manage servers from, and does not advertise them. + */ +import type { + OrcadManagedCancelStopResult, + OrcadManagedDeployResult, + OrcadManagedRecoveryResult, + OrcadManagedRollbackResult, + OrcadManagedRuntimeStatus, + OrcadManagedStopResult +} from '../../shared/orcad-managed-runtime' + +export type ManagedServerActions = { + status: (selector: string) => Promise + update: (selector: string, force: boolean) => Promise + rollback: (selector: string) => Promise + /** `acceptChangedState` restores the snapshot over state a rejected build changed. */ + recover: (selector: string, acceptChangedState?: boolean) => Promise + stop: (selector: string) => Promise + cancelStop: (selector: string) => Promise +} + +let registered: ManagedServerActions | null = null + +export function registerManagedServerActions(actions: ManagedServerActions | null): void { + registered = actions +} + +export function getManagedServerActions(): ManagedServerActions | null { + return registered +} diff --git a/src/main/runtime/mobile-session-editor-projection.ts b/src/main/runtime/mobile-session-editor-projection.ts new file mode 100644 index 00000000000..a88e633dd41 --- /dev/null +++ b/src/main/runtime/mobile-session-editor-projection.ts @@ -0,0 +1,168 @@ +/** + * Editor tabs a headless host persisted, such as those a converted SSH host's migration carried + * in. No renderer publishes them there, so the host projects and retires them itself; a paired + * desktop mirrors what it lists. + */ +import type { + RuntimeMobileSessionFileTab, + RuntimeMobileSessionMarkdownTab, + RuntimeMobileSessionTabsSnapshot +} from '../../shared/runtime-types' +import type { Tab } from '../../shared/tab-types' +import type { + PersistedOpenFile, + WorkspaceSessionState +} from '../../shared/workspace-session-state-types' + +export type HeadlessEditorTab = RuntimeMobileSessionMarkdownTab | RuntimeMobileSessionFileTab + +function editorUnifiedTab( + session: WorkspaceSessionState, + worktreeId: string, + filePath: string +): Tab | undefined { + return (session.unifiedTabs?.[worktreeId] ?? []).find( + (tab) => tab.contentType === 'editor' && tab.entityId === filePath + ) +} + +function editorTab( + session: WorkspaceSessionState, + worktreeId: string, + file: PersistedOpenFile +): HeadlessEditorTab { + const unifiedTab = editorUnifiedTab(session, worktreeId, file.filePath) + const common = { + // A file's id is its path, which is also the unified tab's entity. + id: unifiedTab?.id ?? file.filePath, + title: file.relativePath.split(/[\\/]/).pop() || file.relativePath || 'File', + filePath: file.filePath, + relativePath: file.relativePath, + isDirty: file.dirtyDraftContent !== undefined, + isActive: + session.activeTabTypeByWorktree?.[worktreeId] === 'editor' && + session.activeFileIdByWorktree?.[worktreeId] === file.filePath, + color: unifiedTab?.color ?? null, + isPinned: unifiedTab?.isPinned === true + } + if (file.language === 'markdown') { + return { + ...common, + type: 'markdown', + language: 'markdown', + mode: 'edit', + sourceFileId: file.filePath, + sourceFilePath: file.filePath, + sourceRelativePath: file.relativePath, + documentVersion: `file:${file.filePath}` + } + } + return { ...common, type: 'file', language: file.language, mode: 'edit' } +} + +export const HEADLESS_EDITOR_UNSAVED_DRAFT_ERROR = 'editor_tab_has_unsaved_draft' + +export function buildHeadlessMobileSessionEditorTabs( + worktreeId: string, + session: WorkspaceSessionState +): HeadlessEditorTab[] { + return (session.openFilesByWorktree?.[worktreeId] ?? []).map((file) => + editorTab(session, worktreeId, file) + ) +} + +/** The session without one persisted editor tab, or null when it holds no such tab. */ +export function retireHeadlessEditorTab( + session: WorkspaceSessionState, + worktreeId: string, + tab: Pick, + force = false +): WorkspaceSessionState | null { + const files = session.openFilesByWorktree?.[worktreeId] ?? [] + const file = files.find((candidate) => candidate.filePath === tab.filePath) + if (!file) { + return null + } + // No window here can prompt to save, and the draft lives nowhere else. + if (file.dirtyDraftContent !== undefined && !force) { + throw new Error(HEADLESS_EDITOR_UNSAVED_DRAFT_ERROR) + } + const unifiedTabId = editorUnifiedTab(session, worktreeId, tab.filePath)?.id ?? tab.id + const withoutTab = (ids: readonly string[] | undefined): string[] | undefined => + ids?.filter((id) => id !== unifiedTabId) + const groups = (session.tabGroups?.[worktreeId] ?? []).map((group) => ({ + ...group, + activeTabId: group.activeTabId === unifiedTabId ? null : group.activeTabId, + tabOrder: withoutTab(group.tabOrder) ?? [], + ...(group.recentTabIds ? { recentTabIds: withoutTab(group.recentTabIds) } : {}) + })) + const wasActiveFile = session.activeFileIdByWorktree?.[worktreeId] === tab.filePath + return { + ...session, + openFilesByWorktree: { + ...session.openFilesByWorktree, + [worktreeId]: files.filter((file) => file.filePath !== tab.filePath) + }, + unifiedTabs: { + ...session.unifiedTabs, + [worktreeId]: (session.unifiedTabs?.[worktreeId] ?? []).filter( + (candidate) => candidate.id !== unifiedTabId + ) + }, + ...(session.tabGroups ? { tabGroups: { ...session.tabGroups, [worktreeId]: groups } } : {}), + ...(wasActiveFile + ? { activeFileIdByWorktree: { ...session.activeFileIdByWorktree, [worktreeId]: null } } + : {}), + ...(session.activeTabIdByWorktree?.[worktreeId] === unifiedTabId + ? { activeTabIdByWorktree: { ...session.activeTabIdByWorktree, [worktreeId]: null } } + : {}) + } +} + +export type HeadlessEditorRetirementHost = { + getWorkspaceSessionForWorktree(worktreeId: string): WorkspaceSessionState | null | undefined + setWorkspaceSessionForWorktree(worktreeId: string, session: WorkspaceSessionState): void + hydrateHeadlessMobileSessionTabsFromWorkspaceSession( + worktreeId: string, + options: { force: true } + ): unknown + notifyMobileSessionTabsChanged(worktreeId: string): void + mobileSessionTabsByWorktree: Map + storeMobileSessionSnapshot(worktreeId: string, snapshot: RuntimeMobileSessionTabsSnapshot): void +} + +/** Retires a listed editor from the host's own session and republishes; false when not found. */ +export function retireHeadlessMobileSessionEditorTab( + host: HeadlessEditorRetirementHost, + worktreeId: string, + tab: { id: string; type: string; filePath?: string }, + force = false +): boolean { + const session = host.getWorkspaceSessionForWorktree(worktreeId) + const next = + session && (tab.type === 'markdown' || tab.type === 'file') && tab.filePath + ? retireHeadlessEditorTab(session, worktreeId, { id: tab.id, filePath: tab.filePath }, force) + : null + if (!next) { + return false + } + host.setWorkspaceSessionForWorktree(worktreeId, next) + // Why drop it first: a rebuild that finds no tabs left keeps the old snapshot. + const existing = host.mobileSessionTabsByWorktree.get(worktreeId) + if (existing) { + host.storeMobileSessionSnapshot(worktreeId, { + ...existing, + snapshotVersion: existing.snapshotVersion + 1, + activeTabId: existing.activeTabId === tab.id ? null : existing.activeTabId, + tabGroups: existing.tabGroups?.map((group) => ({ + ...group, + activeTabId: group.activeTabId === tab.id ? null : group.activeTabId, + tabOrder: group.tabOrder.filter((id) => id !== tab.id) + })), + tabs: existing.tabs.filter((candidate) => candidate.id !== tab.id) + }) + } + host.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { force: true }) + host.notifyMobileSessionTabsChanged(worktreeId) + return true +} diff --git a/src/main/runtime/orca-runtime-automation-operations.ts b/src/main/runtime/orca-runtime-automation-operations.ts index 5064944215b..32d47c59951 100644 --- a/src/main/runtime/orca-runtime-automation-operations.ts +++ b/src/main/runtime/orca-runtime-automation-operations.ts @@ -198,6 +198,10 @@ export class OrcaRuntimeWithAutomationOperations extends OrcaRuntimeWithPtyForeg return this.legacyWorkerRecovery.reconcile(options) } + stopLegacyWorkerTerminalRecovery(): Promise { + return this.legacyWorkerRecovery.stop() + } + protected updateLegacyWorkerTerminalRecoveryRetry( plan: LegacyWorkerTerminalRecoveryPlan, deferredDispatchIds: ReadonlySet, diff --git a/src/main/runtime/orca-runtime-close-mobile-session-tab.ts b/src/main/runtime/orca-runtime-close-mobile-session-tab.ts index 6f4248a4673..2bfeaec2add 100644 --- a/src/main/runtime/orca-runtime-close-mobile-session-tab.ts +++ b/src/main/runtime/orca-runtime-close-mobile-session-tab.ts @@ -21,6 +21,7 @@ import type { RuntimeCommandSurfaceHost } from './orca-runtime-core' import { SESSION_TAB_NOT_FOUND_ERROR } from '../../shared/session-tab-close' import { rendererPublicationThrottle } from '../window/renderer-publication-throttle' import { structuredAgentSessionTabCloseCause } from './structured-agent-session-tab-close-cause' +import { retireHeadlessMobileSessionEditorTab } from './mobile-session-editor-projection' export class OrcaRuntimeWithCloseMobileSessionTab extends OrcaRuntimeWithRefuseUnattributedMobileSessionTabClose { async closeMobileSessionTab( @@ -305,10 +306,12 @@ export class OrcaRuntimeWithCloseMobileSessionTab extends OrcaRuntimeWithRefuseU tab, structuredAgentSessionTabCloseCause(options.reason) ) - } else { - if (!this.notifier?.closeSessionTab) { + } else if (!this.notifier?.closeSessionTab) { + // Why: a headless host listed this editor from its own session, so it retires it there. + if (!retireHeadlessMobileSessionEditorTab(this, worktreeId, tab, options.force)) { throw new Error('runtime_unavailable') } + } else { await this.notifier.closeSessionTab(tab.id, worktreeId) } return finishCommittedClose() diff --git a/src/main/runtime/orca-runtime-get-status.ts b/src/main/runtime/orca-runtime-get-status.ts index 3451cd8af30..26e9abda8b3 100644 --- a/src/main/runtime/orca-runtime-get-status.ts +++ b/src/main/runtime/orca-runtime-get-status.ts @@ -10,6 +10,7 @@ import { BROWSER_CERTIFICATE_TRUST_RUNTIME_CAPABILITY, BROWSER_HEADLESS_RUNTIME_CAPABILITY, BROWSER_IDENTITY_RUNTIME_CAPABILITY, + MANAGED_SERVER_RUNTIME_CAPABILITY, MIN_COMPATIBLE_RUNTIME_CLIENT_VERSION, REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY, RUNTIME_CAPABILITIES, @@ -38,6 +39,7 @@ import { parsePaneKey } from '../../shared/stable-pane-id' import { wakeFolderRepoGitUpgradeWatch } from '../ipc/folder-repo-git-upgrade-wake' import { runWorktreeChangeInvalidators } from '../ipc/worktree-change-invalidators' import { MACHINE_NAME_PUBLISH_WAIT_MS } from './runtime-machine-name' +import { getManagedServerActions } from './managed-server-actions-registry' type RuntimeStatusHost = { getAvailableAuthoritativeWindow(): unknown @@ -108,6 +110,9 @@ export class OrcaRuntimeWithGetStatus extends OrcaRuntimeWithGetRuntimeId { if (canBrowse) { capabilities.push(BROWSER_CERTIFICATE_TRUST_RUNTIME_CAPABILITY) } + if (getManagedServerActions()) { + capabilities.push(MANAGED_SERVER_RUNTIME_CAPABILITY) + } // Why not a static capability: dev trees and `orca serve` installs may carry no // out/mobile-web, and advertising a bundle this install cannot produce would promise a // download that only ever answers mobile_web_bundle_unavailable. diff --git a/src/main/runtime/orca-runtime-get-worktree-ps.ts b/src/main/runtime/orca-runtime-get-worktree-ps.ts index 8b7d19af50b..174fcae7085 100644 --- a/src/main/runtime/orca-runtime-get-worktree-ps.ts +++ b/src/main/runtime/orca-runtime-get-worktree-ps.ts @@ -9,6 +9,7 @@ import { applyRuntimeWorktreePsSessionActivity, applyRuntimeWorktreePsTerminalActivity } from './runtime-worktree-ps-activity' +import { applyRuntimeWorktreePsUnverifiableTerminals } from './runtime-worktree-ps-unverifiable-terminals' import { attachRuntimeWorktreeAgentRows } from './runtime-worktree-agent-rows' import { compareWorktreePs } from './runtime-worktree-status-projection' import type { Repo } from '../../shared/repo-types' @@ -86,11 +87,13 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis ) const missingRuntimeWorktreeIds = new Set() const session = this.store?.getWorkspaceSession?.() + const countedPtyIds = new Set() const workingTerminalEvidenceByWorktreeId = applyRuntimeWorktreePsTerminalActivity({ summaries, pathIndex: runtimeWorktreeSummaryPathIndex, missingIds: missingRuntimeWorktreeIds, freshPtyLiveness, + countedPtyIds, leaves: this.leaves.values(), ptysById: this.ptysById, tabs: this.tabs, @@ -100,6 +103,17 @@ export class OrcaRuntimeWithGetWorktreePs extends OrcaRuntimeWithStartTuiIdleVis getSummary: (summaryMap, pathIndex, missingIds, worktreeId) => this.getSummaryForRuntimeWorktreeId(summaryMap, pathIndex, missingIds, worktreeId) }) + applyRuntimeWorktreePsUnverifiableTerminals({ + summaries, + pathIndex: runtimeWorktreeSummaryPathIndex, + missingIds: missingRuntimeWorktreeIds, + countedPtyIds, + leaves: this.leaves.values(), + ptysById: this.ptysById, + getLivenessVerdict: (ptyId) => this.getPtyLivenessVerdict(ptyId), + getSummary: (summaryMap, pathIndex, missingIds, worktreeId) => + this.getSummaryForRuntimeWorktreeId(summaryMap, pathIndex, missingIds, worktreeId) + }) const { mirroredWorktreeIdByTabId, connectedPtyEvidence } = applyRuntimeWorktreePsSessionActivity({ store: this.store, diff --git a/src/main/runtime/orca-runtime-has-exact-persisted-terminal-surface-identity.ts b/src/main/runtime/orca-runtime-has-exact-persisted-terminal-surface-identity.ts index 265f2ac28c0..13f65c30374 100644 --- a/src/main/runtime/orca-runtime-has-exact-persisted-terminal-surface-identity.ts +++ b/src/main/runtime/orca-runtime-has-exact-persisted-terminal-surface-identity.ts @@ -205,6 +205,10 @@ export class OrcaRuntimeWithHasExactPersistedTerminalSurfaceIdentity extends Orc this.automation.setService(service) } + releaseFinishedAutomationRunTerminals(): Promise { + return this.automation.releaseFinishedRunTerminals() + } + setArtifactService(service: ArtifactCloudService): void { this.artifacts.setService(service) } diff --git a/src/main/runtime/orca-runtime-headless-terminal-close.test.ts b/src/main/runtime/orca-runtime-headless-terminal-close.test.ts new file mode 100644 index 00000000000..425a84c7bf9 --- /dev/null +++ b/src/main/runtime/orca-runtime-headless-terminal-close.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { + PTY_ID, + TAB_ID, + WORKTREE_ID, + createHarness, + type CloseContinuityHarness +} from './__fixtures__/orca-runtime-terminal-close-continuity-fixtures' + +async function closeByPty(harness: CloseContinuityHarness): Promise { + const terminal = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals.find( + (candidate) => candidate.ptyId === PTY_ID + ) + if (!terminal) { + throw new Error('fixture pane has no terminal handle') + } + return harness.runtime.closeTerminal(terminal.handle) +} + +/** A PTY-backed tab with no paired-viewer surface: the path orcad serves to the CLI. */ +function headlessPtyTab(): CloseContinuityHarness { + const harness = createHarness({ registerPtyBacked: true }) + harness.syncFixtureTabWithoutLeaf() + harness.setVerifiedStopResult(true) + return harness +} + +describe('closing a terminal by PTY on a host without a renderer tab', () => { + it('stops the PTY without asking a missing renderer to close the tab', async () => { + const harness = headlessPtyTab() + harness.syncEmptyGraph() + + await expect(closeByPty(harness)).resolves.toMatchObject({ tabId: TAB_ID }) + + expect(harness.closeTerminalTab).not.toHaveBeenCalled() + expect(harness.stopAndWait).toHaveBeenCalledWith(PTY_ID, expect.anything()) + }) + + it('does not fail the close when the advisory renderer notification throws', async () => { + const harness = headlessPtyTab() + harness.syncEmptyGraph() + harness.closeTerminal.mockImplementation(() => { + throw new Error('renderer gone') + }) + + await expect(closeByPty(harness)).resolves.toMatchObject({ tabId: TAB_ID }) + }) +}) diff --git a/src/main/runtime/orca-runtime-hydrate-headless-mobile-session-tabs-from-workspace-session.ts b/src/main/runtime/orca-runtime-hydrate-headless-mobile-session-tabs-from-workspace-session.ts index 0f0bde61d4a..49fdc91573a 100644 --- a/src/main/runtime/orca-runtime-hydrate-headless-mobile-session-tabs-from-workspace-session.ts +++ b/src/main/runtime/orca-runtime-hydrate-headless-mobile-session-tabs-from-workspace-session.ts @@ -4,8 +4,8 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state import { getRuntimeBrowserPageRegistry } from './runtime-browser-page-registry' import { splitWorktreeIdForFilesystem } from '../../shared/worktree/id' import { buildHeadlessMobileSessionTerminalTabs } from './mobile-session-terminal-projection' +import { buildHeadlessMobileSessionEditorTabs } from './mobile-session-editor-projection' import type { - RuntimeMobileSessionBrowserTab, RuntimeMobileSessionSnapshotTab, RuntimeMobileSessionTabGroup, RuntimeMobileSessionTabsSnapshot, @@ -73,10 +73,16 @@ export class OrcaRuntimeWithHydrateHeadlessMobileSessionTabsFromWorkspaceSession ) { return reconciledWorktreeIds } + // A worktree whose only tabs are editors has no tabsByWorktree key. const entries = worktreeId !== undefined ? ([[worktreeId, session.tabsByWorktree[worktreeId] ?? []]] as const) - : Object.entries(session.tabsByWorktree ?? {}) + : [ + ...new Set([ + ...Object.keys(session.tabsByWorktree ?? {}), + ...Object.keys(session.openFilesByWorktree ?? {}) + ]) + ].map((id) => [id, session.tabsByWorktree?.[id] ?? []] as const) // Why: workspaceSession keys are `${repoId}::${path}` and are not pruned when // a repo disappears from this client's view (e.g. removed on another client, // or a stale browser-persisted session). Hydrating such a key would surface a @@ -130,13 +136,22 @@ export class OrcaRuntimeWithHydrateHeadlessMobileSessionTabsFromWorkspaceSession // so include them on every hydrate regardless of the onlyRuntimeOwnedTerminals // filter, which is about terminal PTY ownership and never applies to browsers. const browserTabs = this.buildHeadlessMobileSessionBrowserTabs(entryWorktreeId) - const tabs: RuntimeMobileSessionSnapshotTab[] = [...terminalTabs, ...browserTabs] + // Why not in the runtime-owned pass: that merges into a renderer's publication, which owns its editors. + const editorTabs = runtimeOwnedOnly + ? [] + : buildHeadlessMobileSessionEditorTabs(entryWorktreeId, session) + const tabs: RuntimeMobileSessionSnapshotTab[] = [ + ...terminalTabs, + ...editorTabs, + ...browserTabs + ] if (tabs.length === 0) { continue } const activeTab = pickHeadlessActiveTerminalTab(terminalTabs) const tabOrder = [ ...collectHeadlessParentTabOrder(terminalTabs), + ...editorTabs.map((tab) => tab.id), ...browserTabs.map((tab) => tab.id) ] const groupId = getHeadlessMobileSessionGroupId(entryWorktreeId) @@ -147,13 +162,15 @@ export class OrcaRuntimeWithHydrateHeadlessMobileSessionTabsFromWorkspaceSession const mergedActiveTab = existing?.tabs.find((tab) => tab.id === existing.activeTabId) ?? activeTab ?? + editorTabs.find((tab) => tab.isActive) ?? mergedTabs[0] ?? null const mergedTerminalTabs = mergedTabs.filter( (tab): tab is RuntimeMobileSessionTerminalTab => tab.type === 'terminal' ) + // Editors ride with browsers: both keep their persisted group, unlike terminal parents. const mergedBrowserOrder = mergedTabs - .filter((tab): tab is RuntimeMobileSessionBrowserTab => tab.type === 'browser') + .filter((tab) => tab.type === 'browser' || tab.type === 'markdown' || tab.type === 'file') .map((tab) => tab.id) // Why: a persisted multi-group split must be restored on cold rebuild, or // the headless serve coalesces the user's group layout back into one group diff --git a/src/main/runtime/orca-runtime-mark-pty-liveness-unverifiable.ts b/src/main/runtime/orca-runtime-mark-pty-liveness-unverifiable.ts index 446083fff03..b022dfa3a47 100644 --- a/src/main/runtime/orca-runtime-mark-pty-liveness-unverifiable.ts +++ b/src/main/runtime/orca-runtime-mark-pty-liveness-unverifiable.ts @@ -109,6 +109,28 @@ export class OrcaRuntimeWithMarkPtyLivenessUnverifiable extends OrcaRuntimeWithO return unsubscribe } + /** Resolves once a registered PTY's exit reaches its runtime record; false on timeout. */ + waitForPtyExitRecord(ptyId: string, timeoutMs: number): Promise { + if (!this.ptysById.has(ptyId) || this.isPtyKnownExited(ptyId)) { + return Promise.resolve(true) + } + return new Promise((resolve) => { + let unsubscribe = (): void => {} + const timer = setTimeout( + () => { + unsubscribe() + resolve(false) + }, + Math.max(0, timeoutMs) + ) + timer.unref?.() + unsubscribe = this.subscribeToPtyExit(ptyId, () => { + clearTimeout(timer) + resolve(true) + }) + }) + } + protected rememberPtyLivenessVerdict(ptyId: string, verdict: PtyLivenessVerdict): void { // An earned death certificate is KEPT, not dropped, so the register is three-valued on disk as // well as in the type. Its only writer is a host-delivered exit frame; nothing weaker may diff --git a/src/main/runtime/orca-runtime-merge-preserved-headless-mobile-session-tabs.ts b/src/main/runtime/orca-runtime-merge-preserved-headless-mobile-session-tabs.ts index 8e282c79dfe..2ea80067093 100644 --- a/src/main/runtime/orca-runtime-merge-preserved-headless-mobile-session-tabs.ts +++ b/src/main/runtime/orca-runtime-merge-preserved-headless-mobile-session-tabs.ts @@ -11,7 +11,7 @@ import { mergeMobileSessionSnapshotTabs, mergeMobileSessionTabGroups } from './mobile-session-tab-merge' -import { terminalLayoutContainsLeaf } from './headless-terminal-split-layout' +import { terminalLayoutContainsLeaf } from '../../shared/workspace-session-pane-ownership' import { getHeadlessMobileSessionGroupId } from './mobile-session-layout-projection' export class OrcaRuntimeWithMergePreservedHeadlessMobileSessionTabs extends OrcaRuntimeWithSyncMobileSessionTabs { diff --git a/src/main/runtime/orca-runtime-migration-catalog.ts b/src/main/runtime/orca-runtime-migration-catalog.ts new file mode 100644 index 00000000000..02953076a24 --- /dev/null +++ b/src/main/runtime/orca-runtime-migration-catalog.ts @@ -0,0 +1,90 @@ +import type { + OrcadMigrationCatalogAbortResult, + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { + OrcadMigrationSnapshotChunkRequest, + OrcadMigrationSnapshotChunkResult +} from '../../shared/orcad-migration-scrollback' +import { + abortStagedOrcadMigrationCatalogDurably, + commitStagedOrcadMigrationCatalogDurably, + getOrcadMigrationCatalogState, + stageOrcadMigrationCatalogDurably +} from './orcad-migration-catalog-import' +import { OrcaRuntimeWithResolveWaiter } from './orca-runtime-resolve-waiter' +import type { RuntimeStore } from './runtime-store-contract' +import type { Store } from '../persistence' + +/** The destination half of a dormant catalog migration; every mutation is flushed before it answers. */ +export class OrcaRuntimeWithMigrationCatalog extends OrcaRuntimeWithResolveWaiter { + async stageOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + options: { signal?: AbortSignal } = {} + ): Promise { + return stageOrcadMigrationCatalogDurably({ + store: this.requireMigrationStore('stageOrcadMigrationCatalog', 'flushPendingOrThrowAsync'), + manifest, + signal: options.signal + }) + } + + async commitStagedOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + options: { signal?: AbortSignal } = {} + ): Promise { + return commitStagedOrcadMigrationCatalogDurably({ + store: this.requireMigrationStore( + 'commitStagedOrcadMigrationCatalog', + 'flushPendingOrThrowAsync' + ), + manifest, + signal: options.signal, + onDurableCommit: () => { + this.invalidateResolvedWorktreeCache() + this.notifyReposChanged() + } + }) + } + + stageOrcadMigrationSnapshotChunk( + request: OrcadMigrationSnapshotChunkRequest + ): OrcadMigrationSnapshotChunkResult { + return this.requireMigrationStore( + 'stageOrcadMigrationSnapshotChunk' + ).stageOrcadMigrationSnapshotChunk(request) + } + + async abortStagedOrcadMigrationCatalog( + manifest: OrcadMigrationManifest, + options: { signal?: AbortSignal } = {} + ): Promise { + return abortStagedOrcadMigrationCatalogDurably({ + store: this.requireMigrationStore( + 'abortStagedOrcadMigrationCatalog', + 'flushPendingOrThrowAsync' + ), + manifest, + signal: options.signal + }) + } + + getOrcadMigrationCatalogState(manifest: OrcadMigrationManifest): OrcadMigrationCatalogState { + return getOrcadMigrationCatalogState({ + store: this.requireMigrationStore('getOrcadMigrationCatalogState'), + manifest + }) + } + + /** A partial store (tests, headless hosts) lacks some optional methods; refuse rather than half-run. */ + private requireMigrationStore( + ...methods: K[] + ): Pick { + const store = this.store + if (!store || methods.some((method) => typeof store[method] !== 'function')) { + throw new Error('runtime_unavailable') + } + return this.requireStore() + } +} diff --git a/src/main/runtime/orca-runtime-preserved-branch-cleanup.ts b/src/main/runtime/orca-runtime-preserved-branch-cleanup.ts index 7ea35dcdb4d..21f0d73cf7a 100644 --- a/src/main/runtime/orca-runtime-preserved-branch-cleanup.ts +++ b/src/main/runtime/orca-runtime-preserved-branch-cleanup.ts @@ -1,5 +1,6 @@ // @ts-nocheck -- mechanically split from OrcaRuntimeService; behavior is covered by AST equivalence and characterization tests. import { OrcaRuntimeWithTerminalDrivers } from './orca-runtime-terminal-drivers' +import { confirmRunTerminalShellAlone, readRunTerminalClientUse } from './run-terminal-client-use' import { ALL_EXECUTION_HOSTS_SCOPE, type ExecutionHostScope } from '../../shared/execution-host' import { RuntimePreservedBranchCleanup } from './runtime-preserved-branch-cleanup' import type { IPtyProvider } from '../providers/types' @@ -85,6 +86,25 @@ export class OrcaRuntimeWithPreservedBranchCleanup extends OrcaRuntimeWithTermin | ((paneKey: string) => AgentStatusIpcPayload[]) | null + /** See run-terminal-client-use.ts. */ + readTerminalClientUse(ptyId: string): 'used' | 'unused' | 'unknown' { + return readRunTerminalClientUse(this, ptyId) + } + + confirmTerminalShellAlone(ptyId: string): Promise { + return confirmRunTerminalShellAlone(this.ptyController, ptyId) + } + + /** The PTY a terminal handle drives now; a restarted pane answers with its new PTY. */ + getTerminalPtyIdForHandle(handle: string): string | null { + return this.getLivePtyForHandle(handle)?.pty.ptyId ?? null + } + + /** Agent status rows this host holds for a pane, from hooks, OSC and titles alike. */ + getAgentStatusRowsForPane(paneKey: string): AgentStatusIpcPayload[] { + return this.getAgentProviderSessionRowsForPaneFn?.(paneKey) ?? [] + } + protected readonly attestAgentHookCompatibilityAuthorityFn: | ((candidate: { paneKey: string diff --git a/src/main/runtime/orca-runtime-register-pty.ts b/src/main/runtime/orca-runtime-register-pty.ts index bd9c96a5c0f..cdb8542ba60 100644 --- a/src/main/runtime/orca-runtime-register-pty.ts +++ b/src/main/runtime/orca-runtime-register-pty.ts @@ -27,8 +27,17 @@ export class OrcaRuntimeWithRegisterPty extends OrcaRuntimeWithInvalidateAllHand isWsl?: boolean ): void { this.assertPtyDidNotExitBeforeRegistration(ptyId, binding?.incarnationId) - this.invalidatePtyControllerInventoryForLifecycle(ptyId, connectionId) const existingPty = this.ptysById.get(ptyId) + // Why: a relay reattach can finish registering after a stop recorded that incarnation's exit. + if ( + binding?.incarnationId !== undefined && + existingPty?.incarnationId === binding.incarnationId && + !existingPty.connected && + this.getPtyLivenessVerdict(ptyId)?.status === 'exited' + ) { + return + } + this.invalidatePtyControllerInventoryForLifecycle(ptyId, connectionId) const replacementHandle = binding?.terminalHandle?.trim() const pendingReplacement = this.pendingPtyHandleReplacementFences.get(ptyId) const pendingReplacementMatches = diff --git a/src/main/runtime/orca-runtime-resolve-terminal-split-source-authority.ts b/src/main/runtime/orca-runtime-resolve-terminal-split-source-authority.ts index 167aee3e21b..4b63ec021c8 100644 --- a/src/main/runtime/orca-runtime-resolve-terminal-split-source-authority.ts +++ b/src/main/runtime/orca-runtime-resolve-terminal-split-source-authority.ts @@ -5,7 +5,7 @@ import { runtimeWorktreeIdsEqual } from './runtime-worktree-path-identity' import { makePaneKey } from '../../shared/stable-pane-id' -import { terminalLayoutContainsLeaf } from './headless-terminal-split-layout' +import { terminalLayoutContainsLeaf } from '../../shared/workspace-session-pane-ownership' import type { AgentTeamsTmuxCompatRequest, AgentTeamsTmuxCompatResponse diff --git a/src/main/runtime/orca-runtime-stop-explicitly-closed-tab-ptys.ts b/src/main/runtime/orca-runtime-stop-explicitly-closed-tab-ptys.ts index a243d5ea0a6..ea954fbce8b 100644 --- a/src/main/runtime/orca-runtime-stop-explicitly-closed-tab-ptys.ts +++ b/src/main/runtime/orca-runtime-stop-explicitly-closed-tab-ptys.ts @@ -130,7 +130,7 @@ export class OrcaRuntimeWithStopExplicitlyClosedTabPtys extends OrcaRuntimeWithF if (!(error instanceof Error) || error.message !== 'workspace_session_unavailable') { throw error } - this.notifier.closeTerminal?.(tabId) + this.notifyRendererOfHeadlessTerminalClose(tabId) } const stop = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) @@ -147,17 +147,35 @@ export class OrcaRuntimeWithStopExplicitlyClosedTabPtys extends OrcaRuntimeWithF throw error } const stop = await this.stopExplicitlyClosedTabPtys([pty.pty.ptyId], pty.pty.ptyId) - this.notifier?.closeTerminal(tabId) + this.notifyRendererOfHeadlessTerminalClose(tabId) return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) } const stop = await this.stopExplicitlyClosedTabPtys([pty.pty.ptyId], pty.pty.ptyId) return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) } - if (closesTab && !surface && pty.pty.tabId && this.notifier?.closeTerminalTab) { + // Why the tabs guard: a headless host has no renderer tab to close through the notifier. + if ( + closesTab && + !surface && + pty.pty.tabId && + this.tabs.has(tabId) && + this.notifier?.closeTerminalTab + ) { const ptyIdsToKill = this.getPtyIdsForExplicitTabClose(pty.pty.worktreeId, tabId) - await this.notifier.closeTerminalTab(tabId, { localPtyTeardownOwnedExternally: true }) - const stop = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) - return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) + let tabClosed = true + try { + await this.notifier.closeTerminalTab(tabId, { localPtyTeardownOwnedExternally: true }) + } catch (error) { + // The tab went away concurrently; fall through and close the PTY alone. + if (!(error instanceof Error) || error.message !== 'tab_not_found') { + throw error + } + tabClosed = false + } + if (tabClosed) { + const stop = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) + return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) + } } const stop = await this.stopExplicitlyClosedTabPtys([pty.pty.ptyId], pty.pty.ptyId) if (!closesTab) { @@ -176,10 +194,10 @@ export class OrcaRuntimeWithStopExplicitlyClosedTabPtys extends OrcaRuntimeWithF if (!(error instanceof Error) || error.message !== 'workspace_session_unavailable') { throw error } - this.notifier?.closeTerminal(tabId) + this.notifyRendererOfHeadlessTerminalClose(tabId) } } else { - this.notifier?.closeTerminal(tabId) + this.notifyRendererOfHeadlessTerminalClose(tabId) } return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, stop) } diff --git a/src/main/runtime/orca-runtime-tests/managed-server-capability.spec.ts b/src/main/runtime/orca-runtime-tests/managed-server-capability.spec.ts new file mode 100644 index 00000000000..a895c8e63da --- /dev/null +++ b/src/main/runtime/orca-runtime-tests/managed-server-capability.spec.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { createRuntime } from '../orca-runtime-test-fixtures.spec' +import { + registerManagedServerActions, + type ManagedServerActions +} from '../managed-server-actions-registry' + +describe('managed server capability', () => { + // A runtime without the desktop's SSH registry must not point clients at methods that refuse. + it('is advertised only where the desktop registered the managed-server actions', () => { + registerManagedServerActions(null) + expect(createRuntime().getStatus().capabilities).not.toContain('managedServer.v1') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: getStatus only checks that actions are registered. + registerManagedServerActions({} as ManagedServerActions) + try { + expect(createRuntime().getStatus().capabilities).toContain('managedServer.v1') + } finally { + registerManagedServerActions(null) + } + }) +}) diff --git a/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts b/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts index 965a1a1bc3f..292a183e251 100644 --- a/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts +++ b/src/main/runtime/orca-runtime-tests/mobile-creation-and-orchestration-part-04.spec.ts @@ -162,6 +162,7 @@ describe('OrcaRuntimeService', () => { unread: false, liveTerminalCount: 1, hasAttachedPty: true, + unverifiableTerminalCount: 0, lastActivityAt: 0, lastOutputAt: 321, preview: 'build green', diff --git a/src/main/runtime/orca-runtime-tests/mobile-session-tabs-cold-serve-hydrate.spec.ts b/src/main/runtime/orca-runtime-tests/mobile-session-tabs-cold-serve-hydrate.spec.ts index 49369bdd030..81cc968e184 100644 --- a/src/main/runtime/orca-runtime-tests/mobile-session-tabs-cold-serve-hydrate.spec.ts +++ b/src/main/runtime/orca-runtime-tests/mobile-session-tabs-cold-serve-hydrate.spec.ts @@ -79,6 +79,29 @@ describe('OrcaRuntimeService', () => { expect(terminalPtyIds(listed?.tabs ?? [])).toEqual([DAEMON_PTY_ID, SERVE_PTY_ID].sort()) }) + it('lists persisted editor tabs beside the terminals after a cold restart', async () => { + const runtime = makeRestartedServeRuntime({ + openFilesByWorktree: { + [TEST_WORKTREE_ID]: [ + { + filePath: '/repo/README.md', + relativePath: 'README.md', + worktreeId: TEST_WORKTREE_ID, + language: 'markdown', + runtimeEnvironmentId: null + } + ] + } + }) + + const listed = await runtime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + + expect(terminalPtyIds(listed.tabs)).toEqual([DAEMON_PTY_ID, SERVE_PTY_ID].sort()) + expect(listed.tabs.filter((tab) => tab.type === 'markdown').map((tab) => tab.id)).toEqual([ + '/repo/README.md' + ]) + }) + it('restores a persisted split group layout on the cold rebuild', async () => { const runtime = makeRestartedServeRuntime({ tabGroups: { diff --git a/src/main/runtime/orca-runtime-tests/mobile-session-tabs-part-14.spec.ts b/src/main/runtime/orca-runtime-tests/mobile-session-tabs-part-14.spec.ts new file mode 100644 index 00000000000..672e75abec1 --- /dev/null +++ b/src/main/runtime/orca-runtime-tests/mobile-session-tabs-part-14.spec.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../orca-runtime-test-mocks.spec' +import { + TEST_WORKTREE_ID, + TEST_WORKTREE_PATH, + makeRuntimeStoreWithWorkspaceSession +} from '../orca-runtime-test-fixtures.spec' +import { getDefaultWorkspaceSession } from '../../../shared/constants' +import type { WorkspaceSessionState } from '../../../shared/workspace-session-state-types' + +const README = `${TEST_WORKTREE_PATH}/README.md` + +/** The editor a converted SSH host's migration writes into the managed server's session. */ +function sessionWithMigratedEditor(): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + openFilesByWorktree: { + [TEST_WORKTREE_ID]: [ + { + filePath: README, + relativePath: 'README.md', + worktreeId: TEST_WORKTREE_ID, + language: 'markdown', + runtimeEnvironmentId: null + } + ] + }, + unifiedTabs: { + [TEST_WORKTREE_ID]: [ + { + id: 'tab-readme', + entityId: README, + groupId: 'group-1', + worktreeId: TEST_WORKTREE_ID, + contentType: 'editor', + label: 'README.md', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + tabGroups: { + [TEST_WORKTREE_ID]: [ + { + id: 'group-1', + worktreeId: TEST_WORKTREE_ID, + activeTabId: 'tab-readme', + tabOrder: ['tab-readme'] + } + ] + }, + activeTabTypeByWorktree: { [TEST_WORKTREE_ID]: 'editor' }, + activeFileIdByWorktree: { [TEST_WORKTREE_ID]: README } + } +} + +function headlessRuntime(session: WorkspaceSessionState) { + const harness = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService(harness.runtimeStore) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: vi.fn(async () => []) + }) + runtime.syncWindowGraph(0, { tabs: [], leaves: [] }) + return { runtime, getSession: harness.getSession } +} + +describe('editor tabs a headless host persisted', () => { + it('lists them, so a paired client can mirror a migrated editor', async () => { + const { runtime } = headlessRuntime(sessionWithMigratedEditor()) + + const listed = await runtime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + + expect(listed.tabs).toEqual([ + expect.objectContaining({ + type: 'markdown', + id: 'tab-readme', + filePath: README, + relativePath: 'README.md', + isActive: true + }) + ]) + expect(listed.activeTabId).toBe('tab-readme') + }) + + it('closes one by retiring it from the host session', async () => { + const { runtime, getSession } = headlessRuntime(sessionWithMigratedEditor()) + await runtime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + + await runtime.closeMobileSessionTab(`id:${TEST_WORKTREE_ID}`, 'tab-readme') + + expect(getSession().openFilesByWorktree?.[TEST_WORKTREE_ID]).toEqual([]) + expect(getSession().unifiedTabs?.[TEST_WORKTREE_ID]).toEqual([]) + expect((await runtime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`)).tabs).toEqual([]) + }) + + it('keeps an unsaved draft unless the close is forced', async () => { + const session = sessionWithMigratedEditor() + const [file] = session.openFilesByWorktree?.[TEST_WORKTREE_ID] ?? [] + session.openFilesByWorktree = { [TEST_WORKTREE_ID]: [{ ...file, dirtyDraftContent: 'draft' }] } + const { runtime, getSession } = headlessRuntime(session) + await runtime.listMobileSessionTabs(`id:${TEST_WORKTREE_ID}`) + + await expect( + runtime.closeMobileSessionTab(`id:${TEST_WORKTREE_ID}`, 'tab-readme') + ).rejects.toThrow('editor_tab_has_unsaved_draft') + expect(getSession().openFilesByWorktree?.[TEST_WORKTREE_ID]?.[0]?.dirtyDraftContent).toBe( + 'draft' + ) + + await runtime.closeMobileSessionTab(`id:${TEST_WORKTREE_ID}`, 'tab-readme', { force: true }) + expect(getSession().openFilesByWorktree?.[TEST_WORKTREE_ID]).toEqual([]) + }) +}) diff --git a/src/main/runtime/orca-runtime-tests/worktree-terminal-close-reattached-relay.spec.ts b/src/main/runtime/orca-runtime-tests/worktree-terminal-close-reattached-relay.spec.ts new file mode 100644 index 00000000000..8036e404194 --- /dev/null +++ b/src/main/runtime/orca-runtime-tests/worktree-terminal-close-reattached-relay.spec.ts @@ -0,0 +1,135 @@ +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../orca-runtime-test-mocks.spec' +import { + HEADLESS_LEAF_ID, + TEST_REPO_ID, + TEST_WORKTREE_ID, + makeHeadlessTerminalLayout, + makeRuntimeStoreWithWorkspaceSession, + makeWorkspaceSessionWithHeadlessTerminal, + store +} from '../orca-runtime-test-fixtures.spec' + +const CONNECTION_ID = 'conn-relay' +const REATTACHED_PTY_ID = `ssh:${CONNECTION_ID}@@pty-1` +const INCARNATION_ID = 'relay-incarnation-1' + +// A relay lease from an earlier app launch, reattached by this one, then closed from the CLI. +function makeReattachedRelayRuntime(): OrcaRuntimeService { + const session = makeWorkspaceSessionWithHeadlessTerminal({ + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: 'host-tab', + ptyId: REATTACHED_PTY_ID, + worktreeId: TEST_WORKTREE_ID, + title: 'Relay Terminal', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'host-tab': makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: REATTACHED_PTY_ID }) + } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const repo = { ...store.getRepo(TEST_REPO_ID)!, connectionId: CONNECTION_ID } + runtimeStore.getRepos = () => [repo] + runtimeStore.getRepo = (id: string) => (id === TEST_REPO_ID ? repo : undefined) + const runtime = new OrcaRuntimeService(runtimeStore) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + return runtime +} + +function reattach(runtime: OrcaRuntimeService): void { + runtime.registerPty(REATTACHED_PTY_ID, TEST_WORKTREE_ID, CONNECTION_ID, { + tabId: 'host-tab', + leafId: HEADLESS_LEAF_ID, + incarnationId: INCARNATION_ID + }) +} + +describe('closing a workspace whose relay terminal was reattached from an earlier launch', () => { + it('reports the host-confirmed exit instead of doubting the retained SSH record', async () => { + const runtime = makeReattachedRelayRuntime() + reattach(runtime) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait: vi.fn(async (ptyId: string) => { + runtime.onPtyExit(ptyId, 0, INCARNATION_ID) + return true + }), + getForegroundProcess: async () => null + }) + + await expect( + runtime.closeTerminalsForWorktree(`id:${TEST_WORKTREE_ID}`) + ).resolves.not.toHaveProperty('ptyStopVerdict') + expect(runtime.getPtyLivenessVerdict(REATTACHED_PTY_ID)).toEqual({ status: 'exited' }) + }) + + it('accepts the host exit even when the stop confirmation carries no incarnation', async () => { + const runtime = makeReattachedRelayRuntime() + reattach(runtime) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait: vi.fn(async (ptyId: string) => { + // Field shape: the reattached relay exit arrives with code 1 and no incarnation. + runtime.onPtyExit(ptyId, 1, undefined, { hostExitConfirmed: true }) + return true + }), + getForegroundProcess: async () => null + }) + + await expect(runtime.closeTerminalsForWorktree(`id:${TEST_WORKTREE_ID}`)).resolves.toEqual( + expect.not.objectContaining({ ptyStopVerdict: expect.anything() }) + ) + expect(runtime.getPtyLivenessVerdict(REATTACHED_PTY_ID)).toEqual({ status: 'exited' }) + }) + + it('keeps the exit when the reattach registration lands after the stop', async () => { + const runtime = makeReattachedRelayRuntime() + reattach(runtime) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait: vi.fn(async (ptyId: string) => { + runtime.onPtyExit(ptyId, 0, INCARNATION_ID) + // The reattach that proved this incarnation alive finishes registering only now. + reattach(runtime) + return true + }), + getForegroundProcess: async () => null + }) + + await expect( + runtime.closeTerminalsForWorktree(`id:${TEST_WORKTREE_ID}`) + ).resolves.not.toHaveProperty('ptyStopVerdict') + expect(runtime.getPtyLivenessVerdict(REATTACHED_PTY_ID)).toEqual({ status: 'exited' }) + }) + + it('still doubts a retained SSH record whose exit was never confirmed', async () => { + const runtime = makeReattachedRelayRuntime() + reattach(runtime) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait: vi.fn(async () => false), + getForegroundProcess: async () => null + }) + + await expect( + runtime.closeTerminalsForWorktree(`id:${TEST_WORKTREE_ID}`) + ).resolves.toMatchObject({ + stopped: 0, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'the owning host did not confirm the PTY exit' + }) + }) +}) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 297c0ab6055..a6ddef6650d 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -12,6 +12,7 @@ await import('./orca-runtime-tests/runtime-availability.spec') await import('./orca-runtime-tests/browser-capabilities.spec') await import('./orca-runtime-tests/browser-capabilities-part-02.spec') await import('./orca-runtime-tests/mobile-web-bundle-capability.spec') +await import('./orca-runtime-tests/managed-server-capability.spec') await import('./orca-runtime-tests/window-authority.spec') await import('./orca-runtime-tests/terminal-handles.spec') await import('./orca-runtime-tests/terminal-handles-part-02.spec') @@ -80,6 +81,7 @@ await import('./orca-runtime-tests/mobile-session-tabs-part-10.spec') await import('./orca-runtime-tests/mobile-session-tabs-part-11.spec') await import('./orca-runtime-tests/mobile-session-tabs-part-12.spec') await import('./orca-runtime-tests/mobile-session-tabs-part-13.spec') +await import('./orca-runtime-tests/mobile-session-tabs-part-14.spec') await import('./orca-runtime-tests/mobile-session-tabs-cold-serve-hydrate.spec') await import('./orca-runtime-tests/exit-retirement-activation.spec') await import('./orca-runtime-tests/mobile-creation-and-orchestration.spec') @@ -95,6 +97,7 @@ await import('./orca-runtime-tests/worktree-ps-agent-row-dismissal.spec') await import('./orca-runtime-tests/terminal-sleep-and-teardown.spec') await import('./orca-runtime-tests/terminal-sleep-and-teardown-part-02.spec') await import('./orca-runtime-tests/terminal-sleep-and-teardown-part-03.spec') +await import('./orca-runtime-tests/worktree-terminal-close-reattached-relay.spec') await import('./orca-runtime-tests/lineage-and-scan-cache.spec') await import('./orca-runtime-tests/lineage-and-scan-cache-part-02.spec') await import('./orca-runtime-tests/lineage-and-scan-cache-part-03.spec') diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index c4078cb2948..5dc601b844e 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -1,5 +1,5 @@ import { installRuntimeLinearCommandSurface } from './runtime-linear-command-surface' -import { OrcaRuntimeWithResolveWaiter } from './orca-runtime-resolve-waiter' +import { OrcaRuntimeWithMigrationCatalog } from './orca-runtime-migration-catalog' import type { RuntimeCommandSurfaceHost } from './orca-runtime-core' import type { AgentLaunchTabPublished, @@ -15,8 +15,8 @@ import { peekOpenedAgentSessionRecordStore } from './agent-session-record-store- import { createAgentLaunchRecordWarmupGate } from './agent-launch-record-warmup-gate' import { registerDetectedWorktreeScanInvalidation } from '../ipc/worktrees/listing/register-detected-worktree-scan-invalidation' -class OrcaRuntimeService extends OrcaRuntimeWithResolveWaiter { - constructor(...args: ConstructorParameters) { +class OrcaRuntimeService extends OrcaRuntimeWithMigrationCatalog { + constructor(...args: ConstructorParameters) { super(...args) // Why: the runtime listing re-runs a scan the worktree-change generation overtook and re-lists // through this runtime's scan cache, so a worktree change must reach both. The desktop IPC diff --git a/src/main/runtime/orcad-migration-catalog-import.test.ts b/src/main/runtime/orcad-migration-catalog-import.test.ts new file mode 100644 index 00000000000..287deb17617 --- /dev/null +++ b/src/main/runtime/orcad-migration-catalog-import.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + OrcadMigrationCatalogAbortResult, + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import { + abortStagedOrcadMigrationCatalogDurably, + commitStagedOrcadMigrationCatalogDurably, + stageOrcadMigrationCatalogDurably +} from './orcad-migration-catalog-import' + +describe('durable orcad migration catalog import', () => { + it('does not acknowledge staging until its dormant manifest is durable', async () => { + const diskError = new Error('disk full') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + const staged = { state: 'staged' } as OrcadMigrationCatalogState + const stage = vi.fn().mockReturnValue(staged) + const flush = vi.fn().mockRejectedValueOnce(diskError).mockResolvedValueOnce(undefined) + const args = { + store: { + stageOrcadMigrationCatalog: stage, + flushPendingOrThrowAsync: flush + }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + manifest: {} as OrcadMigrationManifest + } + + await expect(stageOrcadMigrationCatalogDurably(args)).rejects.toBe(diskError) + await expect(stageOrcadMigrationCatalogDurably(args)).resolves.toBe(staged) + expect(stage).toHaveBeenCalledTimes(2) + expect(flush).toHaveBeenCalledTimes(2) + }) + + it('publishes a committed catalog only after its receipt is durable', async () => { + const diskError = new Error('disk full') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + const committed = { state: 'committed' } as OrcadMigrationCatalogState + const commit = vi.fn().mockReturnValue(committed) + const flush = vi.fn().mockRejectedValueOnce(diskError).mockResolvedValueOnce(undefined) + const onDurableCommit = vi.fn() + const args = { + store: { + commitStagedOrcadMigrationCatalog: commit, + flushPendingOrThrowAsync: flush + }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + manifest: {} as OrcadMigrationManifest, + onDurableCommit + } + + await expect(commitStagedOrcadMigrationCatalogDurably(args)).rejects.toBe(diskError) + expect(onDurableCommit).not.toHaveBeenCalled() + await expect(commitStagedOrcadMigrationCatalogDurably(args)).resolves.toBe(committed) + expect(onDurableCommit).toHaveBeenCalledOnce() + }) + + it('reflushes an already-absent abort after the first flush failed', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + const aborted = { state: 'absent', aborted: true } as OrcadMigrationCatalogAbortResult + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + const unchanged = { state: 'absent', aborted: false } as OrcadMigrationCatalogAbortResult + const abort = vi.fn().mockReturnValueOnce(aborted).mockReturnValueOnce(unchanged) + const flush = vi.fn().mockRejectedValueOnce(new Error('disk full')).mockResolvedValue(undefined) + const args = { + store: { + abortStagedOrcadMigrationCatalog: abort, + flushPendingOrThrowAsync: flush + }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the durable wrappers only pass these values through to the stubbed store. + manifest: {} as OrcadMigrationManifest + } + + await expect(abortStagedOrcadMigrationCatalogDurably(args)).rejects.toThrow('disk full') + await expect(abortStagedOrcadMigrationCatalogDurably(args)).resolves.toEqual({ + ...unchanged, + durableAbsent: true + }) + expect(flush).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/runtime/orcad-migration-catalog-import.ts b/src/main/runtime/orcad-migration-catalog-import.ts new file mode 100644 index 00000000000..4bfd87e3625 --- /dev/null +++ b/src/main/runtime/orcad-migration-catalog-import.ts @@ -0,0 +1,64 @@ +import type { + OrcadMigrationCatalogAbortResult, + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { Store } from '../persistence' + +type OrcadMigrationCatalogFlushStore = Pick +type OrcadMigrationCatalogStageStore = OrcadMigrationCatalogFlushStore & + Pick +type OrcadMigrationCatalogCommitStore = OrcadMigrationCatalogFlushStore & + Pick +type OrcadMigrationCatalogAbortStore = OrcadMigrationCatalogFlushStore & + Pick + +export async function stageOrcadMigrationCatalogDurably(args: { + store: OrcadMigrationCatalogStageStore + manifest: OrcadMigrationManifest + signal?: AbortSignal +}): Promise { + const result = args.store.stageOrcadMigrationCatalog(args.manifest) + await flushMigrationState(args.store, args.signal) + return result +} + +export async function commitStagedOrcadMigrationCatalogDurably(args: { + store: OrcadMigrationCatalogCommitStore + manifest: OrcadMigrationManifest + signal?: AbortSignal + onDurableCommit: () => void +}): Promise { + const result = args.store.commitStagedOrcadMigrationCatalog(args.manifest) + await flushMigrationState(args.store, args.signal) + args.onDurableCommit() + return result +} + +export async function abortStagedOrcadMigrationCatalogDurably(args: { + store: OrcadMigrationCatalogAbortStore + manifest: OrcadMigrationManifest + signal?: AbortSignal +}): Promise { + const result = args.store.abortStagedOrcadMigrationCatalog(args.manifest) + // An already-absent retry may follow an in-memory abort whose flush failed. + if (result.state === 'absent') { + await flushMigrationState(args.store, args.signal) + return { ...result, durableAbsent: true } + } + return result +} + +export function getOrcadMigrationCatalogState(args: { + store: Pick + manifest: OrcadMigrationManifest +}): OrcadMigrationCatalogState { + return args.store.getOrcadMigrationCatalogState(args.manifest) +} + +function flushMigrationState( + store: OrcadMigrationCatalogFlushStore, + signal?: AbortSignal +): Promise { + return store.flushPendingOrThrowAsync({ signal, drainToStableGeneration: false }) +} diff --git a/src/main/runtime/pty-exit-record-wait.test.ts b/src/main/runtime/pty-exit-record-wait.test.ts new file mode 100644 index 00000000000..45a0e792943 --- /dev/null +++ b/src/main/runtime/pty-exit-record-wait.test.ts @@ -0,0 +1,38 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' + +const PTY_ID = 'repo::/tmp/exit-record-wait@@pty' +const WORKTREE_ID = 'repo::/tmp/exit-record-wait' + +afterEach(() => { + vi.useRealTimers() +}) + +describe('waitForPtyExitRecord', () => { + it('resolves true once the exit reaches the runtime record', async () => { + const runtime = new OrcaRuntimeService() + runtime.registerPty(PTY_ID, WORKTREE_ID, null) + const wait = runtime.waitForPtyExitRecord(PTY_ID, 10_000) + runtime.onPtyExit(PTY_ID, 0) + await expect(wait).resolves.toBe(true) + expect(runtime.getPtyLivenessVerdict(PTY_ID)).toEqual({ status: 'exited' }) + }) + + it('resolves false when no exit arrives within the budget', async () => { + vi.useFakeTimers() + const runtime = new OrcaRuntimeService() + runtime.registerPty(PTY_ID, WORKTREE_ID, null) + const wait = runtime.waitForPtyExitRecord(PTY_ID, 1_000) + await vi.advanceTimersByTimeAsync(1_000) + await expect(wait).resolves.toBe(false) + runtime.onPtyExit(PTY_ID, 0) + }) + + it('does not wait for a PTY the runtime never registered or already saw exit', async () => { + const runtime = new OrcaRuntimeService() + await expect(runtime.waitForPtyExitRecord('unknown-pty', 10_000)).resolves.toBe(true) + runtime.registerPty(PTY_ID, WORKTREE_ID, null) + runtime.onPtyExit(PTY_ID, 0) + await expect(runtime.waitForPtyExitRecord(PTY_ID, 10_000)).resolves.toBe(true) + }) +}) diff --git a/src/main/runtime/rpc/methods/index.ts b/src/main/runtime/rpc/methods/index.ts index 267b201edf9..00bd68f5746 100644 --- a/src/main/runtime/rpc/methods/index.ts +++ b/src/main/runtime/rpc/methods/index.ts @@ -1,4 +1,6 @@ import { STATUS_METHODS } from './status' +import { ORCAD_TERMINAL_CENSUS_METHODS } from './orcad-terminal-census' +import { ORCAD_MIGRATION_METHODS } from './orcad-migration' import { AI_VAULT_METHODS } from './ai-vault' import { AUTOMATION_METHODS } from './automations' import { REPO_METHODS } from './repo' @@ -31,6 +33,7 @@ import { LINEAR_METHODS } from './linear' import { LINEAR_AGENT_ACCESS_METHODS } from './linear-agent-access' import { JIRA_METHODS } from './jira' import { SSH_METHODS } from './ssh' +import { MANAGED_SERVER_METHODS } from './managed-server' import { SPEECH_METHODS } from './speech' import { CLIENT_UI_METHODS } from './client-ui' import { CLIENT_EVENT_METHODS } from './client-events' @@ -56,6 +59,8 @@ import { AGENT_LAUNCH_METHODS } from './agent-launch' // auditing the security boundary or wiring new CLI commands. export const ALL_RPC_METHODS = [ ...STATUS_METHODS, + ...ORCAD_TERMINAL_CENSUS_METHODS, + ...ORCAD_MIGRATION_METHODS, ...AGENT_HOOK_METHODS, ...AI_VAULT_METHODS, ...ARTIFACT_METHODS, @@ -94,6 +99,7 @@ export const ALL_RPC_METHODS = [ ...LINEAR_AGENT_ACCESS_METHODS, ...JIRA_METHODS, ...SSH_METHODS, + ...MANAGED_SERVER_METHODS, ...SPEECH_METHODS, ...WORKSPACE_PORT_METHODS, ...PLUGIN_METHODS, diff --git a/src/main/runtime/rpc/methods/managed-server.test.ts b/src/main/runtime/rpc/methods/managed-server.test.ts new file mode 100644 index 00000000000..6134df8239f --- /dev/null +++ b/src/main/runtime/rpc/methods/managed-server.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { + registerManagedServerActions, + type ManagedServerActions +} from '../../managed-server-actions-registry' +import { RpcDispatcher } from '../dispatcher' +import type { RpcRequest } from '../core' +import { MANAGED_SERVER_METHODS } from './managed-server' + +function request(method: string, params?: unknown): RpcRequest { + return { id: 'managed-1', authToken: 'token', method, params } +} + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: these handlers read no runtime member; only the reply envelope needs getRuntimeId. +const RUNTIME = { getRuntimeId: () => 'runtime-1' } as unknown as OrcaRuntimeService + +function dispatcher(): RpcDispatcher { + return new RpcDispatcher({ runtime: RUNTIME, methods: MANAGED_SERVER_METHODS }) +} + +function actions(): ManagedServerActions { + return { + status: vi.fn(), + update: vi.fn(async () => ({ + outcome: 'deferred' as const, + candidateVersion: '2', + code: 'c', + reason: 'r' + })), + rollback: vi.fn(), + recover: vi.fn(async () => ({ outcome: 'none' as const })), + stop: vi.fn(async () => ({ + outcome: 'refused' as const, + verdict: 'live' as const, + code: 'c', + reason: 'r' + })), + cancelStop: vi.fn() + } +} + +afterEach(() => registerManagedServerActions(null)) + +describe('managed server RPC', () => { + it('refuses where no desktop runtime registered the actions', async () => { + const response = await dispatcher().dispatch( + request('managedServer.stop', { selector: 'build-box' }) + ) + expect(response).toMatchObject({ ok: false }) + }) + + it('forwards to the same actions the settings run', async () => { + const registered = actions() + registerManagedServerActions(registered) + await expect( + dispatcher().dispatch(request('managedServer.stop', { selector: 'build-box' })) + ).resolves.toMatchObject({ ok: true, result: { outcome: 'refused' } }) + expect(registered.stop).toHaveBeenCalledWith('build-box') + + await dispatcher().dispatch(request('managedServer.update', { selector: 'build-box' })) + expect(registered.update).toHaveBeenCalledWith('build-box', false) + await dispatcher().dispatch( + request('managedServer.update', { selector: 'build-box', force: true }) + ) + expect(registered.update).toHaveBeenLastCalledWith('build-box', true) + }) + + it('rejects a missing selector before reaching an action', async () => { + const registered = actions() + registerManagedServerActions(registered) + const response = await dispatcher().dispatch(request('managedServer.recover', {})) + expect(response).toMatchObject({ ok: false }) + expect(registered.recover).not.toHaveBeenCalled() + }) + + it('passes the confirmed changed-state restore through to recover', async () => { + const registered = actions() + registerManagedServerActions(registered) + await dispatcher().dispatch( + request('managedServer.recover', { selector: 'build-box', acceptChangedState: true }) + ) + expect(registered.recover).toHaveBeenLastCalledWith('build-box', true) + await dispatcher().dispatch(request('managedServer.recover', { selector: 'build-box' })) + expect(registered.recover).toHaveBeenLastCalledWith('build-box', false) + }) +}) diff --git a/src/main/runtime/rpc/methods/managed-server.ts b/src/main/runtime/rpc/methods/managed-server.ts new file mode 100644 index 00000000000..95902891bae --- /dev/null +++ b/src/main/runtime/rpc/methods/managed-server.ts @@ -0,0 +1,54 @@ +import { + ManagedServerRecover, + ManagedServerSelector, + ManagedServerUpdate +} from '../../../../shared/rpc-contract/managed-server-params' +import { + getManagedServerActions, + type ManagedServerActions +} from '../../managed-server-actions-registry' +import { defineMethod } from '../core' + +// Why a refusal and not a crash: only the desktop main process registers these actions, and a +// client must read their absence the same way it reads an older host's method_not_found. +function actions(): ManagedServerActions { + const registered = getManagedServerActions() + if (!registered) { + throw new Error('managed_server_unavailable') + } + return registered +} + +export const MANAGED_SERVER_METHODS = [ + defineMethod({ + name: 'managedServer.status', + params: ManagedServerSelector, + handler: ({ selector }) => actions().status(selector) + }), + defineMethod({ + name: 'managedServer.update', + params: ManagedServerUpdate, + handler: ({ selector, force }) => actions().update(selector, force === true) + }), + defineMethod({ + name: 'managedServer.rollback', + params: ManagedServerSelector, + handler: ({ selector }) => actions().rollback(selector) + }), + defineMethod({ + name: 'managedServer.recover', + params: ManagedServerRecover, + handler: ({ selector, acceptChangedState }) => + actions().recover(selector, acceptChangedState === true) + }), + defineMethod({ + name: 'managedServer.stop', + params: ManagedServerSelector, + handler: ({ selector }) => actions().stop(selector) + }), + defineMethod({ + name: 'managedServer.cancelStop', + params: ManagedServerSelector, + handler: ({ selector }) => actions().cancelStop(selector) + }) +] as const diff --git a/src/main/runtime/rpc/methods/orcad-migration.test.ts b/src/main/runtime/rpc/methods/orcad-migration.test.ts new file mode 100644 index 00000000000..3c40126336c --- /dev/null +++ b/src/main/runtime/rpc/methods/orcad-migration.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it, vi } from 'vitest' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../../../shared/orcad-migration-manifest' +import { computeOrcadMigrationManifestSha256 } from '../../../orcad/orcad-migration-manifest-digest' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { eraseRpcMethods, isStreamingMethod, type RpcContext } from '../core' +import { ALL_RPC_METHODS } from './index' +import { ORCAD_MIGRATION_METHODS } from './orcad-migration' + +function manifest(): OrcadMigrationManifest { + const unsigned = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-08-30T12:00:00.000Z', + source: { + sshTargetId: 'ssh-prod', + sshTargetGeneration: 7, + targetLabel: 'Production' + }, + payload: { repositories: [], projectGroups: [], folderWorkspaces: [] } + } + return { + ...unsigned, + manifestSha256: computeOrcadMigrationManifestSha256(unsigned) + } +} + +const MIGRATION_METHOD_NAMES = [ + 'orcad.migration.stageCatalog', + 'orcad.migration.commitCatalog', + 'orcad.migration.stageSnapshotChunk', + 'orcad.migration.abortCatalog', + 'orcad.migration.catalogState' +] as const + +function migrationMethod( + name: (typeof MIGRATION_METHOD_NAMES)[number] = MIGRATION_METHOD_NAMES[0] +) { + const method = eraseRpcMethods(ORCAD_MIGRATION_METHODS).find( + (candidate) => candidate.name === name + ) + if (!method || isStreamingMethod(method)) { + throw new Error(`Missing or invalid ${name} method`) + } + return method +} + +function context( + stageOrcadMigrationCatalog: ReturnType, + overrides: Partial = {}, + runtimeOverrides: Record = {} +): RpcContext { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the migration methods call only the runtime members stubbed here. + runtime: { stageOrcadMigrationCatalog, ...runtimeOverrides } as unknown as OrcaRuntimeService, + clientKind: 'runtime', + pairedDeviceId: 'paired-desktop', + ...overrides + } +} + +describe('orcad migration RPC', () => { + it('registers every staged-cutover method exactly once', () => { + for (const name of MIGRATION_METHOD_NAMES) { + expect(ALL_RPC_METHODS.filter((method) => method.name === name)).toHaveLength(1) + } + }) + + it('stages, commits, aborts, and reconciles through the authenticated runtime client', async () => { + const input = manifest() + const signal = new AbortController().signal + const stage = vi.fn().mockResolvedValue({ state: 'staged' }) + const commit = vi.fn().mockResolvedValue({ state: 'committed' }) + const snapshot = vi.fn().mockReturnValue({ acknowledgedOffset: 1 }) + const abort = vi.fn().mockResolvedValue({ state: 'absent', aborted: true }) + const state = vi.fn().mockReturnValue({ state: 'staged' }) + const runtime = { + stageOrcadMigrationCatalog: stage, + commitStagedOrcadMigrationCatalog: commit, + stageOrcadMigrationSnapshotChunk: snapshot, + abortStagedOrcadMigrationCatalog: abort, + getOrcadMigrationCatalogState: state + } + const caller = context(vi.fn(), { signal }, runtime) + + await expect( + migrationMethod('orcad.migration.stageCatalog').handler({ manifest: input }, caller) + ).resolves.toEqual({ state: 'staged' }) + await expect( + migrationMethod('orcad.migration.commitCatalog').handler({ manifest: input }, caller) + ).resolves.toEqual({ state: 'committed' }) + const snapshotRequest = migrationSnapshotRequest(input) + await expect( + migrationMethod('orcad.migration.stageSnapshotChunk').handler(snapshotRequest, caller) + ).resolves.toEqual({ acknowledgedOffset: 1 }) + await expect( + migrationMethod('orcad.migration.abortCatalog').handler({ manifest: input }, caller) + ).resolves.toEqual({ state: 'absent', aborted: true }) + await expect( + migrationMethod('orcad.migration.catalogState').handler({ manifest: input }, caller) + ).resolves.toEqual({ state: 'staged' }) + expect(stage).toHaveBeenCalledWith(input, { signal }) + expect(commit).toHaveBeenCalledWith(input, { signal }) + expect(snapshot).toHaveBeenCalledWith(snapshotRequest) + expect(abort).toHaveBeenCalledWith(input, { signal }) + expect(state).toHaveBeenCalledWith(input) + }) + + it.each([ + ['mobile client', { clientKind: 'mobile', pairedDeviceId: 'paired-phone' }], + ['unpaired runtime client', { clientKind: 'runtime', pairedDeviceId: undefined }], + ['local caller', { clientKind: undefined, pairedDeviceId: undefined }] + ] as const)('rejects a %s before any migration mutation', async (_label, caller) => { + for (const name of MIGRATION_METHOD_NAMES) { + const invoke = vi.fn() + const method = migrationMethod(name) + const runtimeMethod = + name === 'orcad.migration.stageCatalog' + ? 'stageOrcadMigrationCatalog' + : name === 'orcad.migration.commitCatalog' + ? 'commitStagedOrcadMigrationCatalog' + : name === 'orcad.migration.stageSnapshotChunk' + ? 'stageOrcadMigrationSnapshotChunk' + : name === 'orcad.migration.abortCatalog' + ? 'abortStagedOrcadMigrationCatalog' + : 'getOrcadMigrationCatalogState' + + await expect( + method.handler( + method.params?.parse(migrationMethodInput(name)), + context(invoke, caller, { [runtimeMethod]: invoke }) + ) + ).rejects.toThrow('orcad_migration_runtime_client_required') + expect(invoke).not.toHaveBeenCalled() + } + }) + + it('rejects malformed and tampered manifests before staging', async () => { + const stageCatalog = vi.fn() + const method = migrationMethod() + const input = manifest() + + await expect( + method.handler( + method.params?.parse({ manifest: { ...input, manifestSha256: 'invalid' } }), + context(stageCatalog) + ) + ).rejects.toThrow('orcad_migration_manifest_digest_invalid') + await expect( + method.handler( + method.params?.parse({ + manifest: { ...input, source: { ...input.source, targetLabel: 'Tampered' } } + }), + context(stageCatalog) + ) + ).rejects.toThrow('orcad_migration_manifest_digest_mismatch') + expect(stageCatalog).not.toHaveBeenCalled() + }) + + it('verifies a newer client signature over fields this host does not know', async () => { + const stageCatalog = vi.fn() + const method = migrationMethod() + const { manifestSha256: _, ...known } = manifest() + const unsigned = { + ...known, + futureField: { enabled: true }, + source: { ...known.source, futureSourceField: 'x' } + } + const signed = { + ...unsigned, + manifestSha256: computeOrcadMigrationManifestSha256(unsigned) + } + + await method.handler(method.params?.parse({ manifest: signed }), context(stageCatalog)) + expect(stageCatalog).toHaveBeenCalledWith( + expect.objectContaining({ manifestSha256: signed.manifestSha256, source: known.source }), + expect.anything() + ) + await expect( + method.handler( + method.params?.parse({ manifest: { ...signed, futureField: { enabled: false } } }), + context(stageCatalog) + ) + ).rejects.toThrow('orcad_migration_manifest_digest_mismatch') + const { futureField: __, ...stripped } = signed + await expect( + method.handler(method.params?.parse({ manifest: stripped }), context(stageCatalog)), + 'a peer that drops a signed field' + ).rejects.toThrow('orcad_migration_manifest_digest_mismatch') + expect(stageCatalog).toHaveBeenCalledTimes(1) + }) +}) + +function migrationSnapshotRequest(input: OrcadMigrationManifest) { + return { + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + ref: `v1-${'1'.repeat(32)}`, + offset: 0, + bytesBase64: 'YQ==' + } +} + +function migrationMethodInput(name: (typeof MIGRATION_METHOD_NAMES)[number]) { + const input = manifest() + return name === 'orcad.migration.stageSnapshotChunk' + ? migrationSnapshotRequest(input) + : { manifest: input } +} diff --git a/src/main/runtime/rpc/methods/orcad-migration.ts b/src/main/runtime/rpc/methods/orcad-migration.ts new file mode 100644 index 00000000000..69c24d294e8 --- /dev/null +++ b/src/main/runtime/rpc/methods/orcad-migration.ts @@ -0,0 +1,68 @@ +import type { z } from 'zod' +import { OrcadMigrationCatalogParams } from '../../../../shared/rpc-contract/orcad-migration-params' +import { parseOrcadMigrationManifest } from '../../../../shared/orcad-migration-manifest' +import { OrcadMigrationSnapshotChunkRequestSchema } from '../../../../shared/orcad-migration-scrollback' +import { assertOrcadMigrationManifestDigest } from '../../../orcad/orcad-migration-manifest-digest' +import { defineMethod, type RpcContext } from '../core' + +export const ORCAD_MIGRATION_METHODS = [ + defineMethod({ + name: 'orcad.migration.stageCatalog', + params: OrcadMigrationCatalogParams, + handler: async (params, context) => { + requireMigrationRuntimeClient(context) + return context.runtime.stageOrcadMigrationCatalog(migrationManifest(params), { + signal: context.signal + }) + } + }), + defineMethod({ + name: 'orcad.migration.commitCatalog', + params: OrcadMigrationCatalogParams, + handler: async (params, context) => { + requireMigrationRuntimeClient(context) + return context.runtime.commitStagedOrcadMigrationCatalog(migrationManifest(params), { + signal: context.signal + }) + } + }), + defineMethod({ + name: 'orcad.migration.stageSnapshotChunk', + params: OrcadMigrationSnapshotChunkRequestSchema, + handler: async (params, context) => { + requireMigrationRuntimeClient(context) + return context.runtime.stageOrcadMigrationSnapshotChunk(params) + } + }), + defineMethod({ + name: 'orcad.migration.abortCatalog', + params: OrcadMigrationCatalogParams, + handler: async (params, context) => { + requireMigrationRuntimeClient(context) + return context.runtime.abortStagedOrcadMigrationCatalog(migrationManifest(params), { + signal: context.signal + }) + } + }), + defineMethod({ + name: 'orcad.migration.catalogState', + params: OrcadMigrationCatalogParams, + handler: async (params, context) => { + requireMigrationRuntimeClient(context) + return context.runtime.getOrcadMigrationCatalogState(migrationManifest(params)) + } + }) +] + +function migrationManifest(params: z.infer) { + const manifest = parseOrcadMigrationManifest(params.manifest) + // Raw, not parsed: parsing drops fields this host predates, which a newer client signed. + assertOrcadMigrationManifestDigest(params.manifest) + return manifest +} + +function requireMigrationRuntimeClient(context: RpcContext): void { + if (context.clientKind !== 'runtime' || !context.pairedDeviceId) { + throw new Error('orcad_migration_runtime_client_required') + } +} diff --git a/src/main/runtime/rpc/methods/orcad-terminal-census.test.ts b/src/main/runtime/rpc/methods/orcad-terminal-census.test.ts new file mode 100644 index 00000000000..5759f9f8de1 --- /dev/null +++ b/src/main/runtime/rpc/methods/orcad-terminal-census.test.ts @@ -0,0 +1,48 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const collect = vi.hoisted(() => vi.fn()) +vi.mock('../../../orcad/orcad-terminal-census', () => ({ collectOrcadTerminalCensus: collect })) + +import { ORCAD_TERMINAL_CENSUS_METHODS } from './orcad-terminal-census' + +const handler = ORCAD_TERMINAL_CENSUS_METHODS[0]!.handler +const idle = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 7 } + +function call(params: Record, released: number) { + const runtime = { releaseFinishedAutomationRunTerminals: vi.fn(async () => released) } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the handler reads only the runtime member faked above. + const result = handler({ activatedAt: 1, ...params } as never, { runtime } as never) + return { runtime, result } +} + +afterEach(() => { + vi.useRealTimers() + collect.mockReset() +}) + +describe('orcad.terminalCensus', () => { + it('lets an update through on a host whose only terminals are finished automation shells', async () => { + vi.useFakeTimers() + // Three idle run shells, closed by the release; the daemon drops them a moment later. + collect + .mockResolvedValueOnce({ ...idle, liveSessions: 3 }) + .mockResolvedValueOnce({ ...idle, liveSessions: 3 }) + .mockResolvedValue(idle) + const { runtime, result } = call({ releaseFinishedAutomationTerminals: true }, 3) + await vi.advanceTimersByTimeAsync(1_000) + + await expect(result).resolves.toEqual(idle) + expect(runtime.releaseFinishedAutomationRunTerminals).toHaveBeenCalledOnce() + }) + + it('still counts terminals it may not close, and closes nothing for a plain census', async () => { + collect.mockResolvedValue({ ...idle, liveSessions: 2 }) + const plain = call({}, 0) + await expect(plain.result).resolves.toMatchObject({ liveSessions: 2 }) + expect(plain.runtime.releaseFinishedAutomationRunTerminals).not.toHaveBeenCalled() + + // A used or adopted shell is never released, so the update still sees it running. + const update = call({ releaseFinishedAutomationTerminals: true }, 0) + await expect(update.result).resolves.toMatchObject({ liveSessions: 2 }) + }) +}) diff --git a/src/main/runtime/rpc/methods/orcad-terminal-census.ts b/src/main/runtime/rpc/methods/orcad-terminal-census.ts new file mode 100644 index 00000000000..f48c553156d --- /dev/null +++ b/src/main/runtime/rpc/methods/orcad-terminal-census.ts @@ -0,0 +1,43 @@ +import { defineMethod } from '../core' +import { + ORCAD_TERMINAL_CENSUS_METHOD, + OrcadTerminalCensusParamsSchema, + type OrcadTerminalCensus +} from '../../../../shared/orcad-terminal-census' + +export const ORCAD_TERMINAL_CENSUS_METHODS = [ + defineMethod({ + name: ORCAD_TERMINAL_CENSUS_METHOD, + params: OrcadTerminalCensusParamsSchema, + handler: async (params, { runtime }) => { + // Why lazy: the census reaches the daemon modules, whose xterm polyfill defines a global + // `window`; importing them statically would load it into every process with the dispatcher. + const { collectOrcadTerminalCensus } = await import('../../../orcad/orcad-terminal-census') + if (!params.releaseFinishedAutomationTerminals) { + return collectOrcadTerminalCensus(params.activatedAt) + } + const before = await collectOrcadTerminalCensus(params.activatedAt) + const released = await runtime.releaseFinishedAutomationRunTerminals() + return settledCensus( + () => collectOrcadTerminalCensus(params.activatedAt), + before.liveSessions === null ? null : before.liveSessions - released + ) + } + }) +] + +/** Closed sessions leave the daemon a moment after the close; wait briefly for the count to drop. */ +async function settledCensus( + collect: () => Promise, + expectedAtMost: number | null +): Promise { + let census = await collect() + for (let attempt = 0; attempt < 15 && expectedAtMost !== null; attempt += 1) { + if (census.liveSessions === null || census.liveSessions <= expectedAtMost) { + break + } + await new Promise((resolve) => setTimeout(resolve, 200)) + census = await collect() + } + return census +} diff --git a/src/main/runtime/rpc/node-websocket-lifecycle.ts b/src/main/runtime/rpc/node-websocket-lifecycle.ts new file mode 100644 index 00000000000..e37e38375da --- /dev/null +++ b/src/main/runtime/rpc/node-websocket-lifecycle.ts @@ -0,0 +1,153 @@ +import type { Server as HttpServer } from 'node:http' +import type { Server as HttpsServer } from 'node:https' +import type { WebSocket, WebSocketServer } from 'ws' +import type { RemoteRuntimeServerHeartbeat } from './remote-runtime-server-heartbeat' + +type WebSocketMessagePayload = string | Uint8Array +export type WebSocketMessageHandler = { + bivarianceHack( + msg: WebSocketMessagePayload, + reply: (response: string) => void, + ws: WebSocket + ): void +}['bivarianceHack'] + +export type WebSocketConnectionCloseHandler = ( + clientId: string | null, + ws: WebSocket, + hasOtherConnections: boolean +) => void + +// Why: WS connections are long-lived and multiplex many RPCs by `id`; auth and dispatch are delegated to the message handler. +export function attachNodeWebSocketLifecycle(args: { + ws: WebSocket + heartbeat: RemoteRuntimeServerHeartbeat + preAuthTimeoutMs: number + preAuthTimers: WeakMap> + clientIds: Map + heartbeatConnections: Set + // Why: read lazily so a handler registered after start still reaches sockets accepted earlier. + getMessageHandler: () => WebSocketMessageHandler | null + getConnectionCloseHandler: () => WebSocketConnectionCloseHandler | null +}): void { + const { ws } = args + let finalized = false + const onPong = (): void => args.heartbeat.noteAlive(ws) + const onMessage = (data: WebSocket.RawData, isBinary: boolean): void => { + // Why: any inbound frame counts as proof of life, so an actively-talking client isn't reaped mid-request. + args.heartbeat.noteAlive(ws) + const message = + typeof data === 'string' ? data : isBinary ? toBinaryPayload(data) : data.toString() + args.getMessageHandler()?.( + message, + (response) => { + // Why: mobile clients disconnect often; guard the write so we don't throw on a dead socket. + if (ws.readyState === ws.OPEN) { + ws.send(response) + } + }, + ws + ) + } + const finalize = (): void => { + if (finalized) { + return + } + finalized = true + ws.off('pong', onPong) + ws.off('message', onMessage) + ws.off('close', finalize) + ws.off('error', onError) + clearNodeWebSocketPreAuthTimer(ws, args.preAuthTimers) + args.heartbeatConnections.delete(ws) + if (args.heartbeatConnections.size === 0) { + args.heartbeat.stop() + } + const clientId = args.clientIds.get(ws) ?? null + args.clientIds.delete(ws) + const hasOtherConnections = + clientId !== null && Array.from(args.clientIds.values()).includes(clientId) + args.getConnectionCloseHandler()?.(clientId, ws, hasOtherConnections) + } + const onError = (): void => { + // Why: close isn't guaranteed after every error path; finalize here too so pre-auth E2EE state and connection ids can't leak. + finalize() + ws.close() + } + const preAuthTimer = setTimeout(() => { + if (!args.clientIds.has(ws)) { + // Why: a silent auto-ponging client would otherwise hold a finite mobile slot forever without starting the E2EE handshake. + ws.terminate() + } + }, args.preAuthTimeoutMs) + preAuthTimer.unref?.() + args.preAuthTimers.set(ws, preAuthTimer) + ws.on('pong', onPong) + ws.on('message', onMessage) + // Why: clean up connection-scoped state (e.g. mobile-fit overrides) so a dropped phone doesn't leave orphaned phone-fit on desktop. + ws.on('close', finalize) + ws.on('error', onError) + // Why: install lifecycle ownership before periodic heartbeat ticks can observe this socket. + args.heartbeatConnections.add(ws) + args.heartbeat.noteAlive(ws) + if (args.heartbeatConnections.size === 1) { + // Unauthenticated sockets are protected by the pre-auth timeout; heartbeat probes begin only + // after E2EE binds a client id, avoiding control frames during the handshake. + args.heartbeat.start(() => args.clientIds.keys()) + } +} + +function toBinaryPayload(data: Exclude): Uint8Array { + return Array.isArray(data) ? Buffer.concat(data) : new Uint8Array(data) +} + +export function clearNodeWebSocketPreAuthTimer( + ws: WebSocket, + preAuthTimers: WeakMap> +): void { + const timer = preAuthTimers.get(ws) + if (timer) { + clearTimeout(timer) + preAuthTimers.delete(ws) + } +} + +export async function stopNodeWebSocketTransport(args: { + wss: WebSocketServer | null + httpServer: HttpServer | HttpsServer | null + heartbeat: RemoteRuntimeServerHeartbeat + heartbeatConnections: Set +}): Promise { + args.heartbeat.stop() + args.heartbeatConnections.clear() + if (args.wss) { + for (const client of args.wss.clients) { + // Why: a half-open mobile socket may never answer a close frame, which keeps httpServer.close pending. + client.terminate() + } + args.wss.close() + } + const httpServer = args.httpServer + if (httpServer) { + await new Promise((resolve, reject) => { + httpServer.close((error) => { + if (error) { + reject(error) + return + } + resolve() + }) + // Why: idle keep-alive static-web connections would otherwise hold close() open. + httpServer.closeAllConnections() + }) + } +} + +// Why: force-terminate soon after the 1013 close since a half-open phone may never ack and would hold the descriptor past the WS cap; the 'error' listener absorbs a reset while closing. +export function rejectNodeWebSocketOverCapacity(ws: WebSocket): void { + ws.on('error', () => {}) + ws.close(1013, 'Maximum connections reached') + const terminateTimer = setTimeout(() => ws.terminate(), 1_000) + terminateTimer.unref?.() + ws.once('close', () => clearTimeout(terminateTimer)) +} diff --git a/src/main/runtime/rpc/ws-transport.test.ts b/src/main/runtime/rpc/ws-transport.test.ts index 45798a89528..f354e9c467c 100644 --- a/src/main/runtime/rpc/ws-transport.test.ts +++ b/src/main/runtime/rpc/ws-transport.test.ts @@ -1,10 +1,12 @@ import { EventEmitter } from 'node:events' import { mkdtempSync } from 'node:fs' +import { connect } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it, afterEach, vi } from 'vitest' import WebSocket from 'ws' import { WebSocketTransport } from './ws-transport' +import { rejectNodeWebSocketOverCapacity } from './node-websocket-lifecycle' import { loadOrCreateTlsCertificate } from '../tls-certificate' // Why: disable TLS verification for self-signed certs in tests. @@ -433,9 +435,7 @@ describe('WebSocketTransport', () => { vi.useFakeTimers() try { - ;(transport as unknown as { rejectOverCapacity(ws: WebSocket): void }).rejectOverCapacity( - serverSocket! - ) + rejectNodeWebSocketOverCapacity(serverSocket!) vi.advanceTimersByTime(1_000) } finally { vi.useRealTimers() @@ -455,6 +455,24 @@ describe('WebSocketTransport', () => { await transport.stop() }) + it('stops while an HTTP client holds an unanswered request open', async () => { + const transport = new WebSocketTransport({ host: '127.0.0.1', port: 0 }) + transports.push(transport) + await transport.start() + const socket = connect(transport.resolvedPort, '127.0.0.1') + await new Promise((resolve) => socket.once('connect', () => resolve())) + socket.on('error', () => {}) + // Why: the server's automatic 100 Continue proves the request is parsed and in flight. + const continued = new Promise((resolve) => socket.once('data', () => resolve())) + socket.write( + 'POST /unanswered HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 1\r\nExpect: 100-continue\r\n\r\n' + ) + await continued + + await transport.stop() + socket.destroy() + }) + it('is safe to stop without starting', async () => { const { transport } = await createTransport() await transport.stop() diff --git a/src/main/runtime/rpc/ws-transport.ts b/src/main/runtime/rpc/ws-transport.ts index 3ba8a17fe60..353eae44062 100644 --- a/src/main/runtime/rpc/ws-transport.ts +++ b/src/main/runtime/rpc/ws-transport.ts @@ -4,22 +4,23 @@ import { createServer as createHttpServer, type Server as HttpServer } from 'nod import { WebSocketServer, type WebSocket } from 'ws' import type { RpcTransport } from './transport' import { createStaticWebClientHandler } from './static-web-client-handler' +import { + attachNodeWebSocketLifecycle, + clearNodeWebSocketPreAuthTimer, + rejectNodeWebSocketOverCapacity, + stopNodeWebSocketTransport, + type WebSocketConnectionCloseHandler, + type WebSocketMessageHandler +} from './node-websocket-lifecycle' import { RemoteRuntimeServerHeartbeat } from './remote-runtime-server-heartbeat' -const MAX_WS_MESSAGE_BYTES = 1024 * 1024 +const WEBSOCKET_TRANSPORT_MAX_MESSAGE_BYTES = 1024 * 1024 // Why: one desktop remote-host client can hold many concurrent streams, so keep the cap high enough that stale streams don't starve control RPCs. -const MAX_WS_CONNECTIONS = 128 +const WEBSOCKET_TRANSPORT_MAX_CONNECTIONS = 128 // Why: bound pre-upgrade descriptor use above the WS cap so raw sockets can't grow without bound. -const MAX_TCP_CONNECTIONS = MAX_WS_CONNECTIONS * 2 +const WEBSOCKET_TRANSPORT_MAX_TCP_CONNECTIONS = WEBSOCKET_TRANSPORT_MAX_CONNECTIONS * 2 + const PRE_AUTH_TIMEOUT_MS = 10_000 -type WebSocketMessagePayload = string | Uint8Array -type WebSocketMessageHandler = { - bivarianceHack( - msg: WebSocketMessagePayload, - reply: (response: string) => void, - ws: WebSocket - ): void -}['bivarianceHack'] // Why: mobile clients background-suspend sockets with no TCP FIN, leaving half-opens that otherwise only the OS keepalive (~2h) reaps; a 15s ping/pong sweep bounds that to ~60s (clients auto-pong per RFC 6455), since a reap needs consecutive unanswered probes rather than one (STA-3320). const HEARTBEAT_INTERVAL_MS = 15_000 @@ -56,9 +57,7 @@ export class WebSocketTransport implements RpcTransport { private httpServer: HttpsServer | HttpServer | null = null private wss: WebSocketServer | null = null private messageHandler: WebSocketMessageHandler | null = null - private connectionCloseHandler: - | ((clientId: string | null, ws: WebSocket, hasOtherConnections: boolean) => void) - | null = null + private connectionCloseHandler: WebSocketConnectionCloseHandler | null = null // Why: maps each socket to its authenticated clientId so close can report which device disconnected. private wsClientIds = new Map() private heartbeatConnections = new Set() @@ -83,7 +82,7 @@ export class WebSocketTransport implements RpcTransport { this.heartbeat = new RemoteRuntimeServerHeartbeat( heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS, heartbeatNow, - MAX_WS_CONNECTIONS + WEBSOCKET_TRANSPORT_MAX_CONNECTIONS ) this.preAuthTimeoutMs = preAuthTimeoutMs ?? PRE_AUTH_TIMEOUT_MS this.staticRoot = staticRoot @@ -96,15 +95,13 @@ export class WebSocketTransport implements RpcTransport { } // Why: pass the closing `ws` and whether other sockets share its deviceToken, so client-scoped teardown fires only on the last disconnect. - onConnectionClose( - handler: (clientId: string | null, ws: WebSocket, hasOtherConnections: boolean) => void - ): void { + onConnectionClose(handler: WebSocketConnectionCloseHandler): void { this.connectionCloseHandler = handler } setClientId(ws: WebSocket, clientId: string): void { this.wsClientIds.set(ws, clientId) - this.clearPreAuthTimer(ws) + clearNodeWebSocketPreAuthTimer(ws, this.preAuthTimers) } terminateClientConnections(clientId: string): number { @@ -137,7 +134,6 @@ export class WebSocketTransport implements RpcTransport { if (this.wss) { return } - // Why: bind a persisted fallback first so devices paired to it aren't stranded (STA-1511); serve --port flips to pinned-first (issue #8535); on failure each candidate falls through to OS-assigned port 0. const persistedFallbackPort = this.fallbackPort !== undefined && this.fallbackPort !== 0 && this.fallbackPort !== this.port @@ -192,16 +188,16 @@ export class WebSocketTransport implements RpcTransport { }) // Why: the WS cap applies only post-upgrade; a separate TCP cap bounds raw/pre-upgrade descriptor use. - httpServer.maxConnections = MAX_TCP_CONNECTIONS + httpServer.maxConnections = WEBSOCKET_TRANSPORT_MAX_TCP_CONNECTIONS const wss = new WebSocketServer({ server: httpServer, - maxPayload: MAX_WS_MESSAGE_BYTES + maxPayload: WEBSOCKET_TRANSPORT_MAX_MESSAGE_BYTES }) wss.on('connection', (ws) => { - if (wss.clients.size > MAX_WS_CONNECTIONS) { - this.rejectOverCapacity(ws) + if (wss.clients.size > WEBSOCKET_TRANSPORT_MAX_CONNECTIONS) { + rejectNodeWebSocketOverCapacity(ws) return } this.handleConnection(ws) @@ -211,130 +207,30 @@ export class WebSocketTransport implements RpcTransport { this.wss = wss } - // Why: force-terminate soon after the 1013 close since a half-open phone may never ack and would hold the descriptor past the WS cap; the 'error' listener absorbs a reset while closing. - private rejectOverCapacity(ws: WebSocket): void { - ws.on('error', () => {}) - ws.close(1013, 'Maximum connections reached') - const terminateTimer = setTimeout(() => ws.terminate(), 1_000) - terminateTimer.unref?.() - ws.once('close', () => clearTimeout(terminateTimer)) - } - async stop(): Promise { const wss = this.wss const httpServer = this.httpServer this.wss = null this.httpServer = null - this.heartbeat.stop() - this.heartbeatConnections.clear() - - if (wss) { - for (const client of wss.clients) { - // Why: a half-open mobile socket may never answer a close frame, which keeps httpServer.close pending. - client.terminate() - } - wss.close() - } - - if (httpServer) { - await new Promise((resolve, reject) => { - httpServer.close((error) => { - if (error) { - reject(error) - return - } - resolve() - }) - }) - } + await stopNodeWebSocketTransport({ + wss, + httpServer, + heartbeat: this.heartbeat, + heartbeatConnections: this.heartbeatConnections + }) } - // Why: WS connections are long-lived and multiplex many RPCs by `id`; auth and dispatch are delegated to the message handler. private handleConnection(ws: WebSocket): void { - let finalized = false - const onPong = (): void => { - this.heartbeat.noteAlive(ws) - } - const onMessage = (data: WebSocket.RawData, isBinary: boolean): void => { - // Why: any inbound frame counts as proof of life, so an actively-talking client isn't reaped mid-request. - this.heartbeat.noteAlive(ws) - const msg = - typeof data === 'string' - ? data - : isBinary - ? new Uint8Array(data as Buffer) - : data.toString() - this.messageHandler?.( - msg, - (response) => { - // Why: mobile clients disconnect often; guard the write so we don't throw on a dead socket. - if (ws.readyState === ws.OPEN) { - ws.send(response) - } - }, - ws - ) - } - const onError = (): void => { - // Why: close isn't guaranteed after every error path; finalize here too so pre-auth E2EE state and connection ids can't leak. - finalizeConnection() - ws.close() - } - const finalizeConnection = (): void => { - if (finalized) { - return - } - finalized = true - ws.off('pong', onPong) - ws.off('message', onMessage) - ws.off('close', finalizeConnection) - ws.off('error', onError) - this.clearPreAuthTimer(ws) - this.heartbeatConnections.delete(ws) - if (this.heartbeatConnections.size === 0) { - this.heartbeat.stop() - } - const clientId = this.wsClientIds.get(ws) ?? null - this.wsClientIds.delete(ws) - const hasOtherConnections = - clientId !== null && Array.from(this.wsClientIds.values()).includes(clientId) - this.connectionCloseHandler?.(clientId, ws, hasOtherConnections) - } - - const preAuthTimer = setTimeout(() => { - if (!this.wsClientIds.has(ws)) { - // Why: a silent auto-ponging client would otherwise hold a finite mobile slot forever without starting the E2EE handshake. - ws.terminate() - } - }, this.preAuthTimeoutMs) - if (typeof preAuthTimer.unref === 'function') { - preAuthTimer.unref() - } - this.preAuthTimers.set(ws, preAuthTimer) - - ws.on('pong', onPong) - ws.on('message', onMessage) - - // Why: clean up connection-scoped state (e.g. mobile-fit overrides) so a dropped phone doesn't leave orphaned phone-fit on desktop. - ws.on('close', finalizeConnection) - ws.on('error', onError) - - // Why: install lifecycle ownership before periodic heartbeat ticks can observe this socket. - this.heartbeatConnections.add(ws) - this.heartbeat.noteAlive(ws) - if (this.heartbeatConnections.size === 1) { - // Unauthenticated sockets are protected by the pre-auth timeout; heartbeat probes begin only - // after E2EE binds a client id, avoiding control frames during the handshake. - this.heartbeat.start(() => this.wsClientIds.keys()) - } - } - - private clearPreAuthTimer(ws: WebSocket): void { - const timer = this.preAuthTimers.get(ws) - if (timer) { - clearTimeout(timer) - this.preAuthTimers.delete(ws) - } + attachNodeWebSocketLifecycle({ + ws, + heartbeat: this.heartbeat, + preAuthTimeoutMs: this.preAuthTimeoutMs, + preAuthTimers: this.preAuthTimers, + clientIds: this.wsClientIds, + heartbeatConnections: this.heartbeatConnections, + getMessageHandler: () => this.messageHandler, + getConnectionCloseHandler: () => this.connectionCloseHandler + }) } } diff --git a/src/main/runtime/run-terminal-client-use.ts b/src/main/runtime/run-terminal-client-use.ts new file mode 100644 index 00000000000..912dbaca177 --- /dev/null +++ b/src/main/runtime/run-terminal-client-use.ts @@ -0,0 +1,56 @@ +/** + * What the headless automation sweep asks a runtime about a run's terminal before closing it: + * whether a client used it, and whether only its shell still runs. + */ +import type { RuntimePtyController } from './runtime-pty-controller-contract' +import type { TerminalRunFactsRegister } from './terminal-run-facts' +import { + confirmRootShellAloneFromProcessTable, + inspectionShowsShellAlone +} from './run-terminal-shell-alone' + +/** + * Whether a client drove or is viewing a PTY's current process, for closing finished run + * terminals: `unknown` when this process cannot tell (it adopted the PTY rather than spawned it). + */ +export function readRunTerminalClientUse( + host: { + hasRawTerminalViewSubscriber: (ptyId: string) => boolean + terminalRunFacts: Pick + }, + ptyId: string +): 'used' | 'unused' | 'unknown' { + if (host.hasRawTerminalViewSubscriber(ptyId)) { + return 'used' + } + const facts = host.terminalRunFacts.read(ptyId, undefined) + if (facts.firstUserInputAt !== null) { + return 'used' + } + return facts.freshSpawn ? 'unused' : 'unknown' +} + +/** + * Fresh execution-host proof that only the spawned shell runs in a PTY, at its prompt, on POSIX + * and Windows alike; false whenever that cannot be proven. + */ +export async function confirmRunTerminalShellAlone( + controller: RuntimePtyController | null | undefined, + ptyId: string +): Promise { + try { + if (await controller?.confirmShellForeground?.(ptyId)) { + return true + } + if (process.platform === 'win32') { + return inspectionShowsShellAlone( + (await controller?.inspectProcess?.(ptyId, { scanChildProcesses: true })) ?? null + ) + } + const processes = (await controller?.listProcesses?.(null)) ?? [] + const rootPid = processes.find((entry) => entry.id === ptyId)?.rootProcessId + return rootPid ? await confirmRootShellAloneFromProcessTable(rootPid) : false + } catch { + return false + } +} diff --git a/src/main/runtime/run-terminal-shell-alone.node-pty.test.ts b/src/main/runtime/run-terminal-shell-alone.node-pty.test.ts new file mode 100644 index 00000000000..75a4cbcb500 --- /dev/null +++ b/src/main/runtime/run-terminal-shell-alone.node-pty.test.ts @@ -0,0 +1,80 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { createPtySubprocess } from '../daemon/pty-subprocess' +import { Session } from '../daemon/session' +import { confirmRootShellAloneFromProcessTable } from './run-terminal-shell-alone' + +// A real daemon session on a real PTY: the process table, not a fake, decides. +const SHELL = process.platform === 'win32' ? null : (['/bin/bash'].find(existsSync) ?? null) +let session: Session | undefined +let root: string | undefined + +async function shellThatRan(command: string, settledMarker: string): Promise { + root = mkdtempSync(join(tmpdir(), 'orca-run-shell-')) + writeFileSync(join(root, '.bash_profile'), "PS1='prompt> '\n") + vi.stubEnv('HOME', root) + const subprocess = await createPtySubprocess({ + sessionId: 'run-shell', + cols: 120, + rows: 30, + cwd: root, + shellOverride: SHELL!, + env: { HOME: root, SHELL: SHELL!, TERM: 'xterm-256color' } + }) + session = new Session({ + sessionId: 'run-shell', + cols: 120, + rows: 30, + subprocess, + shellReadySupported: false + }) + let output = '' + session.attachClient({ onExit: () => {}, onData: (data) => (output += data) }) + await vi.waitFor(() => expect(output).toContain('prompt> '), { timeout: 5000 }) + session.write(`${command}\n`) + await vi.waitFor(() => expect(output).toContain(settledMarker), { timeout: 5000 }) + return session +} + +afterEach(async () => { + if (session) { + await session.forceKillAndWaitForExit(3000) + session.dispose() + session = undefined + } + vi.unstubAllEnvs() + if (root) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe.skipIf(!SHELL)('run terminal shell-alone proof on a real daemon session', () => { + it('proves the shell alone after an agent command that is not installed', async () => { + const shell = await shellThatRan('goose-not-installed run', 'command not found') + await vi.waitFor( + async () => expect(await confirmRootShellAloneFromProcessTable(shell.pid)).toBe(true), + { + timeout: 5000 + } + ) + // The daemon's ownership flag never turns 'shell' for a plain failed command. + expect(await shell.confirmShellForeground()).toBe(false) + }) + + it('proves the shell alone after an agent that ran and exited', async () => { + const shell = await shellThatRan("sh -c 'printf AGENT_%s DONE'", 'AGENT_DONE') + await vi.waitFor( + async () => expect(await confirmRootShellAloneFromProcessTable(shell.pid)).toBe(true), + { + timeout: 5000 + } + ) + }) + + it('never proves it while an agent still runs in the shell', async () => { + const shell = await shellThatRan("sh -c 'printf AGENT_%s UP; sleep 30'", 'AGENT_UP') + expect(await confirmRootShellAloneFromProcessTable(shell.pid)).toBe(false) + }) +}) diff --git a/src/main/runtime/run-terminal-shell-alone.test.ts b/src/main/runtime/run-terminal-shell-alone.test.ts new file mode 100644 index 00000000000..ddc62cf6b7f --- /dev/null +++ b/src/main/runtime/run-terminal-shell-alone.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { inspectionShowsShellAlone } from './run-terminal-shell-alone' + +describe('inspectionShowsShellAlone (the Windows proof)', () => { + it('proves a shell alone only on an observed empty child census', () => { + expect( + inspectionShowsShellAlone({ + foregroundProcess: 'pwsh.exe', + hasChildProcesses: false, + childProcessEvidence: 'no-children' + }) + ).toBe(true) + }) + + it.each([ + [ + 'a child still runs', + { + foregroundProcess: 'node.exe', + hasChildProcesses: true, + childProcessEvidence: 'children' as const + } + ], + [ + 'the host could not read its process table', + { + foregroundProcess: 'pwsh.exe', + hasChildProcesses: false, + childProcessEvidence: 'unverifiable' as const + } + ], + [ + 'an older host sent no child census', + { foregroundProcess: 'pwsh.exe', hasChildProcesses: false } + ], + [ + 'the host could not be reached', + { + foregroundProcess: null, + hasChildProcesses: false as const, + verdict: 'unverifiable' as const, + reason: 'timeout' + } + ] + ])('stays unproven when %s', (_case, inspection) => { + expect(inspectionShowsShellAlone(inspection)).toBe(false) + expect(inspectionShowsShellAlone(null)).toBe(false) + }) +}) diff --git a/src/main/runtime/run-terminal-shell-alone.ts b/src/main/runtime/run-terminal-shell-alone.ts new file mode 100644 index 00000000000..3015f56315b --- /dev/null +++ b/src/main/runtime/run-terminal-shell-alone.ts @@ -0,0 +1,36 @@ +/** + * Proof that a run terminal's shell is alone at its prompt, read from the execution host's own + * process table. The daemon's shell-ownership flag cannot carry it: that flag turns 'shell' only + * after a full-screen command exits, so a plain "command not found" or an agent that already + * exited never sets it. Every failure to observe reads as unproven. + */ +import { confirmShellForegroundProcess } from '../providers/agent-foreground-process' +import { getFreshProcessTableSnapshot } from '../../shared/process-table-snapshot-reader' +import { getProcessTableIndex } from '../../shared/process-table-index' +import { isShellProcess } from '../../shared/shell-process-detection' +import type { TerminalProcessInspection } from '../../shared/terminal-process-inspection' + +/** POSIX: the PTY's root shell owns the terminal's foreground group and nothing under it is stopped. */ +export async function confirmRootShellAloneFromProcessTable(rootPid: number): Promise { + try { + const root = getProcessTableIndex(await getFreshProcessTableSnapshot()).byPid.get(rootPid) + const executable = root?.command.trim().split(/\s+/, 1)[0] ?? '' + if (!isShellProcess(executable)) { + return false + } + return await confirmShellForegroundProcess(rootPid, executable) + } catch { + return false + } +} + +/** Windows: the host's job-based child census found nothing under the shell. */ +export function inspectionShowsShellAlone(inspection: TerminalProcessInspection | null): boolean { + if (!inspection || inspection.verdict === 'unverifiable') { + return false + } + return ( + inspection.childProcessEvidence === 'no-children' && + (inspection.foregroundProcess === null || isShellProcess(inspection.foregroundProcess)) + ) +} diff --git a/src/main/runtime/runtime-automation-controller.ts b/src/main/runtime/runtime-automation-controller.ts index 20d3725c34e..c90dcea7c8c 100644 --- a/src/main/runtime/runtime-automation-controller.ts +++ b/src/main/runtime/runtime-automation-controller.ts @@ -55,6 +55,11 @@ export class RuntimeAutomationController { this.service = service } + /** Completed automation run terminals no client used, closed before an update; 0 off-headless. */ + releaseFinishedRunTerminals(): Promise { + return this.service?.releaseFinishedRunTerminals?.() ?? Promise.resolve(0) + } + /** Keep runtime-owned automation work ahead of queued external probes. */ withExternalProbePriority(run: () => T): T { const wrap = this.service?.externalProbePriority diff --git a/src/main/runtime/runtime-environment-identity-verification.ts b/src/main/runtime/runtime-environment-identity-verification.ts new file mode 100644 index 00000000000..61a1d650210 --- /dev/null +++ b/src/main/runtime/runtime-environment-identity-verification.ts @@ -0,0 +1,67 @@ +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { verifyRemotePairingRuntimeStatus } from '../../shared/remote-pairing-verification' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/electron-remote-runtime-client-capabilities' +import type { RuntimeStatus } from '../../shared/runtime-types' +import type { PairingOffer } from '../../shared/pairing' + +export const RUNTIME_IDENTITY_MISMATCH_MESSAGE = + 'The endpoint does not match this paired runtime identity.' + +export function verifyRuntimeEnvironmentIdentity( + environment: KnownRuntimeEnvironment, + options: { endpoint?: string; signal?: AbortSignal } = {} +) { + return verifyRuntimePairingIdentity( + { + ...getPreferredPairingOffer(environment), + ...(options.endpoint ? { endpoint: options.endpoint } : {}) + }, + environment, + options.signal + ) +} + +/** Proves the runtime at `pairing` holds its keys and is the expected runtime. */ +export async function verifyRuntimePairingIdentity( + verifiedPairing: PairingOffer, + expected: Pick, + signal?: AbortSignal +) { + signal?.throwIfAborted() + const response = await sendRemoteRuntimeRequest( + verifiedPairing, + 'status.get', + undefined, + 15_000, + undefined, + signal, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + signal?.throwIfAborted() + if (!response.ok) { + throw new Error(`Runtime identity verification failed: ${response.error.message}`) + } + const status = verifyRemotePairingRuntimeStatus(response.result) + if (!status.ok) { + throw new Error(status.message) + } + const runtimeId = status.runtimeStatus.runtimeId + if ( + response._meta.runtimeId !== runtimeId || + (expected.runtimeId !== null && runtimeId !== expected.runtimeId) || + (expected.pairedDeviceId !== undefined && + status.runtimeStatus.pairedDeviceId !== undefined && + status.runtimeStatus.pairedDeviceId !== expected.pairedDeviceId) + ) { + throw new Error(RUNTIME_IDENTITY_MISMATCH_MESSAGE) + } + return { + verifiedPairing, + verifiedRuntimeId: runtimeId, + runtimeStatus: status.runtimeStatus + } +} diff --git a/src/main/runtime/runtime-legacy-worker-recovery-lifetime.test.ts b/src/main/runtime/runtime-legacy-worker-recovery-lifetime.test.ts new file mode 100644 index 00000000000..ab17d878085 --- /dev/null +++ b/src/main/runtime/runtime-legacy-worker-recovery-lifetime.test.ts @@ -0,0 +1,145 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { RuntimeLegacyWorkerTerminalRecoveryController } from './runtime-legacy-worker-terminal-recovery-controller' +import type { LegacyWorkerRecoveryPorts } from './runtime-legacy-worker-terminal-recovery-types' +import type { + LegacyWorkerTerminalRecoveryCandidate, + LegacyWorkerTerminalRecoveryPlan +} from './orchestration/orchestration-legacy-worker-terminal-recovery' + +function candidate(ptyId: string, dispatchId: string): LegacyWorkerTerminalRecoveryCandidate { + return { + dispatchId, + dispatchStatus: 'dispatched', + contractVersion: 1, + taskId: 'task', + worktreeId: 'worktree', + terminalHandle: 'handle', + paneKey: 'tab:leaf', + tabId: 'tab', + leafId: 'leaf', + processIncarnation: `${ptyId}:incarnation`, + ptyId, + incarnationId: 'incarnation' + } +} + +// A local worker the pass must resolve, so the pass is still running when shutdown begins. +const ACTIVE_CANDIDATE: LegacyWorkerTerminalRecoveryPlan['candidates'][number] = { + dispatchId: 'dispatch-1', + dispatchStatus: 'dispatched', + contractVersion: 1, + taskId: 'task-1', + worktreeId: 'repo-1::/tmp/worktree-a', + terminalHandle: 'handle-1', + paneKey: 'tab-1:pane-1', + tabId: 'tab-1', + leafId: 'pane-1', + processIncarnation: 'pty-1:inc-1', + ptyId: 'pty-1', + incarnationId: 'inc-1' +} + +function setup() { + const ports = { + preparePlan: vi.fn((): LegacyWorkerTerminalRecoveryPlan => ({ + candidates: [], + ambiguousDispatchIds: [] + })), + resolveWorkspace: vi.fn(async () => { + throw new Error('unused') + }), + refreshInventory: vi.fn(async () => null), + runMutation: (_worktreeId: string, operation: () => Promise) => operation(), + getActivation: () => ({}), + hasExactPersistedSurface: () => false, + hasExactSurface: () => false, + adopt: vi.fn(async () => {}), + getRendererEpoch: () => 0, + reveal: vi.fn(async () => null), + onPtyExit: vi.fn(), + persist: vi.fn(async (): Promise> => new Set()), + rollback: vi.fn(), + reconcileMissing: () => false, + notifyResolution: vi.fn(), + canRecoverPersistentLocalPtys: () => true, + reconcileRequestedReleases: vi.fn(async (): Promise => undefined), + hasRequestedReleases: () => false, + reconcile: vi.fn(async () => ({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: [] + })), + updateRetry: vi.fn() + } satisfies LegacyWorkerRecoveryPorts + const controller = new RuntimeLegacyWorkerTerminalRecoveryController(ports) + return { controller, ports } +} +afterEach(() => vi.useRealTimers()) + +it('cancels both local and SSH retry timers and refuses later recovery work', async () => { + vi.useFakeTimers() + const { controller, ports } = setup() + const plan: LegacyWorkerTerminalRecoveryPlan = { + ambiguousDispatchIds: [], + candidates: [ + candidate('local-pty', 'local-dispatch'), + candidate('ssh:host@@remote-pty', 'remote-dispatch') + ] + } + const deferred = new Set(['local-dispatch', 'remote-dispatch']) + controller.updateRetry(plan, deferred, {}) + controller.updateRetry(plan, deferred, { connectionId: 'host' }) + expect(vi.getTimerCount()).toBe(2) + await controller.stop() + controller.updateRetry(plan, deferred, {}) + await vi.advanceTimersByTimeAsync(60_000) + expect(vi.getTimerCount()).toBe(0) + expect(ports.reconcile).not.toHaveBeenCalled() + await expect(controller.reconcile()).rejects.toThrow('recovery_stopped') +}) + +it('drains an active recovery pass but refuses a queued pass after shutdown', async () => { + const { controller, ports } = setup() + ports.preparePlan.mockReturnValueOnce({ + candidates: [ACTIVE_CANDIDATE], + ambiguousDispatchIds: [] + }) + let finish!: () => void + ports.resolveWorkspace.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + finish = () => reject(new Error('workspace gone')) + }) + ) + const first = controller.reconcile() + await vi.waitFor(() => expect(ports.resolveWorkspace).toHaveBeenCalledOnce()) + const queued = expect(controller.reconcile()).rejects.toThrow('recovery_stopped') + const settled = vi.fn() + const stopping = controller.stop().then(settled) + await Promise.resolve() + expect(settled).not.toHaveBeenCalled() + finish() + await Promise.all([first, queued, stopping]) + expect(ports.preparePlan).toHaveBeenCalledOnce() + expect(ports.resolveWorkspace).toHaveBeenCalledOnce() +}) + +it('holds shutdown until requested-release reconciliation finishes', async () => { + const { controller, ports } = setup() + let finish!: () => void + ports.reconcileRequestedReleases.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = () => resolve(undefined) + }) + ) + const recovery = controller.reconcile() + await vi.waitFor(() => expect(ports.reconcileRequestedReleases).toHaveBeenCalledOnce()) + const settled = vi.fn() + const stopping = controller.stop().then(settled) + await Promise.resolve() + expect(settled).not.toHaveBeenCalled() + finish() + await Promise.all([recovery, stopping]) + expect(settled).toHaveBeenCalledOnce() +}) diff --git a/src/main/runtime/runtime-legacy-worker-terminal-recovery-controller.ts b/src/main/runtime/runtime-legacy-worker-terminal-recovery-controller.ts index d078639a508..6fa019c40e6 100644 --- a/src/main/runtime/runtime-legacy-worker-terminal-recovery-controller.ts +++ b/src/main/runtime/runtime-legacy-worker-terminal-recovery-controller.ts @@ -31,12 +31,31 @@ export class RuntimeLegacyWorkerTerminalRecoveryController { private readonly retries = new Map() private readonly receiptEpochByPane = new Map() private readonly recoveredPtys = new Set() + private readonly backgroundWork = new Set>() + private stopped = false constructor(private readonly ports: LegacyWorkerRecoveryPorts) {} + /** Cancels retries and refuses new passes; resolves once running and released work drained. */ + async stop(): Promise { + this.stopped = true + this.cancelAllRetries() + await this.queue + await Promise.all(this.backgroundWork) + } + + /** Work a pass starts without awaiting; shutdown still waits for it. */ + trackBackgroundWork(work: Promise): void { + this.backgroundWork.add(work) + void work.finally(() => this.backgroundWork.delete(work)) + } + reconcile( options: LegacyWorkerRecoveryOptions = {} ): Promise { + if (this.stopped) { + return Promise.reject(new Error('worker_terminal_recovery_stopped')) + } if (!options.retry) { this.cancelScope(options.connectionId ? `ssh:${options.connectionId}` : 'local') } @@ -48,6 +67,9 @@ export class RuntimeLegacyWorkerTerminalRecoveryController { }) const run = this.queue.then(async () => { try { + if (this.stopped) { + throw new Error('worker_terminal_recovery_stopped') + } if (!options.retry) { this.cancelScope(options.connectionId ? `ssh:${options.connectionId}` : 'local') } @@ -82,6 +104,9 @@ export class RuntimeLegacyWorkerTerminalRecoveryController { deferredDispatchIds: ReadonlySet, options: LegacyWorkerRecoveryOptions ): void { + if (this.stopped) { + return + } const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local' const dispatchIds = plan.candidates.flatMap((candidate) => { const sshPty = parseAppSshPtyId(candidate.ptyId) @@ -136,7 +161,7 @@ export class RuntimeLegacyWorkerTerminalRecoveryController { } private armRetry(scopeKey: string, retry: RecoveryRetry): void { - if (retry.timer) { + if (this.stopped || retry.timer) { return } const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000) diff --git a/src/main/runtime/runtime-legacy-worker-terminal-recovery-runner.ts b/src/main/runtime/runtime-legacy-worker-terminal-recovery-runner.ts index 1c9b8454322..681c6b94986 100644 --- a/src/main/runtime/runtime-legacy-worker-terminal-recovery-runner.ts +++ b/src/main/runtime/runtime-legacy-worker-terminal-recovery-runner.ts @@ -173,9 +173,11 @@ export async function runLegacyWorkerTerminalRecovery( ports.updateRetry(plan, deferredDispatchIds, options) // Why: releases may only finish after the owning provider's terminals are rediscovered. if (!options.retry || pendingResolutions.length > 0 || ports.hasRequestedReleases()) { - void ports.reconcileRequestedReleases().catch((error) => { - console.warn('[orchestration] worker terminal release reconciliation failed', { error }) - }) + controller.trackBackgroundWork( + ports.reconcileRequestedReleases().catch((error) => { + console.warn('[orchestration] worker terminal release reconciliation failed', { error }) + }) + ) } return result } diff --git a/src/main/runtime/runtime-mobile-session-incarnation-projection.test.ts b/src/main/runtime/runtime-mobile-session-incarnation-projection.test.ts new file mode 100644 index 00000000000..1abf698debe --- /dev/null +++ b/src/main/runtime/runtime-mobile-session-incarnation-projection.test.ts @@ -0,0 +1,66 @@ +import { expect, it, vi } from 'vitest' +import { projectRuntimeMobileSessionTabs } from './runtime-mobile-session-projection' +import type { RuntimeMobileSessionProjectionHost } from './runtime-mobile-session-projection-contract' +import type { RuntimeMobileSessionTabsSnapshot } from '../../shared/runtime-types' +import type { RuntimePtyWorktreeRecord } from './runtime-terminal-state-records' + +it.each(['pty', 'leaf', 'unknown', 'disconnected'] as const)( + 'publishes only the handle-owning incarnation for %s', + (kind) => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: projection reads only ptyId, connected and incarnationId from the record. + const pty = { + ptyId: 'live-pty', + connected: kind !== 'disconnected', + incarnationId: kind === 'unknown' ? null : 'live-incarnation' + } as RuntimePtyWorktreeRecord + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stub implements every host member this projection path calls. + const host = { + tabs: new Map(), + leaves: new Map(kind === 'leaf' ? [['leaf', { ptyId: pty.ptyId, connected: true }]] : []), + ptysById: new Map([[pty.ptyId, pty]]), + getLiveBrowserTabs: () => new Map(), + getProviderSessionRows: () => [], + getProviderSessionSnapshot: () => [], + getLeafKey: () => 'leaf', + findPty: () => pty, + getRetainedStatus: () => null, + getTrackedTitle: () => null, + getTitleDisplayClear: () => null, + issuePtyHandle: vi.fn(() => 'handle'), + recordPty: vi.fn(() => pty), + buildPtyStatus: () => ({}), + sanitizeGroups: () => [], + pruneGroupLayout: () => null, + collectTabIds: () => new Set() + } as unknown as RuntimeMobileSessionProjectionHost + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: one terminal tab is the only snapshot field this projection path reads. + const snapshot = { + worktree: 'workspace', + publicationEpoch: 'headless:epoch', + tabs: [ + { + type: 'terminal', + id: 'tab::leaf', + parentTabId: 'tab', + leafId: 'leaf', + ptyId: 'stale-pty', + incarnationId: 'stale-incarnation', + title: 'Shell', + isActive: true + } + ] + } as RuntimeMobileSessionTabsSnapshot + + const tab = projectRuntimeMobileSessionTabs(snapshot, host).tabs[0] + + if (kind === 'unknown' || kind === 'disconnected') { + expect(tab).not.toHaveProperty('incarnationId') + } else { + expect(tab).toHaveProperty('incarnationId', 'live-incarnation') + } + expect(tab).toHaveProperty('status', kind === 'disconnected' ? 'pending-handle' : 'ready') + if (kind !== 'disconnected') { + expect(host.issuePtyHandle).toHaveBeenCalledWith(pty) + } + } +) diff --git a/src/main/runtime/runtime-mobile-session-projection.ts b/src/main/runtime/runtime-mobile-session-projection.ts index 36e44b58d3b..990039940c8 100644 --- a/src/main/runtime/runtime-mobile-session-projection.ts +++ b/src/main/runtime/runtime-mobile-session-projection.ts @@ -256,17 +256,14 @@ export function projectRuntimeMobileSessionTabs( } : null // Why: web/mobile clients hold handles across renderer graph syncs; leaf handles are epoch-bound but PTY handles stay streamable. - const terminalHandle = liveLeafPtyId - ? host.issuePtyHandle( - host.recordPty(liveLeafPtyId, snapshot.worktree, { - tabId: tab.parentTabId, - paneKey, - connected: true - }) - ) + const terminalPty = liveLeafPtyId + ? host.recordPty(liveLeafPtyId, snapshot.worktree, { + tabId: tab.parentTabId, + paneKey, + connected: true + }) : livePty - ? host.issuePtyHandle(livePty) - : null + const terminalHandle = terminalPty ? host.issuePtyHandle(terminalPty) : null const projectedAgentStatus = agentStatus ?? host.buildPtyStatus( @@ -299,6 +296,8 @@ export function projectRuntimeMobileSessionTabs( leafId: tab.leafId, title, ...(tab.ptyId ? { ptyId: tab.ptyId } : {}), + // Bind identity to the handle's live owner, never a stale persisted surface. + ...(terminalPty?.incarnationId ? { incarnationId: terminalPty.incarnationId } : {}), ...(tab.terminalTheme ? { terminalTheme: tab.terminalTheme } : {}), ...(launchAgent ? { launchAgent } : {}), ...clientAgentStatus, diff --git a/src/main/runtime/runtime-rpc-client-activity.test.ts b/src/main/runtime/runtime-rpc-client-activity.test.ts new file mode 100644 index 00000000000..12b6c3a5206 --- /dev/null +++ b/src/main/runtime/runtime-rpc-client-activity.test.ts @@ -0,0 +1,34 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { OrcaRuntimeRpcServer } from './runtime-rpc' +import { readRuntimeMetadata } from './runtime-metadata' +import { sendRequest } from './runtime-rpc-test-harness' + +describe('OrcaRuntimeRpcServer client activity', () => { + it('records each request so an idle host restarts its quiet period', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-activity-')) + const server = new OrcaRuntimeRpcServer({ runtime: new OrcaRuntimeService(), userDataPath }) + await server.start() + try { + const before = server.readClientActivity() + expect(before).toMatchObject({ openConnections: 0, requestsInFlight: 0 }) + + const metadata = readRuntimeMetadata(userDataPath) + await new Promise((resolve) => setTimeout(resolve, 5)) + await sendRequest(metadata!.transports[0]!.endpoint, { + id: 'req_status', + authToken: metadata!.authToken, + method: 'status.get' + }) + + const after = server.readClientActivity() + expect(after.requestsInFlight).toBe(0) + expect(after.lastRequestAt).toBeGreaterThan(before.lastRequestAt) + } finally { + await server.stop() + } + }) +}) diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts index 156ff03210e..efa05957f40 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-lifecycle.ts @@ -46,7 +46,7 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { // Why: the `.catch` guarantees reply() always fires so a throw can't strand the client or leak the AbortController. socketTransport.onMessage((msg, reply, context) => { - void this.handleMessage(msg, context) + void this.trackClientRequest(() => this.handleMessage(msg, context)) .then((response) => { reply(JSON.stringify(response)) }) @@ -216,17 +216,22 @@ export class RuntimeRpcLifecycle extends RuntimeRpcWebSocketDispatch { deviceRegistry, e2eeKeypair, onText: (socket, plaintext, reply, sendBinary) => { - void this.handleWebSocketMessage( - plaintext, - reply, - sendBinary, - undefined, - socket.ws, - socket.device.deviceToken, - socket + void this.trackClientRequest(() => + this.handleWebSocketMessage( + plaintext, + reply, + sendBinary, + undefined, + socket.ws, + socket.device.deviceToken, + socket + ) ) }, - onBinary: (socket, bytes) => this.handleWebSocketBinaryMessage(bytes, socket.ws), + onBinary: (socket, bytes) => { + this.lastClientRequestAt = Date.now() + this.handleWebSocketBinaryMessage(bytes, socket.ws) + }, onReady: (socket) => { // Why: first authenticated mobile/remote client (direct WS and // cloud relay both attach here) starts path-candidate tracking. diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts index f79834fbcc2..c16bcda5510 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-shutdown.ts @@ -1,11 +1,26 @@ import { RuntimeRpcMobilePairing } from './runtime-rpc-mobile-pairing' +export type RuntimeRpcClientActivity = { + openConnections: number + requestsInFlight: number + lastRequestAt: number +} + export class RuntimeRpcShutdown extends RuntimeRpcMobilePairing { /** Why: test-only seam — runs one ownership check instead of waiting out the poll interval. */ checkRuntimeMetadataOwnership(): Promise { return this.metadataOwnershipWatch?.check() ?? Promise.resolve() } + /** What a host's idle exit reads to know whether any client is still using this server. */ + readClientActivity(): RuntimeRpcClientActivity { + return { + openConnections: this.mobileSocketWiring?.connectionCount ?? 0, + requestsInFlight: this.clientRequestsInFlight, + lastRequestAt: this.lastClientRequestAt + } + } + async stop(): Promise { // Why: STA-2370 — refuse new widens, then let any in-flight pairing widen settle into the live // transport arrays before snapshotting them, so a racing rebind can't strand a wide 0.0.0.0 listener diff --git a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts index e7f56ceed13..b6492a6683b 100644 --- a/src/main/runtime/runtime-rpc/runtime-rpc-state.ts +++ b/src/main/runtime/runtime-rpc/runtime-rpc-state.ts @@ -91,6 +91,8 @@ export class RuntimeRpcState { protected activeAskLongPolls = 0 protected activeBrowserHostLongPolls = 0 protected readonly activeBrowserHostLongPollsByDevice = new Map() + protected clientRequestsInFlight = 0 + protected lastClientRequestAt = Date.now() constructor({ runtime, @@ -134,4 +136,14 @@ export class RuntimeRpcState { this.pushUnregisterOutbox = new PushUnregisterOutbox(userDataPath) this.runtime.configureNotificationDismissalStore(userDataPath) } + + /** Counts a client message for idle exit, from receipt until its dispatch settles. */ + protected trackClientRequest(work: () => Promise): Promise { + this.clientRequestsInFlight += 1 + this.lastClientRequestAt = Date.now() + return work().finally(() => { + this.clientRequestsInFlight -= 1 + this.lastClientRequestAt = Date.now() + }) + } } diff --git a/src/main/runtime/runtime-store-contract.ts b/src/main/runtime/runtime-store-contract.ts index ef4e016b392..a3787e7f302 100644 --- a/src/main/runtime/runtime-store-contract.ts +++ b/src/main/runtime/runtime-store-contract.ts @@ -51,6 +51,11 @@ export type RuntimeStore = { runDurableMutation?: Store['runDurableMutation'] flushOrThrow?: Store['flushOrThrow'] flushPendingOrThrowAsync?: Store['flushPendingOrThrowAsync'] + stageOrcadMigrationCatalog?: Store['stageOrcadMigrationCatalog'] + commitStagedOrcadMigrationCatalog?: Store['commitStagedOrcadMigrationCatalog'] + stageOrcadMigrationSnapshotChunk?: Store['stageOrcadMigrationSnapshotChunk'] + abortStagedOrcadMigrationCatalog?: Store['abortStagedOrcadMigrationCatalog'] + getOrcadMigrationCatalogState?: Store['getOrcadMigrationCatalogState'] persistPtyBinding?: Store['persistPtyBinding'] getSshRemotePtyLeases?: Store['getSshRemotePtyLeases'] getUI?: Store['getUI'] diff --git a/src/main/runtime/runtime-terminal-client-use.test.ts b/src/main/runtime/runtime-terminal-client-use.test.ts new file mode 100644 index 00000000000..6ace691eec1 --- /dev/null +++ b/src/main/runtime/runtime-terminal-client-use.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' + +describe('readTerminalClientUse', () => { + it('reads a spawned terminal nobody typed into or views as unused', () => { + const runtime = new OrcaRuntimeService() + runtime.terminalRunFacts.recordSpawnCommit({ id: 'pty-1' }) + runtime.terminalRunFacts.recordInput('pty-1', 'launch', 'claude "run checks"\r') + expect(runtime.readTerminalClientUse('pty-1')).toBe('unused') + }) + + it('reads a terminal a client typed into as used', () => { + const runtime = new OrcaRuntimeService() + runtime.terminalRunFacts.recordSpawnCommit({ id: 'pty-1' }) + runtime.terminalRunFacts.recordInput('pty-1', 'driving', 'git status\r') + expect(runtime.readTerminalClientUse('pty-1')).toBe('used') + }) + + it('reads a terminal a client is viewing as used', () => { + const runtime = new OrcaRuntimeService() + runtime.terminalRunFacts.recordSpawnCommit({ id: 'pty-1' }) + // What a client's terminal stream subscribe registers. + const release = runtime.registerRemoteTerminalViewSubscriber('pty-1') + expect(runtime.readTerminalClientUse('pty-1')).toBe('used') + release() + expect(runtime.readTerminalClientUse('pty-1')).toBe('unused') + }) + + it('cannot tell for a terminal this process adopted rather than spawned', () => { + const runtime = new OrcaRuntimeService() + expect(runtime.readTerminalClientUse('pty-adopted')).toBe('unknown') + }) +}) diff --git a/src/main/runtime/runtime-worktree-ps-activity.ts b/src/main/runtime/runtime-worktree-ps-activity.ts index d8d0b6a0109..496acb46def 100644 --- a/src/main/runtime/runtime-worktree-ps-activity.ts +++ b/src/main/runtime/runtime-worktree-ps-activity.ts @@ -36,6 +36,8 @@ export function applyRuntimeWorktreePsTerminalActivity(args: { pathIndex: RuntimeWorktreeSummaryPathIndex missingIds: Set freshPtyLiveness: ReadonlySet | null + /** Filled with every PTY this pass counts as live. */ + countedPtyIds: Set leaves: Iterable ptysById: ReadonlyMap tabs: ReadonlyMap @@ -59,7 +61,7 @@ export function applyRuntimeWorktreePsTerminalActivity(args: { } } } - const countedPtyIds = new Set() + const countedPtyIds = args.countedPtyIds for (const leaf of args.leaves) { if ( !leaf.ptyId || @@ -153,6 +155,7 @@ export function applyRuntimeWorktreePsTerminalActivity(args: { if (!summary) { continue } + countedPtyIds.add(pty.ptyId) const previousLastOutputAt = summary.lastOutputAt summary.liveTerminalCount += 1 summary.hasAttachedPty = true diff --git a/src/main/runtime/runtime-worktree-ps-summaries.ts b/src/main/runtime/runtime-worktree-ps-summaries.ts index 1d5a161a8a5..f42caf04806 100644 --- a/src/main/runtime/runtime-worktree-ps-summaries.ts +++ b/src/main/runtime/runtime-worktree-ps-summaries.ts @@ -70,6 +70,7 @@ export function buildRuntimeWorktreePsSummaries(args: { unread: meta?.isUnread ?? false, liveTerminalCount: 0, hasAttachedPty: false, + unverifiableTerminalCount: 0, lastOutputAt: null, preview: '', status: 'inactive', @@ -116,6 +117,7 @@ export function buildRuntimeWorktreePsSummaries(args: { unread: worktree.isUnread, liveTerminalCount: 0, hasAttachedPty: false, + unverifiableTerminalCount: 0, lastOutputAt: null, preview: '', status: 'inactive', diff --git a/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.test.ts b/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.test.ts new file mode 100644 index 00000000000..8696b75ceba --- /dev/null +++ b/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import type { RuntimeWorktreePsSummary } from '../../shared/runtime-types' +import { applyRuntimeWorktreePsUnverifiableTerminals } from './runtime-worktree-ps-unverifiable-terminals' + +describe('unverifiable terminal attribution', () => { + it('follows the host PTY record, not a stale pane, as the live pass does', () => { + const summaries = new Map>([ + ['wt-old', {}], + ['wt-new', {}] + ]) + applyRuntimeWorktreePsUnverifiableTerminals({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the counters this pass writes are read back. + summaries: summaries as Map, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: getSummary below ignores the index. + pathIndex: {} as never, + missingIds: new Set(), + countedPtyIds: new Set(), + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pass reads only ptyId and worktreeId. + leaves: [{ ptyId: 'pty-1', worktreeId: 'wt-old' } as never], + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the pass reads only ptyId and worktreeId. + ptysById: new Map([['pty-1', { ptyId: 'pty-1', worktreeId: 'wt-new' } as never]]), + getLivenessVerdict: () => ({ status: 'unverifiable', reason: 'lost contact' }), + getSummary: (all, _index, _missing, worktreeId) => all.get(worktreeId) ?? null + }) + expect(summaries.get('wt-new')?.unverifiableTerminalCount).toBe(1) + expect(summaries.get('wt-old')?.unverifiableTerminalCount).toBeUndefined() + }) +}) diff --git a/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.ts b/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.ts new file mode 100644 index 00000000000..23ee5dcab07 --- /dev/null +++ b/src/main/runtime/runtime-worktree-ps-unverifiable-terminals.ts @@ -0,0 +1,49 @@ +import type { RuntimeWorktreePsSummary } from '../../shared/runtime-types' +import type { PtyLivenessVerdict } from '../../shared/pty-liveness-verdict' +import type { RuntimeLeafRecord, RuntimePtyWorktreeRecord } from './runtime-terminal-state-records' +import type { RuntimeWorktreeSummaryPathIndex } from './runtime-worktree-summary-paths' + +/** + * Counts each terminal that was not counted live but whose host only lost contact. + * A relay drop disconnects every one of the host's PTY records at once, so without this + * the row read as no terminals at all (docs/reference/ssh-execution-boundary.md). + */ +export function applyRuntimeWorktreePsUnverifiableTerminals(args: { + summaries: Map + pathIndex: RuntimeWorktreeSummaryPathIndex + missingIds: Set + countedPtyIds: ReadonlySet + leaves: Iterable + ptysById: ReadonlyMap + getLivenessVerdict: (ptyId: string) => PtyLivenessVerdict | null + getSummary: ( + summaries: Map, + pathIndex: RuntimeWorktreeSummaryPathIndex, + missingIds: Set, + worktreeId: string + ) => RuntimeWorktreePsSummary | null +}): void { + // Like the live pass, the host's PTY record owns the worktree; a pane only fills in a PTY with no record. + const ownerByPtyId = new Map() + for (const pty of args.ptysById.values()) { + ownerByPtyId.set(pty.ptyId, pty.worktreeId) + } + for (const leaf of args.leaves) { + if (leaf.ptyId && !ownerByPtyId.has(leaf.ptyId)) { + ownerByPtyId.set(leaf.ptyId, leaf.worktreeId) + } + } + for (const [ptyId, worktreeId] of ownerByPtyId) { + if ( + args.countedPtyIds.has(ptyId) || + args.getLivenessVerdict(ptyId)?.status !== 'unverifiable' + ) { + continue + } + const summary = args.getSummary(args.summaries, args.pathIndex, args.missingIds, worktreeId) + if (summary) { + summary.unverifiableTerminalCount = (summary.unverifiableTerminalCount ?? 0) + 1 + summary.hasHostSidebarActivity = true + } + } +} diff --git a/src/main/runtime/worktree-ps-unverifiable-terminals.test.ts b/src/main/runtime/worktree-ps-unverifiable-terminals.test.ts new file mode 100644 index 00000000000..575dc71150a --- /dev/null +++ b/src/main/runtime/worktree-ps-unverifiable-terminals.test.ts @@ -0,0 +1,149 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { getDefaultWorkspaceSession } from '../../shared/constants' + +vi.mock('electron', () => ({ + BrowserWindow: { fromId: vi.fn(() => null) }, + webContents: { fromId: vi.fn(() => null) }, + ipcMain: { on: vi.fn(), removeListener: vi.fn(), emit: vi.fn(() => true) }, + app: { getPath: vi.fn(() => '/tmp'), isPackaged: false } +})) + +const getSshGitProviderMock = vi.hoisted(() => vi.fn()) +vi.mock('../providers/ssh-git-dispatch', () => ({ + getSshGitProvider: getSshGitProviderMock, + getSshGitProviderGeneration: vi.fn(() => 0), + SSH_GIT_PROVIDER_UNAVAILABLE_MESSAGE: 'unavailable', + requireSshGitProvider: (connectionId: string) => getSshGitProviderMock(connectionId) +})) + +import { OrcaRuntimeService } from './orca-runtime' + +// BUG-9: a network drop to a relay host made `worktree ps` print `live:0 pty:no` for a terminal +// that was still running. Lost contact is `unverifiable`, never zero. + +const CONNECTION_ID = 'conn-1' +const REPO_PATH = '/home/user/app' +const WORKTREE_ID = `repo-ssh::${REPO_PATH}` +const PTY_ID = `ssh:${CONNECTION_ID}@@relay-1` + +const REPO = { + id: 'repo-ssh', + path: REPO_PATH, + displayName: 'app', + badgeColor: '#000000', + addedAt: 0, + connectionId: CONNECTION_ID +} + +function makeStore() { + const session = getDefaultWorkspaceSession() + return { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn(), + getRepos: vi.fn(() => [REPO]), + getRepo: vi.fn((id: string) => (id === REPO.id ? REPO : undefined)), + getAllWorktreeMeta: vi.fn(() => ({})), + getWorktreeMeta: vi.fn(() => undefined), + setWorktreeMeta: vi.fn(), + removeWorktreeMeta: vi.fn(), + getAllWorktreeLineage: vi.fn(() => ({})), + getAllWorkspaceLineage: vi.fn(() => ({})), + getGitHubCache: vi.fn(() => undefined), + getSettings: vi.fn(() => ({ workspaceDir: '/tmp/workspaces' })), + getProjects: vi.fn(() => []) + } +} + +function makeRuntime(controller: { + hasPty: () => boolean | null + listProcesses: () => Promise<{ id: string; worktreeId: string }[]> +}): OrcaRuntimeService { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the runtime reads only the store methods this fixture defines. + const runtime = new OrcaRuntimeService(makeStore() as never) + runtime.setPtyController( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: worktree.ps reaches only listProcesses and hasPty on this controller. + { + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + ...controller + } as never + ) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + runtime.registerPty(PTY_ID, WORKTREE_ID, CONNECTION_ID) + return runtime +} + +async function psRow(runtime: OrcaRuntimeService) { + const result = await runtime.getWorktreePs(100) + return result.worktrees.find((row) => row.worktreeId === WORKTREE_ID) +} + +describe('worktree.ps terminal verdicts for an SSH host', () => { + beforeEach(() => { + getSshGitProviderMock.mockReset() + getSshGitProviderMock.mockReturnValue({ + listWorktrees: vi.fn(async () => [ + { path: REPO_PATH, head: 'abc', branch: 'main', isBare: false, isMainWorktree: true } + ]) + }) + }) + + it('counts a terminal the reachable host lists as live', async () => { + const runtime = makeRuntime({ + hasPty: () => true, + listProcesses: async () => [{ id: PTY_ID, worktreeId: WORKTREE_ID }] + }) + + const row = await psRow(runtime) + + expect(row).toMatchObject({ liveTerminalCount: 1, hasAttachedPty: true }) + expect(row?.unverifiableTerminalCount).toBe(0) + }) + + it('reports a terminal on an unreachable host as unverifiable, not zero', async () => { + const runtime = makeRuntime({ + hasPty: () => null, + listProcesses: async () => { + throw new Error('relay connection lost') + } + }) + // The relay drop reports every PTY of the host with an unconfirmed exit. + runtime.onPtyExit(PTY_ID, -1) + + const row = await psRow(runtime) + + expect(row).toMatchObject({ + liveTerminalCount: 0, + hasAttachedPty: false, + unverifiableTerminalCount: 1 + }) + }) + + it('reports nothing for a terminal whose host confirmed the exit', async () => { + const runtime = makeRuntime({ hasPty: () => false, listProcesses: async () => [] }) + runtime.onPtyExit(PTY_ID, -1, undefined, { hostExitConfirmed: true }) + + const row = await psRow(runtime) + + expect(row).toMatchObject({ liveTerminalCount: 0, hasAttachedPty: false }) + expect(row?.unverifiableTerminalCount).toBe(0) + }) + + it('returns to live once the reconnected host lists the terminal again', async () => { + let reachable = false + const runtime = makeRuntime({ + hasPty: () => (reachable ? true : null), + listProcesses: async () => (reachable ? [{ id: PTY_ID, worktreeId: WORKTREE_ID }] : []) + }) + runtime.onPtyExit(PTY_ID, -1) + expect((await psRow(runtime))?.unverifiableTerminalCount).toBe(1) + + reachable = true + const row = await psRow(runtime) + + expect(row).toMatchObject({ liveTerminalCount: 1, hasAttachedPty: true }) + expect(row?.unverifiableTerminalCount).toBe(0) + }) +}) diff --git a/src/main/runtime/worktree-pty-stop-verdict.ts b/src/main/runtime/worktree-pty-stop-verdict.ts index 23abfcabcc4..1c1d1c15c49 100644 --- a/src/main/runtime/worktree-pty-stop-verdict.ts +++ b/src/main/runtime/worktree-pty-stop-verdict.ts @@ -18,6 +18,10 @@ export function summarizeWorktreePtyStopVerdict( if (verdict?.status === 'live') { return { ptyStopVerdict: 'live' } } + // Why: an SSH record outlives its host-confirmed exit, so presence alone is not doubt. + if (verdict?.status === 'exited') { + continue + } if (verdict?.status === 'unverifiable') { ptyStopVerdict = 'unverifiable' ptyStopReason ??= verdict.reason diff --git a/src/main/server/serve-pairing-output.ts b/src/main/server/serve-pairing-output.ts new file mode 100644 index 00000000000..52189705b3f --- /dev/null +++ b/src/main/server/serve-pairing-output.ts @@ -0,0 +1,29 @@ +/** Serve-readiness pieces shared by the desktop `--serve` host and orcad, so their stdout matches. */ +import { statSync } from 'node:fs' +import { isAbsolute } from 'node:path' + +export async function renderServePairingQr(pairingUrl: string): Promise { + // Why dynamic: qrcode is only reachable from mobile pairing, so launch should + // not parse it for the majority who never pair a device. + const QRCode = await import('qrcode') + try { + return await QRCode.toString(pairingUrl, { type: 'terminal', small: true }) + } catch { + try { + return await QRCode.toString(pairingUrl, { type: 'utf8' }) + } catch { + return null + } + } +} + +/** The recipe line names this root, so it must be a real absolute directory. */ +export function assertServeProjectRoot(projectRoot: string): string { + if (!isAbsolute(projectRoot)) { + throw new Error(`--serve-project-root must be absolute: ${projectRoot}`) + } + if (!statSync(projectRoot).isDirectory()) { + throw new Error(`--serve-project-root must be a directory: ${projectRoot}`) + } + return projectRoot +} diff --git a/src/main/speech/speech-model-download-response.test.ts b/src/main/speech/speech-model-download-response.test.ts index 2ea9a5cd2eb..19e312cab66 100644 --- a/src/main/speech/speech-model-download-response.test.ts +++ b/src/main/speech/speech-model-download-response.test.ts @@ -1,9 +1,6 @@ import { describe, expect, it } from 'vitest' -import { - isRetryableDownloadError, - parseContentRange, - parseRetryAfterMs -} from './speech-model-download-response' +import { parseContentRange, parseRetryAfterMs } from './speech-model-download-response' +import { isRetryableDownloadError } from '../network/transient-download-error' describe('speech model download response contracts', () => { it('accepts only internally consistent byte ranges', () => { diff --git a/src/main/speech/speech-model-download-response.ts b/src/main/speech/speech-model-download-response.ts index ac1a3f09ecf..226f0053b2c 100644 --- a/src/main/speech/speech-model-download-response.ts +++ b/src/main/speech/speech-model-download-response.ts @@ -3,11 +3,6 @@ export type DownloadIncomingMessage = Electron.IncomingMessage & headers: Record destroy?: () => void } -export type HttpStatusError = Error & { - httpStatusCode?: number - retryAfterMs?: number - retryable?: boolean -} export type DownloadTotals = { totalBytes: number completedBytes: number @@ -24,27 +19,6 @@ export const MAX_NO_PROGRESS_ATTEMPTS = DOWNLOAD_RETRY_DELAYS_MS.length + 1 export const MAX_TOTAL_DOWNLOAD_REQUESTS = 4_096 // Why: cap honored Retry-After; a longer server window is surfaced for manual retry, not a multi-minute stall. export const MAX_RETRY_AFTER_MS = 120_000 -export const RETRYABLE_NET_ERROR = - /net::ERR_(CONTENT_LENGTH_MISMATCH|INCOMPLETE_CHUNKED_ENCODING|CONNECTION_(RESET|CLOSED|ABORTED|REFUSED|TIMED_OUT)|EMPTY_RESPONSE|NETWORK_CHANGED|TIMED_OUT|INTERNET_DISCONNECTED|ADDRESS_UNREACHABLE|NAME_NOT_RESOLVED|SOCKET_NOT_CONNECTED|HTTP2_PROTOCOL_ERROR|QUIC_PROTOCOL_ERROR)\b/ -export const RETRYABLE_HTTP_STATUSES = new Set([408, 416, 425, 429, 500, 502, 503, 504]) - -export function isRetryableDownloadError(error: unknown): boolean { - if (!(error instanceof Error)) { - return false - } - const downloadError = error as HttpStatusError - if (downloadError.retryable === true) { - return true - } - const statusCode = downloadError.httpStatusCode - if (statusCode !== undefined) { - return RETRYABLE_HTTP_STATUSES.has(statusCode) - } - return ( - RETRYABLE_NET_ERROR.test(error.message) || error.message.includes('without network activity') - ) -} - export function getHeaderValue(value: string | string[] | undefined): string | undefined { return Array.isArray(value) ? value[0] : value } diff --git a/src/main/speech/speech-model-download-transport.ts b/src/main/speech/speech-model-download-transport.ts index fcdc1fd150f..929158a8ba6 100644 --- a/src/main/speech/speech-model-download-transport.ts +++ b/src/main/speech/speech-model-download-transport.ts @@ -6,11 +6,10 @@ import { MAX_RETRY_AFTER_MS, MAX_TOTAL_DOWNLOAD_REQUESTS, describeInterruptedDownload, - isRetryableDownloadError, sleepUnlessAborted, - type DownloadTotals, - type HttpStatusError + type DownloadTotals } from './speech-model-download-response' +import { isRetryableDownloadError, type HttpStatusError } from '../network/transient-download-error' import { SpeechModelHttpDownload } from './speech-model-http-download' export abstract class SpeechModelDownloadTransport extends SpeechModelHttpDownload { diff --git a/src/main/speech/speech-model-http-download.ts b/src/main/speech/speech-model-http-download.ts index 3bd2f24ab97..7c059db783d 100644 --- a/src/main/speech/speech-model-http-download.ts +++ b/src/main/speech/speech-model-http-download.ts @@ -7,9 +7,9 @@ import { parseContentRange, parseRetryAfterMs, type DownloadIncomingMessage, - type DownloadTotals, - type HttpStatusError + type DownloadTotals } from './speech-model-download-response' +import type { HttpStatusError } from '../network/transient-download-error' export abstract class SpeechModelHttpDownload { protected abstract reportDownloadProgress(modelId: string, progress: number): void diff --git a/src/main/ssh/managed-server-fence-recheck.test.ts b/src/main/ssh/managed-server-fence-recheck.test.ts new file mode 100644 index 00000000000..a2920e94905 --- /dev/null +++ b/src/main/ssh/managed-server-fence-recheck.test.ts @@ -0,0 +1,84 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' +import { + FENCE_RECHECK_INTERVAL_MS, + recheckFencedManagedServer, + scheduleManagedServerFenceRecheck +} from './managed-server-fence-recheck' +import { MANAGED_ORCAD_FENCED_DETAIL } from './orcad-managed-serving' + +const TARGET: SshTarget = { id: 'box', label: 'Box', host: 'box', port: 22, username: 'me' } + +function hostDeps(states: ('fenced' | 'serving')[], update: 'fence-busy' | 'current') { + return { + ensureServing: vi.fn(async () => + states.shift() === 'fenced' + ? { state: 'unverifiable' as const, detail: MANAGED_ORCAD_FENCED_DETAIL } + : { state: 'serving' as const } + ), + autoUpdate: vi.fn(async () => + update === 'fence-busy' + ? { + outcome: 'deferred' as const, + code: 'orcad_activation_fence_busy', + reason: 'Another update holds this host.' + } + : { outcome: 'skipped' as const, reason: 'current' as const } + ), + recordedUpdateFailure: () => null, + recordUpdateFailure: vi.fn(), + clearUpdateFailure: vi.fn() + } +} + +beforeEach(() => { + vi.useFakeTimers() +}) +afterEach(() => { + vi.useRealTimers() +}) + +describe('a desktop that met another desktop’s update fence', () => { + it('rechecks until the host serves again, then publishes it without the note', async () => { + const deps = hostDeps(['fenced', 'fenced', 'serving'], 'current') + const publish = vi.fn() + scheduleManagedServerFenceRecheck(TARGET.id, { + stillCurrent: () => true, + recheck: () => recheckFencedManagedServer(TARGET, 'env-1', deps), + publish + }) + + await vi.advanceTimersByTimeAsync(FENCE_RECHECK_INTERVAL_MS * 2) + expect(publish).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(FENCE_RECHECK_INTERVAL_MS) + expect(publish).toHaveBeenCalledWith({ route: 'managed', environmentId: 'env-1' }) + await vi.advanceTimersByTimeAsync(FENCE_RECHECK_INTERVAL_MS * 3) + expect(deps.ensureServing).toHaveBeenCalledTimes(3) + }) + + it('keeps rechecking while the update is still deferred on the fence', async () => { + const deps = hostDeps(['serving', 'serving'], 'fence-busy') + const publish = vi.fn() + scheduleManagedServerFenceRecheck(TARGET.id, { + stillCurrent: () => true, + recheck: () => recheckFencedManagedServer(TARGET, 'env-1', deps), + publish + }) + await vi.advanceTimersByTimeAsync(FENCE_RECHECK_INTERVAL_MS * 2) + expect(publish).not.toHaveBeenCalled() + expect(deps.autoUpdate).toHaveBeenCalledTimes(2) + }) + + it('stops once the host disconnected or another connect took over', async () => { + const deps = hostDeps(['serving'], 'current') + const publish = vi.fn() + scheduleManagedServerFenceRecheck(TARGET.id, { + stillCurrent: () => false, + recheck: () => recheckFencedManagedServer(TARGET, 'env-1', deps), + publish + }) + await vi.advanceTimersByTimeAsync(FENCE_RECHECK_INTERVAL_MS * 3) + expect(deps.ensureServing).not.toHaveBeenCalled() + expect(publish).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/managed-server-fence-recheck.ts b/src/main/ssh/managed-server-fence-recheck.ts new file mode 100644 index 00000000000..55033b4f137 --- /dev/null +++ b/src/main/ssh/managed-server-fence-recheck.ts @@ -0,0 +1,99 @@ +/** + * A host another desktop was updating answers this one with "holds this host" or a fence-busy + * deferral. That clears within minutes, but this connect's status would keep it until the next + * reconnect; so the connect checks again on a timer until the host answers without the fence. + */ +import type { SshTarget } from '../../shared/ssh-types' +import { checkManagedServerUpdate } from './managed-server-update-check' +import { MANAGED_ORCAD_FENCED_DETAIL } from './orcad-managed-serving' +import type { + HostServerOnConnectDeps, + HostServerOnConnectResult +} from './ssh-host-server-on-connect' + +type ManagedResult = Extract + +export const FENCE_RECHECK_INTERVAL_MS = 45_000 +// About half an hour: past the 20-minute stale install lock a crashed updater can leave. +const FENCE_RECHECK_MAX_ATTEMPTS = 40 + +/** The connect's serving check and update check again, without its progress statuses. */ +export async function recheckFencedManagedServer( + target: SshTarget, + environmentId: string, + deps: Pick & + Parameters[2] +): Promise { + const serving = await deps.ensureServing(environmentId) + if (serving.state === 'unverifiable') { + return { + route: 'managed', + environmentId, + serving, + ...(serving.detail === MANAGED_ORCAD_FENCED_DETAIL ? { fenceHeld: true as const } : {}) + } + } + const { note, fenceBusy } = await checkManagedServerUpdate(target, environmentId, deps, () => {}) + return { + route: 'managed', + environmentId, + ...(note ? { update: note } : {}), + ...(fenceBusy ? { fenceHeld: true as const } : {}) + } +} + +export type FenceRecheckLoop = { + /** False once the host disconnected, or another connect replaced this one's status. */ + stillCurrent: () => boolean + recheck: () => Promise + publish: (result: ManagedResult) => void + intervalMs?: number +} + +const loops = new Map void>() + +/** Replaces any loop for the target; publishes the first answer free of the fence, then stops. */ +export function scheduleManagedServerFenceRecheck(targetId: string, loop: FenceRecheckLoop): void { + loops.get(targetId)?.() + let timer: ReturnType | null = null + let stopped = false + const stop = (): void => { + stopped = true + if (timer) { + clearTimeout(timer) + } + if (loops.get(targetId) === stop) { + loops.delete(targetId) + } + } + loops.set(targetId, stop) + const tick = async (attempt: number): Promise => { + if (stopped || !loop.stillCurrent()) { + stop() + return + } + const result = await loop.recheck().catch((error: unknown) => { + console.warn('[ssh] Could not recheck the managed Orca server:', error) + return null + }) + if (stopped || !loop.stillCurrent()) { + stop() + return + } + if (result && !result.fenceHeld) { + stop() + loop.publish(result) + return + } + if (attempt + 1 >= FENCE_RECHECK_MAX_ATTEMPTS) { + stop() + return + } + timer = setTimeout(() => void tick(attempt + 1), loop.intervalMs ?? FENCE_RECHECK_INTERVAL_MS) + } + timer = setTimeout(() => void tick(0), loop.intervalMs ?? FENCE_RECHECK_INTERVAL_MS) +} + +export function cancelManagedServerFenceRecheck(targetId: string): void { + loops.get(targetId)?.() +} diff --git a/src/main/ssh/managed-server-update-check.ts b/src/main/ssh/managed-server-update-check.ts new file mode 100644 index 00000000000..56dc08e256d --- /dev/null +++ b/src/main/ssh/managed-server-update-check.ts @@ -0,0 +1,98 @@ +/** + * A managed host's update check, shared by the connect and the launch-time tunnel restore, turned + * into its status note and telemetry reason. + */ +import type { SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import type { ManagedOrcadAutoUpdateOutcome } from './orcad-managed-auto-update' +import type { HostServerUpdateReason } from './ssh-host-server-connect-events' +import { ORCAD_ACTIVATION_FENCE_BUSY_CODE } from './orcad-activation-fence-hold' + +export type ManagedServerUpdateDeps = { + /** Runs the Managed servers update when the host is behind this app; `onUpdating` fires first. */ + autoUpdate: ( + environmentId: string, + options: { failedBefore: boolean; onUpdating: () => void } + ) => Promise + /** Why an update to this app version already failed on the host, so it isn't retried. */ + recordedUpdateFailure: (target: SshTarget) => string | null + recordUpdateFailure: (target: SshTarget, reason: string) => void + clearUpdateFailure: (target: SshTarget) => void +} + +export type HostServerUpdateOnConnect = { + note: SshManagedServerUpdateNote | undefined + /** Deferred only because another desktop's update holds the host; worth checking again soon. */ + fenceBusy?: true + reason: HostServerUpdateReason + /** True when a recorded failure for this app version skipped the update without a try. */ + recorded: boolean +} + +/** The host keeps serving whatever happens here, so nothing in it can fail the caller. */ +export async function checkManagedServerUpdate( + target: SshTarget, + environmentId: string, + deps: ManagedServerUpdateDeps, + onUpdating: () => void +): Promise { + const failure = deps.recordedUpdateFailure(target) + let result: ManagedOrcadAutoUpdateOutcome + try { + result = await deps.autoUpdate(environmentId, { + failedBefore: failure !== null, + onUpdating + }) + } catch (error) { + console.warn('[ssh] Could not check the managed Orca server for an update:', error) + return { note: undefined, reason: 'update_check_failed', recorded: false } + } + // Why clear on current too: a Managed servers update may have landed the build since. + if ( + failure !== null && + (result.outcome === 'updated' || (result.outcome === 'skipped' && result.reason === 'current')) + ) { + deps.clearUpdateFailure(target) + } + switch (result.outcome) { + case 'updated': + return { note: undefined, reason: 'updated', recorded: false } + case 'deferred': + return { + note: { state: 'deferred', detail: result.reason }, + reason: 'update_deferred', + recorded: false, + ...(result.code === ORCAD_ACTIVATION_FENCE_BUSY_CODE ? { fenceBusy: true as const } : {}) + } + case 'failed': + deps.recordUpdateFailure(target, result.reason) + return { + note: { state: 'failed', detail: result.reason }, + reason: 'update_failed', + recorded: false + } + case 'skipped': + return skipped(result.reason, failure) + } +} + +function skipped( + reason: Extract['reason'], + failure: string | null +): HostServerUpdateOnConnect { + switch (reason) { + case 'host-newer': + return { note: { state: 'host-newer' }, reason: 'update_host_newer', recorded: false } + case 'rolled-back': + return { note: undefined, reason: 'update_rolled_back', recorded: false } + case 'failed-before': + return { + note: failure ? { state: 'failed', detail: failure } : undefined, + reason: 'update_failed', + recorded: true + } + case 'current': + case 'no-template': + case 'migrating': + return { note: undefined, reason: 'connected', recorded: false } + } +} diff --git a/src/main/ssh/managed-server-update-deps.ts b/src/main/ssh/managed-server-update-deps.ts new file mode 100644 index 00000000000..dd2904fb10a --- /dev/null +++ b/src/main/ssh/managed-server-update-deps.ts @@ -0,0 +1,24 @@ +/** The live update-check collaborators, shared by the connect decision and the tunnel restore. */ +import { getAppEnvironment } from '../../shared/app-environment' +import type { ManagedServerUpdateDeps } from './managed-server-update-check' +import { autoUpdateManagedOrcadEnvironment } from './orcad-managed-auto-update' +import { requireManagedOrcadTargetStore } from './orcad-managed-runtime-context' + +export function managedServerUpdateDeps(userDataPath: string): ManagedServerUpdateDeps { + const registry = requireManagedOrcadTargetStore() + const appVersion = getAppEnvironment().getVersion() + return { + autoUpdate: (environmentId, options) => + autoUpdateManagedOrcadEnvironment(userDataPath, { environmentId, appVersion, ...options }), + recordedUpdateFailure: (target) => + target.managedServerUpdateFailure?.appVersion === appVersion + ? target.managedServerUpdateFailure.reason + : null, + recordUpdateFailure: (target, reason) => { + registry.updateTarget(target.id, { managedServerUpdateFailure: { reason, appVersion } }) + }, + clearUpdateFailure: (target) => { + registry.updateTarget(target.id, { managedServerUpdateFailure: undefined }) + } + } +} diff --git a/src/main/ssh/orcad-activation-crash-recovery.test.ts b/src/main/ssh/orcad-activation-crash-recovery.test.ts new file mode 100644 index 00000000000..a3a4e933682 --- /dev/null +++ b/src/main/ssh/orcad-activation-crash-recovery.test.ts @@ -0,0 +1,288 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as RecordFile from './orcad-remote-record-file' +import type * as InstallLock from './ssh-relay-install-lock' +import type * as VersionedInstall from './ssh-relay-versioned-install' +import type * as Crypto from 'node:crypto' + +const uuid = vi.hoisted((): { fixed: string | null } => ({ fixed: null })) +vi.mock('node:crypto', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, randomUUID: () => uuid.fixed ?? actual.randomUUID() } +}) + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn() +})) +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal()), + readLocalFullVersion: () => '0.2.0+bb01' +})) +vi.mock('./orcad-remote-install', () => ({ installOrcadBundle: vi.fn() })) +vi.mock('./orcad-remote-preflight', () => ({ preflightInstalledOrcad: vi.fn() })) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' +import { deployOrcad } from './orcad-remote-deploy' +import { rollbackOrcad } from './orcad-remote-rollback' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { + BUILD_HASH, + FakeOrcadHost, + NEW, + OLD, + type CrashMode +} from './orcad-activation-host-test-harness' + +let host = new FakeOrcadHost() + +const slot = { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z') +} +const census = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 } + +const deploy = (): Promise => + deployOrcad({ ...slot, localOrcadDir: '/local/out/orcad', target: 'linux-x64-glibc', census }) +const rollback = (): Promise => + rollbackOrcad({ + ...slot, + record: FakeOrcadHost.newRecord(), + census, + targetBuildHash: BUILD_HASH, + targetDaemonProtocol: { protocolVersion: 3, previousProtocolVersions: [1, 2] } + }) + +beforeEach(() => { + vi.clearAllMocks() + uuid.fixed = null + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async (_conn, dir, _host, options) => { + if (!host.acquireFence(options)) { + const { RemoteInstallLockBusyError } = await vi.importActual( + './ssh-relay-install-lock' + ) + throw new RemoteInstallLockBusyError(dir, 0) + } + }) + vi.mocked(writeAtomicOrcadRemoteRecord).mockImplementation(async (_target, path, contents) => + host.write(path, contents) + ) +}) + +/** The invariant: the host serves exactly the slot its record names, on state that slot reads. */ +function expectExactlyTheRecordedSlot(): void { + const active = host.activeVersion() + expect([...host.alive]).toEqual(active ? [active] : []) + expect(host.isReadableBy(active)).toBe(true) + expect(host.journal).toBeNull() + expect(host.fence).toBe(false) +} + +/** Unconfirmed termination never deletes a slot directory or a snapshot. */ +function expectNoSlotOrSnapshotRemoved(): void { + const removals = host.commands.filter((command) => /\brm -r?f\b/u.test(command)) + for (const command of removals) { + // A file inside a slot (a consumed stop request) may go; the slot directory never does. + expect(command).not.toMatch(/rm -r?f '[^']*\/orcad-\d[^'/]*\/?'/u) + expect(command).not.toMatch(/rm -r?f '[^']*orcad-state-snapshots/u) + } +} + +function countMutations( + scenario: () => FakeOrcadHost, + run: () => Promise +): Promise { + host = scenario() + return run().then(() => host.mutations) +} + +const scenarios: [string, () => FakeOrcadHost, () => Promise][] = [ + ['an update over an incumbent', FakeOrcadHost.deployedOld, deploy], + ['a first activation', () => new FakeOrcadHost(), deploy], + ['a rollback', FakeOrcadHost.activatedNew, rollback] +] + +describe.each(scenarios)('%s interrupted at every mutation', (_name, scenario, run) => { + it('completes cleanly when nothing interrupts it', async () => { + host = scenario() + await run() + expectExactlyTheRecordedSlot() + expect(host.activeVersion()).toBe(run === rollback ? OLD : NEW) + }) + + it.each(['before', 'after'])( + 'recovers to exactly one recorded slot when the host answer is lost %s applying it', + async (mode) => { + const total = await countMutations(scenario, run) + expect(total).toBeGreaterThan(3) + for (let crashAt = 1; crashAt <= total; crashAt += 1) { + host = scenario() + host.crashAt = crashAt + host.crashMode = mode + await run().catch(() => undefined) + host.crashAt = null + const result = await recoverInterruptedOrcadActivation({ + ...slot, + acceptChangedState: true + }) + expect(['recovered', 'none'], `mutation ${crashAt}`).toContain(result.outcome) + expectExactlyTheRecordedSlot() + expectNoSlotOrSnapshotRemoved() + } + } + ) +}) + +describe('the rollback terminal barrier', () => { + it.each(['live', 'unverifiable'] as const)( + 'restores nothing and restarts the newer build when the stop finds %s work after the census', + async (retirement) => { + host = FakeOrcadHost.activatedNew() + host.retirement = retirement + const before = host.data + await expect(rollback()).resolves.toMatchObject({ + outcome: 'refused', + code: 'orcad_rollback_terminals_at_stop' + }) + expect(host.data).toBe(before) + expect(host.activeVersion()).toBe(NEW) + expect(host.alive.has(NEW)).toBe(true) + expect(host.journal).toBeNull() + expect(host.fence).toBe(false) + } + ) +}) + +describe('recovery refusals keep the fence', () => { + async function interruptedAfterCandidateLaunch(): Promise { + host = FakeOrcadHost.deployedOld() + const total = await countMutations(FakeOrcadHost.deployedOld, deploy) + host = FakeOrcadHost.deployedOld() + // The last three mutations are: candidate-ready journal, record, fence release. + host.crashAt = total - 2 + host.crashMode = 'before' + await deploy().catch(() => undefined) + host.crashAt = null + expect(host.alive.has(NEW)).toBe(true) + } + + it('keeps changed state, unverifiable, until an operator accepts restoring over it', async () => { + await interruptedAfterCandidateLaunch() + const result = await recoverInterruptedOrcadActivation(slot) + expect(result).toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_changed_state' + }) + expect(host.fence).toBe(true) + expect(host.journal).not.toBeNull() + expect(host.alive.size).toBe(0) + + // BUG-17: the refused takeover left the lock ownerless, so the accepting re-run need not wait. + await expect( + recoverInterruptedOrcadActivation({ ...slot, acceptChangedState: true }) + ).resolves.toMatchObject({ outcome: 'recovered', resolution: 'restored-incumbent' }) + expect(host.journal).toBeNull() + expect(host.fence).toBe(false) + expectExactlyTheRecordedSlot() + }) + + it('never treats an unverifiable incumbent as exited', async () => { + host = FakeOrcadHost.deployedOld() + host.crashAt = 3 + await deploy().catch(() => undefined) + host.crashAt = null + host.pidFiles.delete(OLD) + host.alive.delete(OLD) + const result = await recoverInterruptedOrcadActivation({ ...slot }) + expect(result).toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expect(host.fence).toBe(true) + expect(host.alive.size).toBe(0) + }) + + it('reports a fresh fence as pending instead of taking it over', async () => { + host = FakeOrcadHost.deployedOld() + host.crashAt = 3 + await deploy().catch(() => undefined) + host.crashAt = null + const journal = host.journal + expect(journal).not.toBeNull() + const { RemoteInstallLockBusyError } = await vi.importActual( + './ssh-relay-install-lock' + ) + vi.mocked(acquireInstallLock).mockRejectedValueOnce(new RemoteInstallLockBusyError('/l', 0)) + expect(await recoverInterruptedOrcadActivation({ ...slot })).toMatchObject({ + outcome: 'pending' + }) + expect(host.journal).toBe(journal) + }) + + it('keeps a journal this client cannot read', async () => { + host = FakeOrcadHost.deployedOld() + host.journal = JSON.stringify({ schemaVersion: 1, operation: 'decommission' }) + host.fence = true + expect(await recoverInterruptedOrcadActivation({ ...slot })).toMatchObject({ + outcome: 'refused', + code: 'orcad_recovery_unverifiable' + }) + expect(host.fence).toBe(true) + }) + + it('drops a fence whose release was cut short after the journal went', async () => { + host = FakeOrcadHost.deployedOld() + host.fence = true + expect(await recoverInterruptedOrcadActivation({ ...slot })).toEqual({ outcome: 'none' }) + expect(host.fence).toBe(false) + }) +}) + +describe('every launch is a managed one', () => { + it.each(scenarios)( + '%s starts orcad with idle exit and its activation fence', + async (_n, scenario, run) => { + host = scenario() + await run() + const launches = host.commands.filter((command) => command.includes('nohup')) + expect(launches.length).toBeGreaterThan(0) + for (const launch of launches) { + expect(launch).toContain( + "ORCA_ORCAD_MANAGED_ACTIVATION_ROOT='/home/u/.orca-remote/.orcad-activation-transaction'" + ) + } + } + ) +}) + +// About 1 in 125 random fence tokens contains `-cf`, which the fake host once read as a capture. +it('restores the pre-activation snapshot under a fence token that contains a tar flag', async () => { + uuid.fixed = '00000000-cf00-4000-8000-000000000000' + host = FakeOrcadHost.activatedNew() + await rollback() + expect(host.activeVersion()).toBe(OLD) + expect(host.commands.filter((command) => command.includes('nohup')).length).toBeGreaterThan(0) +}) diff --git a/src/main/ssh/orcad-activation-fence-generation.test.ts b/src/main/ssh/orcad-activation-fence-generation.test.ts new file mode 100644 index 00000000000..11ba1a7c76c --- /dev/null +++ b/src/main/ssh/orcad-activation-fence-generation.test.ts @@ -0,0 +1,194 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +// Every remote command runs in a real local shell, so the host-side checks are the real ones. +const shell = vi.hoisted((): { env: NodeJS.ProcessEnv | undefined } => ({ env: undefined })) +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => { + const { runProcess } = await import('../../shared/child-process/run-process') + return { + ...(await importOriginal()), + execCommand: async (_conn: unknown, command: string) => { + const result = await runProcess({ program: '/bin/sh', args: ['-c', command], env: shell.env }) + if (result.code !== 0) { + const { sshCommandExitError } = await import('./ssh-relay-exec-command') + throw sshCommandExitError(command, result.code ?? 1, result.stdout) + } + return result.stdout + } + } +}) + +const { withOrcadActivationLock } = await import('./orcad-activation-lock') +const { orcadActivationFenceRefusal } = await import('./orcad-activation-fence-hold') +const { execOrcadRemote } = await import('./orcad-remote-runtime-control') +const { OrcadFenceLostError, runWithOrcadFence } = await import('./orcad-activation-fence-scope') +const { execOrcadStateMutation } = await import('./orcad-state-mutation-exec') +const { clearOrcadStateSnapshotMembersCommand } = await import('./orcad-state-snapshot') +const { getRemoteHostPlatform } = await import('./ssh-remote-platform') + +const homes: string[] = [] +afterEach(() => { + for (const home of homes.splice(0)) { + rmSync(home, { recursive: true, force: true }) + } +}) + +// Astra pass 8: a holder suspended past the stale window resumed, kept acting, and its release +// deleted the successor's fence and recovery journal mid-update. +describe.skipIf(process.platform === 'win32')('a superseded activation fence holder', () => { + it('can neither act nor release once a successor took its fence over', async () => { + const home = mkdtempSync(join(tmpdir(), 'orcad-fence-gen-')) + homes.push(home) + const root = join(home, '.orca-remote', '.orcad-activation-transaction') + const fence = join(root, '.install-lock') + const journal = join(root, 'transaction.json') + const launched = join(home, 'launched-by-stale-holder') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + const conn = {} as never + const options = { conn, host: getRemoteHostPlatform('linux-x64'), remoteHome: home } + + let resumeStale!: () => void + let staleEntered = false + const stale = withOrcadActivationLock( + options, + async () => { + staleEntered = true + await new Promise((resolve) => (resumeStale = resolve)) + // Resumed: it still believes it holds the fence and tries to start its slot. + await execOrcadRemote(options, `touch '${launched}'`) + return 'acted' + }, + () => 'held' + ) + await vi.waitFor(() => expect(staleEntered).toBe(true)) + // Suspended past the stale window: nothing refreshes its fence. + utimesSync(fence, new Date(0), new Date(0)) + expect(await orcadActivationFenceRefusal(options, 'update')).toMatchObject({ cleared: true }) + + let resumeSuccessor!: () => void + let successorEntered = false + const successor = withOrcadActivationLock( + options, + async () => { + // As the journal store writes it: stamped with the writing generation. + writeFileSync( + journal, + JSON.stringify({ phase: 'mid-update', fenceToken: 'successor' }, null, 2) + ) + successorEntered = true + await new Promise((resolve) => (resumeSuccessor = resolve)) + return 'done' + }, + () => 'held', + 'successor' + ) + await vi.waitFor(() => expect(successorEntered).toBe(true)) + + resumeStale() + await expect(stale).rejects.toBeInstanceOf(OrcadFenceLostError) + expect(existsSync(launched)).toBe(false) + // The successor's fence and journal survive the stale holder's release. + expect(existsSync(fence)).toBe(true) + expect(existsSync(journal)).toBe(true) + + resumeSuccessor() + expect(await successor).toBe('done') + expect(existsSync(fence)).toBe(false) + expect(existsSync(journal)).toBe(false) + }) + + // Astra pass 9: a release already past its token check stalled, a takeover and a successor + // came and went, and the resumed release deleted the successor's journal and lock. + it('leaves a successor’s journal and lock when its own release resumes after a stall', async () => { + const home = mkdtempSync(join(tmpdir(), 'orcad-fence-release-')) + homes.push(home) + const root = join(home, '.orca-remote', '.orcad-activation-transaction') + const fence = join(root, '.install-lock') + const journal = join(root, 'transaction.json') + const bin = join(home, 'bin') + const ready = join(home, 'ready') + const resume = join(home, 'resume') + mkdirSync(bin) + // Stalls the first move of the journal, the release step right after its token check. + writeFileSync( + join(bin, 'mv'), + `#!/bin/sh\nif [ "$1" = '${journal}' ] && [ ! -f '${ready}' ]; then touch '${ready}'; while [ ! -f '${resume}' ]; do sleep 0.05; done; fi\nexec /bin/mv "$@"\n` + ) + chmodSync(join(bin, 'mv'), 0o755) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + const conn = {} as never + const options = { conn, host: getRemoteHostPlatform('linux-x64'), remoteHome: home } + shell.env = { ...process.env, PATH: `${bin}:/usr/bin:/bin` } + let finishSuccessor: (() => void) | undefined + let successor: Promise | undefined + const first = withOrcadActivationLock( + options, + async () => 'first', + () => 'held', + 'first' + ) + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true)) + utimesSync(fence, new Date(0), new Date(0)) + expect(await orcadActivationFenceRefusal(options, 'update')).toMatchObject({ cleared: true }) + successor = withOrcadActivationLock( + options, + async () => { + writeFileSync(journal, JSON.stringify({ fenceToken: 'successor' }, null, 2)) + await new Promise((resolve) => (finishSuccessor = resolve)) + return 'done' + }, + () => 'held', + 'successor' + ) + await vi.waitFor(() => expect(finishSuccessor).toBeDefined()) + writeFileSync(resume, '') + expect(await first).toBe('first') + expect(readFileSync(join(fence, '.orca-fence-owner'), 'utf8')).toBe('successor') + expect(readFileSync(journal, 'utf8')).toContain('"fenceToken": "successor"') + } finally { + writeFileSync(resume, '') + await first.catch(() => {}) + finishSuccessor?.() + await successor + shell.env = undefined + } + }) + + // Astra pass 9 §3: a state mutation from a superseded run must not run either. + it('refuses a superseded run’s state mutation before it touches profile state', async () => { + const home = mkdtempSync(join(tmpdir(), 'orcad-fence-mutation-')) + homes.push(home) + const base = join(home, '.orca-remote') + const root = join(home, 'root') + const lockDir = join(base, '.orcad-activation-transaction', '.install-lock') + mkdirSync(lockDir, { recursive: true }) + mkdirSync(join(root, 'profiles'), { recursive: true }) + writeFileSync(join(lockDir, '.orca-fence-owner'), 'successor') + writeFileSync(join(root, 'profiles', 'state.json'), 'successor-state') + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + const conn = {} as never + const options = { conn, host: getRemoteHostPlatform('linux-x64'), remoteHome: home } + await expect( + runWithOrcadFence({ lockDir, token: 'stale' }, () => + execOrcadStateMutation( + options, + clearOrcadStateSnapshotMembersCommand(options.host, root, base) + ) + ) + ).rejects.toBeInstanceOf(OrcadFenceLostError) + expect(readFileSync(join(root, 'profiles', 'state.json'), 'utf8')).toBe('successor-state') + }) +}) diff --git a/src/main/ssh/orcad-activation-fence-hold.test.ts b/src/main/ssh/orcad-activation-fence-hold.test.ts new file mode 100644 index 00000000000..5e4c6b9f6f2 --- /dev/null +++ b/src/main/ssh/orcad-activation-fence-hold.test.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + stale: vi.fn(), + journal: vi.fn(), + fence: vi.fn(), + takeover: vi.fn() +})) +vi.mock('./ssh-relay-install-lock', () => ({ + isRelayInstallLockStale: mocks.stale, + RELAY_INSTALL_LOCK_NAME: '.install-lock' +})) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: mocks.journal +})) +vi.mock('./orcad-activation-lock', () => ({ + orcadActivationFenceExists: mocks.fence, + withStaleOrcadActivationRecoveryLock: mocks.takeover, + orcadActivationTransactionRoot: () => '/home/u/.orca-remote/.orcad-activation-transaction' +})) + +const { orcadActivationFenceRefusal } = await import('./orcad-activation-fence-hold') +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: every reader of these options is mocked. +const options = { host: { os: 'linux', pathFlavor: 'posix' }, remoteHome: '/home/u' } as never + +beforeEach(() => { + vi.resetAllMocks() + mocks.stale.mockResolvedValue(false) + mocks.journal.mockResolvedValue(null) + mocks.fence.mockResolvedValue(true) + mocks.takeover.mockImplementation(async (_options, run) => run({ retain: vi.fn() })) +}) + +describe('orcadActivationFenceRefusal', () => { + it('reads a fresh fence as busy, even over a live run journal', async () => { + mocks.journal.mockResolvedValue({ operation: 'activate' }) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_fence_busy' + }) + }) + + it('asks for Recover only for a stale lock over a journal, or a journal no fence guards', async () => { + mocks.journal.mockResolvedValue({ operation: 'activate' }) + mocks.stale.mockResolvedValueOnce(true) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_recovery_required' + }) + mocks.fence.mockResolvedValue(false) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_recovery_required' + }) + expect(mocks.takeover).not.toHaveBeenCalled() + }) + + // BUG-21: a wake cut short left a bare stale fence; every update said "Recover it first" while + // Recover answered "none". + it('clears a stale fence no journal backs, so the attempt can run again', async () => { + mocks.stale.mockResolvedValue(true) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_fence_busy', + cleared: true + }) + expect(mocks.takeover).toHaveBeenCalledOnce() + }) + + it('keeps a fence whose journal appeared under the takeover, and asks for Recover', async () => { + mocks.stale.mockResolvedValue(true) + mocks.journal.mockResolvedValueOnce(null).mockResolvedValue({ operation: 'activate' }) + const retain = vi.fn() + mocks.takeover.mockImplementation(async (_options, run) => run({ retain })) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_recovery_required' + }) + expect(retain).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ssh/orcad-activation-fence-hold.ts b/src/main/ssh/orcad-activation-fence-hold.ts new file mode 100644 index 00000000000..3125b8cf6a6 --- /dev/null +++ b/src/main/ssh/orcad-activation-fence-hold.ts @@ -0,0 +1,78 @@ +/** + * Why a fence answered "held": a run that is still working clears on its own and is retried on + * a later connect. A stale fence with no journal is cleared here, since Recover would only drop it. + * Only a stale lock over a journal, or a journal no fence guards, needs Recover: a live run + * journals under a fresh fence too, and Recover cannot take a fresh fence anyway. + */ +import { + orcadActivationFenceExists, + orcadActivationTransactionRoot, + withStaleOrcadActivationRecoveryLock, + type OrcadActivationLockOptions +} from './orcad-activation-lock' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { joinRemotePath } from './ssh-remote-platform' +import { findExitedOwnLockToken } from './orcad-exited-own-lock' +import { orcadRemoteBaseDir } from './orcad-remote-windows-node' + +export const ORCAD_ACTIVATION_FENCE_BUSY_CODE = 'orcad_activation_fence_busy' +export const ORCAD_ACTIVATION_RECOVERY_REQUIRED_CODE = 'orcad_activation_recovery_required' + +export type OrcadActivationFenceRefusal = { + code: string + reason: string + /** A stale fence no journal backed was cleared, so the attempt may run again at once. */ + cleared?: true +} + +export async function orcadActivationFenceRefusal( + options: OrcadActivationLockOptions, + attempt: string +): Promise { + const lockDir = joinRemotePath( + options.host, + orcadActivationTransactionRoot(options.host, options.remoteHome), + RELAY_INSTALL_LOCK_NAME + ) + // An unreadable answer reads as busy: retrying later is never wrong, a sticky failure can be. + const journal = (await readOrcadActivationTransaction(options).catch(() => null)) !== null + const baseDir = orcadRemoteBaseDir(options.host, options.remoteHome) + const stale = + (await findExitedOwnLockToken(options, lockDir, { baseDir, guardsStateMutation: true })) !== + null || (await isRelayInstallLockStale(options.conn, lockDir, options.host)) + if (stale && !journal && (await clearAbandonedFence(options))) { + // A wake or release cut short leaves a bare fence; Recover would only drop it (BUG-21). + return { + code: ORCAD_ACTIVATION_FENCE_BUSY_CODE, + reason: `An abandoned fence held this host and was cleared; the ${attempt} is retried.`, + cleared: true + } + } + const stuck = stale || (journal && !(await orcadActivationFenceExists(options).catch(() => true))) + return stuck + ? { + code: ORCAD_ACTIVATION_RECOVERY_REQUIRED_CODE, + reason: `An interrupted update or stop holds this host, so the ${attempt} did not start. Recover it first.` + } + : { + code: ORCAD_ACTIVATION_FENCE_BUSY_CODE, + reason: `Another run is changing this host's managed server, so the ${attempt} did not start. It is retried on a later connect.` + } +} + +/** Takes the stale fence over and drops it, unless a journal appeared under it meanwhile. */ +async function clearAbandonedFence(options: OrcadActivationLockOptions): Promise { + try { + return await withStaleOrcadActivationRecoveryLock(options, async (lock) => { + if (await readOrcadActivationTransaction(options)) { + lock.retain() + return false + } + return true + }) + } catch { + // Another client took it first, or the host did not answer: classify as before. + return false + } +} diff --git a/src/main/ssh/orcad-activation-fence-scope.ts b/src/main/ssh/orcad-activation-fence-scope.ts new file mode 100644 index 00000000000..fae162592fe --- /dev/null +++ b/src/main/ssh/orcad-activation-fence-scope.ts @@ -0,0 +1,61 @@ +/** + * Which activation fence this run holds, so every remote step it issues can prove it still does. + * + * Age only says a holder went quiet, not that it cannot act: a desktop suspended past the stale + * window resumes believing it owns the fence a successor has since taken (Astra pass 8). Each + * holder writes a generation token into the lock it creates, and each of its commands checks that + * token on the host, in the same command as the step, so a superseded holder aborts instead of + * launching, mutating state, or deleting the successor's fence and journal. + */ +import { AsyncLocalStorage } from 'node:async_hooks' +import { shellEscape } from './ssh-connection-utils' +import { isSshCommandExitError } from './ssh-relay-exec-command' + +export const ORCAD_FENCE_OWNER_FILENAME = '.orca-fence-owner' +export const ORCAD_FENCE_LOST_MARKER = '__ORCAD_FENCE_LOST__' +/** EX_TEMPFAIL: the step did not run, and retrying under this fence never will. */ +export const ORCAD_FENCE_LOST_EXIT = 75 + +export type OrcadFence = { lockDir: string; token: string } + +const scope = new AsyncLocalStorage() + +export function runWithOrcadFence(fence: OrcadFence, run: () => Promise): Promise { + return scope.run(fence, run) +} + +export function currentOrcadFence(): OrcadFence | null { + return scope.getStore() ?? null +} + +export class OrcadFenceLostError extends Error { + constructor() { + super( + "This run's activation fence was taken over by another run, so it stopped before changing the host." + ) + this.name = 'OrcadFenceLostError' + } +} + +/** A POSIX test that succeeds only while the lock still carries this run's token. */ +export function posixOrcadFenceOwnedTest(fence: OrcadFence): string { + const owner = shellEscape(`${fence.lockDir.replace(/\/+$/u, '')}/${ORCAD_FENCE_OWNER_FILENAME}`) + return `[ "$(cat ${owner} 2>/dev/null)" = ${shellEscape(fence.token)} ]` +} + +export function posixOrcadFenceGuard(fence: OrcadFence): string { + return `${posixOrcadFenceOwnedTest(fence)} || { echo ${ORCAD_FENCE_LOST_MARKER}; exit ${ORCAD_FENCE_LOST_EXIT}; };` +} + +/** + * Classified from the exit status and the step's own stdout, never the message: the message + * quotes the command, and every fenced command carries the guard's marker text. + */ +export function isOrcadFenceLost(error: unknown): boolean { + if (!isSshCommandExitError(error)) { + return false + } + // `powershell -Command` reports any nonzero native exit as 1. + const exited = error.exitCode === ORCAD_FENCE_LOST_EXIT || error.exitCode === 1 + return exited && error.stdout.trim().split(/\r?\n/u).at(-1) === ORCAD_FENCE_LOST_MARKER +} diff --git a/src/main/ssh/orcad-activation-gate.test.ts b/src/main/ssh/orcad-activation-gate.test.ts index e3afdcd142e..43ca270448a 100644 --- a/src/main/ssh/orcad-activation-gate.test.ts +++ b/src/main/ssh/orcad-activation-gate.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { evaluateOrcadActivation } from './orcad-activation-gate' +import { classifyOrcadHostUnavailable } from './orcad-host-unavailable' import type { ServeReadiness } from '../server/serve-readiness' import type { OrcadHealth, TerminalDaemonHealth } from '../orcad/orcad-health' @@ -85,6 +86,8 @@ describe('evaluateOrcadActivation', () => { EXPECTED ) expect(verdict).toMatchObject({ decision: 'reject', code: 'orcad_activation_daemon_absent' }) + // A cold daemon start is transient: the host must stay eligible for the next connect's deploy. + expect(classifyOrcadHostUnavailable({ code: 'orcad_activation_daemon_absent' })).toBeNull() }) it('refuses a degraded daemon, whose fresh terminals would not survive a restart', () => { diff --git a/src/main/ssh/orcad-activation-gate.ts b/src/main/ssh/orcad-activation-gate.ts index c9581919117..d514d6858e4 100644 --- a/src/main/ssh/orcad-activation-gate.ts +++ b/src/main/ssh/orcad-activation-gate.ts @@ -105,7 +105,8 @@ export function evaluateOrcadActivation( reason: 'The candidate has no terminal daemon. Every terminal on this host would run in the ' + 'orcad process and die with it, which is the exact regression the daemon exists to ' + - 'prevent. Nothing was activated.' + "prevent. Nothing was activated; orcad.log carries the daemon's startup error, and " + + 'the next connect retries.' } } if (daemon.state === 'degraded') { diff --git a/src/main/ssh/orcad-activation-host-test-harness.ts b/src/main/ssh/orcad-activation-host-test-harness.ts new file mode 100644 index 00000000000..7d909bbf91a --- /dev/null +++ b/src/main/ssh/orcad-activation-host-test-harness.ts @@ -0,0 +1,334 @@ +/** + * A stateful fake orcad host for crash-recovery tests: slot processes, PID files, the shared + * profile state, snapshots, the activation record, the journal and its fence. It can drop the + * connection at any numbered mutation, before or after the host applied it. + */ +import { + emptyOrcadActivationRecord, + parseOrcadActivationRecord, + withActivatedVersion, + type OrcadActivationRecord +} from './orcad-activation-record' +import { isSnapshotCaptureCommand } from './orcad-snapshot-capture-command' +import { sshCommandExitError } from './ssh-relay-exec-command' + +export const OLD = '0.1.0+aa01' +export const NEW = '0.2.0+bb01' +export const BUILD_HASH = 'abc123def4567890' +/** State the incoming build migrates on load; the outgoing build cannot read it. */ +const MIGRATION = '|migrated-by-new' + +export type CrashMode = 'before' | 'after' + +/** The one-shot readiness read, or the launch loop's host-side wait. */ +export function isReadinessRead(command: string): boolean { + return command.startsWith('head -c ') || command.includes('orcad_readiness_wait') +} + +const WAKE_OWNER = '.orca-fence-owner' +const FENCE_GUARD = + /^\[ "\$\(cat '[^']*' 2>\/dev\/null\)" = '([^']*)' \] \|\| \{ echo (__ORCAD_FENCE_LOST__; exit 75|SUPERSEDED; exit 0); \};\s*/u + +export class FakeOrcadHost { + record: string | null = null + journal: string | null = null + fence = false + /** Set by a recovery takeover, which recreates the lock fresh; the ownerless mark clears it. */ + fenceFresh = false + wakeOwner: string | null = null + readonly alive = new Set() + readonly pidFiles = new Set() + data = 'profiles-v1' + readonly snapshots = new Map() + readonly commands: string[] = [] + mutations = 0 + crashAt: number | null = null + crashMode: CrashMode = 'before' + /** How the slot's managed-stop command behaves: orcad exits, or keeps serving. */ + managedStop: 'exits' | 'stays' | 'stays-dispatched' = 'exits' + /** What the daemon reports at a managed stop's terminal fence. */ + retirement: 'retired' | 'live' | 'unverifiable' = 'retired' + readonly dispatched = new Set() + + static deployedOld(): FakeOrcadHost { + const host = new FakeOrcadHost() + host.record = JSON.stringify( + withActivatedVersion(emptyOrcadActivationRecord(), OLD, null, new Date(0)) + ) + host.alive.add(OLD) + host.pidFiles.add(OLD) + return host + } + + static activatedNew(): FakeOrcadHost { + const host = FakeOrcadHost.deployedOld() + host.alive.clear() + host.pidFiles.add(NEW) + host.alive.add(NEW) + host.snapshots.set('pre-0.2.0+bb01-1000', host.data) + host.data += MIGRATION + host.record = JSON.stringify(FakeOrcadHost.newRecord()) + return host + } + + static newRecord(): OrcadActivationRecord { + return withActivatedVersion( + { + ...emptyOrcadActivationRecord(), + active: OLD, + activatedAt: new Date(0).toISOString() + }, + NEW, + { + dirName: 'pre-0.2.0+bb01-1000', + takenBeforeVersion: NEW, + readableByVersion: OLD, + takenAt: new Date(1000).toISOString() + }, + new Date(1000) + ) + } + + activeVersion(): string | null { + const parsed = parseOrcadActivationRecord(this.record) + return parsed.state === 'ok' ? parsed.record.active : null + } + + /** Every state the old build can read: the pre-migration profile. */ + isReadableBy(version: string | null): boolean { + return version !== OLD || !this.data.includes(MIGRATION) + } + + private mutate(apply: () => T): T { + this.mutations += 1 + if (this.crashAt !== this.mutations) { + return apply() + } + if (this.crashMode === 'after') { + apply() + } + throw Object.assign(new Error(`connection lost at mutation ${this.mutations}`), { + sshChannelCloseConfirmed: false + }) + } + + write(path: string, contents: string): void { + this.mutate(() => { + if (path.endsWith('transaction.json')) { + this.journal = contents + } else if (path.endsWith('orcad-active.json')) { + this.record = contents + } + }) + } + + /** Returns false where a stale-only takeover would answer busy. */ + acquireFence(options?: { allowStaleTakeover?: boolean; owner?: { token: string } }): boolean { + if (options?.allowStaleTakeover && this.fence && this.fenceFresh) { + return false + } + this.fenceFresh = options?.allowStaleTakeover === true && this.fence + this.fence = true + // The real lock writes the holder's generation token in the command that creates it. + this.wakeOwner = options?.owner?.token ?? this.wakeOwner + return true + } + + exec(command: string): string { + this.commands.push(command) + return this.execInner(command) + } + + private execInner(command: string): string { + const guard = FENCE_GUARD.exec(command) + if (guard) { + const owned = this.fence && this.wakeOwner === guard[1] + if (command.includes('echo RELEASED')) { + return owned + ? this.mutate(() => { + this.journal = null + this.fence = false + this.wakeOwner = null + return 'RELEASED' + }) + : 'SUPERSEDED' + } + if (!owned) { + throw sshCommandExitError(command, 75, '__ORCAD_FENCE_LOST__\n') + } + return this.execInner(command.slice(guard[0].length)) + } + if (command.startsWith('touch -m -t 200001010000')) { + this.fenceFresh = false + return '' + } + // The holder's token lives inside the fence's lock dir, so the fence's release drops it. + // A state mutation's fence heartbeat names the token only to check it is still its own. + if (command.includes(WAKE_OWNER) && !command.includes('orcad-state-mutation.lock')) { + return this.wakeOwner === null || !this.fence + ? '__ORCAD_RECORD_ABSENT__\n' + : `__ORCAD_RECORD_PRESENT__\n${this.wakeOwner}` + } + const version = /\/orcad-(\d+\.\d+\.\d+\+[0-9a-f]+)/u.exec(command)?.[1] ?? null + const snapshot = /orcad-state-snapshots\/([A-Za-z0-9][A-Za-z0-9.+-]*)/u.exec(command)?.[1] + if (command.includes('__ORCAD_RECORD_PRESENT__') && command.includes('orcad.lock')) { + const owner = [...this.alive][0] + return owner + ? `__ORCAD_RECORD_PRESENT__\n${JSON.stringify(lockRecord(owner))}` + : '__ORCAD_RECORD_ABSENT__\n' + } + if (command.includes('-managed-stop ') && version) { + return this.runManagedStopCommand(command, version) + } + if (command.includes('__ORCAD_RECORD_PRESENT__')) { + const value = command.includes('transaction.json') ? this.journal : this.record + return value === null ? '__ORCAD_RECORD_ABSENT__\n' : `__ORCAD_RECORD_PRESENT__\n${value}` + } + if (command.includes('echo LOCKED || echo OPEN')) { + return this.fence ? 'LOCKED' : 'OPEN' + } + if (command.startsWith('rm -f') && command.includes('transaction.json')) { + return this.mutate(() => { + this.journal = null + this.fence = false + return '' + }) + } + if (command.includes('__ORCAD_BUILD_HASH__')) { + return `__ORCAD_BUILD_HASH__ ${BUILD_HASH}\n` + } + if (command.includes('orca-runtime.json')) { + return this.alive.size > 0 ? 'LIVE orcad.lock 1' : 'CLEAR' + } + if (command.includes('echo LIVE;') && version) { + if (this.alive.has(version)) { + return 'LIVE' + } + return this.pidFiles.has(version) ? 'DEAD' : 'UNKNOWN' + } + if (command.includes('kill -TERM') && version) { + if (!this.pidFiles.has(version)) { + return 'NO_PID' + } + return this.mutate(() => (this.alive.delete(version) ? 'STOPPED' : 'ALREADY_EXITED')) + } + if (command.includes('nohup') && version) { + return this.mutate(() => { + this.pidFiles.add(version) + // The instance lock and port admit one owner; a second launch never becomes ready. + if (this.alive.size === 0) { + this.alive.add(version) + if (version === NEW && !this.data.includes(MIGRATION)) { + this.data += MIGRATION + } + } + return '9999' + }) + } + if (isReadinessRead(command) && version) { + return this.alive.has(version) ? readyLine(version) : '' + } + if (command.includes('echo PRESENT') && snapshot) { + return this.snapshots.has(snapshot) ? 'PRESENT' : 'ABSENT' + } + if (command.includes('verdict=UNCHANGED') && snapshot) { + return this.snapshots.get(snapshot) === this.data ? 'UNCHANGED' : 'CHANGED' + } + if (isSnapshotCaptureCommand(command) && snapshot) { + return this.mutate(() => { + if (this.data === '') { + return 'EMPTY' + } + this.snapshots.set(snapshot, this.data) + return 'CAPTURED' + }) + } + if (command.includes('.orcad-state-restore-stage') && snapshot) { + return this.mutate(() => { + const restored = this.snapshots.get(snapshot) + if (restored === undefined) { + return 'MISSING' + } + this.data = restored + return 'RESTORED' + }) + } + if (command.includes('then echo RESTORED')) { + return this.mutate(() => { + this.data = '' + return 'RESTORED' + }) + } + if (command.includes('stat -c %Y')) { + return 'UNKNOWN' + } + return '' + } + + /** The slot's `--complete-managed-stop` / `--cancel-managed-stop`, as orcad answers them. */ + private runManagedStopCommand(command: string, version: string): string { + const request = JSON.parse(command.slice(command.lastIndexOf(" '{") + 2, -1)) + if (command.includes('--cancel-managed-stop')) { + const outcome = this.dispatched.has(request.transactionId) ? 'dispatched' : 'canceled' + return JSON.stringify({ ...request, kind: 'orcad_managed_stop_cancellation', outcome }) + } + const verdict = this.mutate(() => { + if (this.managedStop === 'exits') { + this.dispatched.add(request.transactionId) + this.alive.delete(version) + } else if (this.managedStop === 'stays-dispatched') { + this.dispatched.add(request.transactionId) + } + return this.alive.has(version) ? 'live' : 'exited' + }) + return JSON.stringify({ + ...request, + kind: 'orcad_managed_stop_completion', + verdict, + receiptPersisted: verdict === 'exited', + ...(verdict === 'exited' ? { retirement: this.retirement } : {}) + }) + } +} + +function lockRecord(version: string) { + return { + pid: 1, + startedAtMs: null, + identity: 'uid-1000', + version, + acquiredAt: new Date(0).toISOString(), + nonce: `nonce-${version}` + } +} + +export function readyLine(version: string): string { + return JSON.stringify({ + type: 'orca_server_ready', + schemaVersion: 1, + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: BUILD_HASH, + buildVersion: version, + nodeVersion: '24.21.0', + nodeAbi: '137', + platform: 'linux', + arch: 'x64', + pid: 1, + stopRequests: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: version, + entryPath: '/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'pty-spawn', verdict: 'healthy', durationMs: 5 } + } + } + }) +} diff --git a/src/main/ssh/orcad-activation-lock.ts b/src/main/ssh/orcad-activation-lock.ts new file mode 100644 index 00000000000..7dc63093dd8 --- /dev/null +++ b/src/main/ssh/orcad-activation-lock.ts @@ -0,0 +1,310 @@ +/** + * One activation or rollback per host, and the fence an interrupted one leaves behind. + * + * The lock lives in the transaction root, so releasing it also removes the journal. A run + * that cannot prove the host is back to one serving slot retains both and marks the lock + * ownerless; only recovery may take a retained fence over, and it waits out the install lock's + * stale window only for a fence whose holder may still be working. + */ +import { randomUUID } from 'node:crypto' +import { + execOrcadRemote, + withoutAbortSignal, + type OrcadRemoteExecTarget +} from './orcad-remote-runtime-control' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { + acquireInstallLock, + RELAY_INSTALL_LOCK_NAME, + RemoteInstallLockBusyError +} from './ssh-relay-install-lock' +import { + orphanInstallLockCommand, + probeInstallLockExistsCommand +} from './ssh-relay-install-lock-commands' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { shellEscape } from './ssh-connection-utils' +import { + ORCAD_FENCE_OWNER_FILENAME, + posixOrcadFenceOwnedTest, + runWithOrcadFence, + type OrcadFence +} from './orcad-activation-fence-scope' +import { orcadRemoteBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { forgetHeldOrcadFence, rememberHeldOrcadFence } from './orcad-held-fence-tokens' +import { exitedOwnLockProof } from './orcad-exited-own-lock' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + ORCAD_ACTIVATION_TRANSACTION_DIRNAME, + ORCAD_ACTIVATION_TRANSACTION_FILENAME +} from './orcad-activation-transaction' + +const ORCAD_ACTIVATION_MAX_READINESS_TIMEOUT_MS = 5 * 60_000 + +export type OrcadActivationLockOptions = OrcadRemoteExecTarget & { remoteHome: string } + +export type OrcadActivationLockControl = { + /** Keep the fence if the run throws: a journal now describes host state. */ + retainOnError(): void + /** Keep the fence even on return: the host is not proven back to one serving slot. */ + retain(): void + /** The host is proven back on its recorded slot: release even if the run then throws. */ + recovered(): void +} + +export function orcadActivationTransactionRoot( + host: RemoteHostPlatform, + remoteHome: string +): string { + return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_TRANSACTION_DIRNAME) +} + +/** Bounded so a crashed holder's lock goes stale long before a live holder could still be waiting. */ +export function resolveOrcadActivationReadinessTimeout( + configured: number | undefined, + fallback: number +): number { + const timeout = configured ?? fallback + if ( + !Number.isSafeInteger(timeout) || + timeout <= 0 || + timeout > ORCAD_ACTIVATION_MAX_READINESS_TIMEOUT_MS + ) { + throw new Error( + `orcad readiness timeout must be an integer from 1 to ${ORCAD_ACTIVATION_MAX_READINESS_TIMEOUT_MS}ms` + ) + } + return timeout +} + +// Long enough for a brief hold (a wake, a retried lock command), short beside a lifecycle queue. +const ORCAD_ACTIVATION_FENCE_WAIT_MS = 5_000 + +/** + * A fence still held after a short wait answers `held()`: a retained one never clears by waiting. + * `token` names this run's generation; a wake passes its own so it can prove an interrupted fence. + */ +export async function withOrcadActivationLock( + options: OrcadActivationLockOptions, + run: (control: OrcadActivationLockControl) => Promise, + held: () => T | Promise, + token: string = randomUUID() +): Promise { + const lockRoot = orcadActivationTransactionRoot(options.host, options.remoteHome) + rememberHeldOrcadFence(token) + try { + await acquireInstallLock(options.conn, lockRoot, options.host, { + signal: options.signal, + relayGcClaim: false, + // A retained fence means state ownership is unresolved. Age cannot make it safe. + allowStaleTakeover: false, + waitTimeoutMs: ORCAD_ACTIVATION_FENCE_WAIT_MS, + owner: { fileName: ORCAD_FENCE_OWNER_FILENAME, token } + }) + } catch (error) { + if (error instanceof RemoteInstallLockBusyError) { + forgetHeldOrcadFence(token) + return await held() + } + throw error + } + const fence = activationFence(options, token) + let retainOnError = false + let retain = false + try { + const result = await runWithOrcadFence(fence, () => + run({ + retainOnError: () => { + retainOnError = true + }, + retain: () => { + retain = true + }, + recovered: () => { + retainOnError = false + } + }) + ) + await (retain ? orphanRetainedFence(options, fence) : releaseActivationFence(options, fence)) + return result + } catch (error) { + // A remote mutation whose teardown is unconfirmed may still be running: keep its fence fresh. + if (!isUnconfirmedSshCommandTermination(error)) { + await ( + retainOnError ? orphanRetainedFence(options, fence) : releaseActivationFence(options, fence) + ).catch((releaseError: unknown) => { + console.warn( + `[orcad] Failed to release activation lock after an error: ${releaseError instanceof Error ? releaseError.message : String(releaseError)}` + ) + }) + } + throw error + } +} + +/** + * Takes over only a stale or previous-boot fence, or one this desktop's exited process left; + * a fresh one throws `RemoteInstallLockBusyError`. + */ +export async function withStaleOrcadActivationRecoveryLock( + options: OrcadActivationLockOptions, + run: (control: Pick) => Promise +): Promise { + const lockRoot = orcadActivationTransactionRoot(options.host, options.remoteHome) + const token = randomUUID() + rememberHeldOrcadFence(token) + // The takeover writes this run's token, so the holder it replaced can no longer act or release. + await acquireInstallLock(options.conn, lockRoot, options.host, { + signal: options.signal, + relayGcClaim: false, + allowStaleTakeover: true, + waitTimeoutMs: 0, + owner: { fileName: ORCAD_FENCE_OWNER_FILENAME, token }, + exitedOwner: exitedOwnLockProof(options, { + baseDir: orcadRemoteBaseDir(options.host, options.remoteHome), + guardsStateMutation: true + }) + }).catch((error: unknown) => { + if (error instanceof RemoteInstallLockBusyError) { + forgetHeldOrcadFence(token) + } + throw error + }) + const fence = activationFence(options, token) + let retain = false + let result: T + try { + result = await runWithOrcadFence(fence, async () => { + await adoptInterruptedJournal(options) + return run({ retain: () => (retain = true) }) + }) + } catch (error) { + // Any throw keeps the fence: recovery failed to prove one serving slot. + if (!isUnconfirmedSshCommandTermination(error)) { + await orphanRetainedFence(options, fence).catch(() => undefined) + } + throw error + } + await (retain ? orphanRetainedFence(options, fence) : releaseActivationFence(options, fence)) + return result +} + +/** Re-stamps a taken-over run's journal with this generation, so this run's release removes it. */ +async function adoptInterruptedJournal(options: OrcadActivationLockOptions): Promise { + // Dynamic: the journal store imports this module for the transaction root. + const store = await import('./orcad-activation-transaction-store') + const journal = await store.readOrcadActivationTransaction(options).catch(() => null) + if (journal) { + await store.writeOrcadActivationTransaction(options, journal) + } +} + +function activationFence(options: OrcadActivationLockOptions, token: string): OrcadFence { + const root = orcadActivationTransactionRoot(options.host, options.remoteHome) + return { lockDir: joinRemotePath(options.host, root, RELAY_INSTALL_LOCK_NAME), token } +} + +/** + * A fence this run keeps after it is done: nothing of ours still works under it, so the next + * recovery may take it over at once. Left fresh, every failed recovery would restart the stale + * window it waits out, and the host could never be recovered. Guarded, so a superseded run + * never ages its successor's fence. + */ +async function orphanRetainedFence( + options: OrcadActivationLockOptions, + fence: OrcadFence +): Promise { + try { + await runWithOrcadFence(fence, () => + execOrcadRemote( + withoutAbortSignal(options), + orphanInstallLockCommand(options.host, fence.lockDir) + ) + ) + forgetHeldOrcadFence(fence.token) + } catch (error) { + // Best effort: the fence still holds; recovery then waits out the stale window as before. + console.warn( + `[orcad] Could not mark a retained activation fence as ownerless: ${String(error)}` + ) + } +} + +/** Whether any lock is held; a lost probe throws rather than reading as open. */ +export async function orcadActivationFenceExists( + options: OrcadActivationLockOptions +): Promise { + const lockDir = joinRemotePath( + options.host, + orcadActivationTransactionRoot(options.host, options.remoteHome), + RELAY_INSTALL_LOCK_NAME + ) + const answer = ( + await execOrcadRemote(options, probeInstallLockExistsCommand(options.host, lockDir)) + ).trim() + if (answer !== 'LOCKED' && answer !== 'OPEN') { + throw new Error('The activation fence probe returned no verifiable answer.') + } + return answer === 'LOCKED' +} + +/** Drops the fence `token` names, and nothing else: a successor's fence carries its own token. */ +export function releaseOrcadActivationFence( + options: OrcadActivationLockOptions, + token: string +): Promise { + return releaseActivationFence(options, activationFence(options, token)) +} + +/** + * Conditional on the host: only while the lock still carries this run's token, journal first, + * then the lock renamed aside and removed, so a successor's fresh lock is never what goes. + */ +async function releaseActivationFence( + options: OrcadActivationLockOptions, + fence: OrcadFence +): Promise { + const lockRoot = orcadActivationTransactionRoot(options.host, options.remoteHome) + const journal = joinRemotePath(options.host, lockRoot, ORCAD_ACTIVATION_TRANSACTION_FILENAME) + // Why no signal: a cancelled run must still be able to drop a fence it proved unnecessary. + const target = withoutAbortSignal(options) + const command = isWindowsRemoteHost(options.host) + ? orcadWindowsHostOpCommand( + options.host, + orcadRemoteBaseDir(options.host, options.remoteHome), + 'fence-release', + [fence.lockDir, journal, fence.token] + ) + : posixReleaseFenceCommand(fence, journal, lockRoot) + const answer = (await execOrcadRemote(target, command)).trim() + forgetHeldOrcadFence(fence.token) + if (answer.endsWith('SUPERSEDED')) { + console.warn('[orcad] A newer run had taken this activation fence over; it was left in place.') + } +} + +/** + * Each piece is renamed aside first and kept only if it is ours, else put straight back: a release + * that stalls after its token check can then never delete a successor's journal or lock. + */ +function posixReleaseFenceCommand(fence: OrcadFence, journal: string, lockRoot: string): string { + const lock = shellEscape(fence.lockDir) + const journalPath = shellEscape(journal) + const stamp = shellEscape(journalFenceStamp(fence.token)) + return [ + `${posixOrcadFenceOwnedTest(fence)} || { echo SUPERSEDED; exit 0; };`, + `ja=${journalPath}.release.$$;`, + `if mv ${journalPath} "$ja" 2>/dev/null; then`, + `grep -qF ${stamp} "$ja" || mv -n "$ja" ${journalPath} 2>/dev/null; rm -f "$ja"; fi;`, + `la=${lock}.released.$$;`, + `if mv ${lock} "$la" 2>/dev/null; then`, + `if [ "$(cat "$la"/${ORCAD_FENCE_OWNER_FILENAME} 2>/dev/null)" = ${shellEscape(fence.token)} ]; then rm -rf "$la";`, + `else mv -n "$la" ${lock} 2>/dev/null; fi; fi;`, + `rmdir ${shellEscape(lockRoot)} 2>/dev/null; echo RELEASED` + ].join(' ') +} + +/** How a journal this generation wrote names it (see writeOrcadActivationTransaction). */ +export function journalFenceStamp(token: string): string { + return `"fenceToken": ${JSON.stringify(token)}` +} diff --git a/src/main/ssh/orcad-activation-outcome-log.test.ts b/src/main/ssh/orcad-activation-outcome-log.test.ts new file mode 100644 index 00000000000..82e03e8831e --- /dev/null +++ b/src/main/ssh/orcad-activation-outcome-log.test.ts @@ -0,0 +1,35 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { logOrcadActivationOutcome } from './orcad-activation-outcome-log' + +afterEach(() => { + vi.restoreAllMocks() +}) + +it('logs the code and reason of an update that was not activated', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + await logOrcadActivationOutcome( + 'update to 0.2.0', + async () => ({ + outcome: 'installed-not-activated', + code: 'orcad_candidate_launch_failed', + reason: 'The candidate failed while starting.' + }), + ['installed-and-activated'] + ) + expect(warn).toHaveBeenCalledWith( + '[orcad] update to 0.2.0 installed-not-activated (orcad_candidate_launch_failed): The candidate failed while starting.' + ) +}) + +it('logs a run that threw, and stays quiet for one that went through', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + await expect( + logOrcadActivationOutcome('rollback to 0.1.0', () => Promise.reject(new Error('lost')), []) + ).rejects.toThrow('lost') + expect(warn).toHaveBeenCalledWith('[orcad] rollback to 0.1.0 failed: lost') + warn.mockClear() + await logOrcadActivationOutcome('rollback to 0.1.0', async () => ({ outcome: 'rolled-back' }), [ + 'rolled-back' + ]) + expect(warn).not.toHaveBeenCalled() +}) diff --git a/src/main/ssh/orcad-activation-outcome-log.ts b/src/main/ssh/orcad-activation-outcome-log.ts new file mode 100644 index 00000000000..b5bc98031f1 --- /dev/null +++ b/src/main/ssh/orcad-activation-outcome-log.ts @@ -0,0 +1,27 @@ +/** + * The app log line for an update or rollback that did not go through. Status keeps only the + * latest deferral, so without this the first refusal's code and reason are lost (BUG-17). + */ +import { errorMessage } from '../../shared/error-message' + +type Unsettled = { outcome: string; code?: string; reason?: string } + +export async function logOrcadActivationOutcome( + operation: string, + run: () => Promise, + settled: readonly string[] +): Promise { + let result: T + try { + result = await run() + } catch (error) { + console.warn(`[orcad] ${operation} failed: ${errorMessage(error)}`) + throw error + } + if (!settled.includes(result.outcome)) { + console.warn( + `[orcad] ${operation} ${result.outcome} (${result.code ?? 'no code'}): ${result.reason ?? ''}` + ) + } + return result +} diff --git a/src/main/ssh/orcad-activation-record-store.ts b/src/main/ssh/orcad-activation-record-store.ts index d60634ec525..a36dcc728e0 100644 --- a/src/main/ssh/orcad-activation-record-store.ts +++ b/src/main/ssh/orcad-activation-record-store.ts @@ -6,32 +6,50 @@ * activated" would deploy over a live install and lose its rollback target. */ import type { SshConnection } from './ssh-connection' -import { execCommand } from './ssh-relay-deploy-helpers' import { RELAY_REMOTE_DIR } from './relay-protocol' import { ORCAD_ACTIVATION_FILENAME, emptyOrcadActivationRecord, parseOrcadActivationRecord, + serializeOrcadActivationRecord, + type OrcadActivationReadResult, type OrcadActivationRecord } from './orcad-activation-record' import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' + +const ORCAD_ACTIVATION_RECORD_MAX_BYTES = 64 * 1024 + +type ActivationRecordTarget = { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + signal?: AbortSignal +} export function orcadActivationPath(host: RemoteHostPlatform, remoteHome: string): string { return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR, ORCAD_ACTIVATION_FILENAME) } -export async function readOrcadActivationRecord(options: { - conn: SshConnection - host: RemoteHostPlatform - remoteHome: string - signal?: AbortSignal -}): Promise { - const path = orcadActivationPath(options.host, options.remoteHome) - const raw = await execCommand(options.conn, `cat ${shellQuote(path)} 2>/dev/null || true`, { - wrapCommand: options.host.commandDialect !== 'powershell', - signal: options.signal - }).catch(() => '') - const parsed = parseOrcadActivationRecord(raw) +/** A failed read rejects; it never reads as absent, because absence would admit a fresh deploy. */ +async function readActivationRecordState( + options: ActivationRecordTarget +): Promise { + const read = await readBoundedOrcadRemoteRecord( + options, + orcadActivationPath(options.host, options.remoteHome), + ORCAD_ACTIVATION_RECORD_MAX_BYTES + ) + return read.state === 'absent' ? read : parseOrcadActivationRecord(read.raw) +} + +export async function readOrcadActivationRecord( + options: ActivationRecordTarget +): Promise { + const parsed = await readActivationRecordState(options) if (parsed.state === 'ok') { return parsed.record } @@ -43,6 +61,18 @@ export async function readOrcadActivationRecord(options: { return emptyOrcadActivationRecord() } -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'\\''`)}'` +/** Refuses to replace a record this client cannot read, e.g. one a newer client wrote. */ +export async function writeOrcadActivationRecord( + options: ActivationRecordTarget, + record: OrcadActivationRecord +): Promise { + const existing = await readActivationRecordState(options) + if (existing.state === 'unreadable') { + throw new Error(`Refusing to overwrite this host's orcad activation record: ${existing.reason}`) + } + await writeAtomicOrcadRemoteRecord( + options, + orcadActivationPath(options.host, options.remoteHome), + serializeOrcadActivationRecord(record) + ) } diff --git a/src/main/ssh/orcad-activation-record.test.ts b/src/main/ssh/orcad-activation-record.test.ts index 4950045daa5..127c3284003 100644 --- a/src/main/ssh/orcad-activation-record.test.ts +++ b/src/main/ssh/orcad-activation-record.test.ts @@ -9,6 +9,7 @@ import { withRolledBackVersion, type OrcadStateSnapshot } from './orcad-activation-record' +import { sameOrcadActivationRecord } from './orcad-activation-transaction' const SNAPSHOT: OrcadStateSnapshot = { dirName: 'pre-0.2.0+bb01-1000', @@ -107,4 +108,43 @@ describe('orcad activation record', () => { ) expect(pinned).toEqual(['orcad-0.3.0+cc01']) }) + + it('records which Orca activated each version, and the build a rollback left', () => { + const first = withActivatedVersion( + emptyOrcadActivationRecord(), + '0.1.0+aa01', + null, + NOW, + '1.4.0' + ) + const second = withActivatedVersion(first, '0.2.0+bb01', SNAPSHOT, NOW, '1.5.0') + expect(second).toMatchObject({ activeAppVersion: '1.5.0', previousAppVersion: '1.4.0' }) + expect(parseOrcadActivationRecord(serializeOrcadActivationRecord(second))).toEqual({ + state: 'ok', + record: second + }) + const rolledBack = withRolledBackVersion(second, NOW) + expect(rolledBack).toMatchObject({ + active: '0.1.0+aa01', + activeAppVersion: '1.4.0', + rolledBackFrom: '0.2.0+bb01' + }) + // A later activation is an explicit choice, so it lifts the hold. + expect(withActivatedVersion(rolledBack, '0.3.0+cc01', null, NOW, '1.6.0')).not.toHaveProperty( + 'rolledBackFrom' + ) + }) + + it('matches a journal from a build that drops the advisory fields', () => { + const current = withActivatedVersion( + emptyOrcadActivationRecord(), + '0.1.0+aa01', + null, + NOW, + '1.5.0' + ) + const { activeAppVersion: _dropped, ...older } = current + expect(sameOrcadActivationRecord(current, older)).toBe(true) + expect(sameOrcadActivationRecord(current, { ...older, active: '0.2.0+bb01' })).toBe(false) + }) }) diff --git a/src/main/ssh/orcad-activation-record.ts b/src/main/ssh/orcad-activation-record.ts index 42b1e6703a8..0b7c7703a5f 100644 --- a/src/main/ssh/orcad-activation-record.ts +++ b/src/main/ssh/orcad-activation-record.ts @@ -36,6 +36,14 @@ export type OrcadActivationRecord = { previous: string | null activatedAt: string | null snapshot: OrcadStateSnapshot | null + /** + * The Orca app versions that activated `active` and `previous`, so a client can tell a host + * deployed by a newer Orca. Optional: builds without them write records that omit both. + */ + activeAppVersion?: string + previousAppVersion?: string + /** The build an explicit rollback left; updating on connect never reactivates it. */ + rolledBackFrom?: string } export function emptyOrcadActivationRecord(): OrcadActivationRecord { @@ -92,7 +100,16 @@ export function parseOrcadActivationRecord(raw: string | null): OrcadActivationR active: typeof record.active === 'string' ? record.active : null, previous: typeof record.previous === 'string' ? record.previous : null, activatedAt: typeof record.activatedAt === 'string' ? record.activatedAt : null, - snapshot: parseSnapshot(record.snapshot) + snapshot: parseSnapshot(record.snapshot), + ...(typeof record.activeAppVersion === 'string' + ? { activeAppVersion: record.activeAppVersion } + : {}), + ...(typeof record.previousAppVersion === 'string' + ? { previousAppVersion: record.previousAppVersion } + : {}), + ...(typeof record.rolledBackFrom === 'string' + ? { rolledBackFrom: record.rolledBackFrom } + : {}) } } } @@ -114,6 +131,20 @@ function parseSnapshot(value: unknown): OrcadStateSnapshot | null { } } +/** + * The fields that commit an activation. Why not the advisory ones: an older client drops them + * when it journals a record, and its journal must still match the record on the host. + */ +export function coreOrcadActivationRecord(record: OrcadActivationRecord): OrcadActivationRecord { + const { + activeAppVersion: _app, + previousAppVersion: _prev, + rolledBackFrom: _held, + ...core + } = record + return core +} + export function serializeOrcadActivationRecord(record: OrcadActivationRecord): string { return `${JSON.stringify(record, null, 2)}\n` } @@ -123,17 +154,22 @@ export function withActivatedVersion( record: OrcadActivationRecord, version: string, snapshot: OrcadStateSnapshot | null, - now: Date + now: Date, + appVersion?: string ): OrcadActivationRecord { + const same = record.active === version + const previousAppVersion = same ? record.previousAppVersion : record.activeAppVersion return { schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, active: version, // Why keep the OLD previous when re-activating the same version: a repeated deploy of // an already-active build is not a version change, so it must not erase the rollback // target by naming the active version as its own predecessor. - previous: record.active === version ? record.previous : record.active, + previous: same ? record.previous : record.active, activatedAt: now.toISOString(), - snapshot: record.active === version ? record.snapshot : snapshot + snapshot: same ? record.snapshot : snapshot, + ...(appVersion ? { activeAppVersion: appVersion } : {}), + ...(previousAppVersion ? { previousAppVersion } : {}) } } @@ -150,6 +186,21 @@ export function withRolledBackVersion( previous: null, activatedAt: now.toISOString(), // The snapshot was taken before `active` ran; once restored it has been consumed. + snapshot: null, + ...(record.previousAppVersion ? { activeAppVersion: record.previousAppVersion } : {}), + ...(record.active ? { rolledBackFrom: record.active } : {}) + } +} + +/** The record after decommissioning `active`: nothing serves; the stopped build stays pinned. */ +export function withDeactivatedVersion(record: OrcadActivationRecord): OrcadActivationRecord { + return { + schemaVersion: ORCAD_ACTIVATION_SCHEMA_VERSION, + active: null, + // Kept so GC leaves the slot whose daemon may still own terminals, and a redeploy can find it. + previous: record.active, + activatedAt: null, + // No version is active, so there is nothing a pre-activation snapshot could roll back to. snapshot: null } } diff --git a/src/main/ssh/orcad-activation-recovery.ts b/src/main/ssh/orcad-activation-recovery.ts new file mode 100644 index 00000000000..03dff2fa689 --- /dev/null +++ b/src/main/ssh/orcad-activation-recovery.ts @@ -0,0 +1,135 @@ +/** + * Finishing or undoing an activation, rollback or decommission that a crash, a lost connection or an + * unverifiable failure left fenced. Either way the host ends serving exactly the slot its + * activation record names, or the fence stays for an operator. + */ +import type { ServeReadiness } from '../server/serve-readiness' +import { + planOrcadTransactionRecovery, + type OrcadActivationTransaction +} from './orcad-activation-transaction' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { + orcadActivationFenceExists, + withStaleOrcadActivationRecoveryLock +} from './orcad-activation-lock' +import { RemoteInstallLockBusyError } from './ssh-relay-install-lock' +import { ensureOrcadSlotServing, resolveOrcadSlotIdentity } from './orcad-recovery-slot' +import { reconcileOrcadDecommission } from './orcad-decommission-recovery' +import { + recoverOrcadIncumbent, + type OrcadIncumbentRecoveryOptions +} from './orcad-incumbent-recovery' +import type { OrcadManagedRefusal } from '../../shared/orcad-managed-runtime' +import { errorMessage } from '../../shared/error-message' + +export type OrcadActivationRecoveryResult = + | { outcome: 'none' } + | { outcome: 'pending'; code: string; reason: string } + | { + outcome: 'recovered' + resolution: 'committed' | 'restored-incumbent' + activeVersion: string | null + readiness: ServeReadiness | null + } + | OrcadManagedRefusal + +export type OrcadActivationRecoveryOptions = OrcadIncumbentRecoveryOptions + +export async function recoverInterruptedOrcadActivation( + options: OrcadActivationRecoveryOptions +): Promise { + try { + if ( + !(await readOrcadActivationTransaction(options)) && + !(await orcadActivationFenceExists(options)) + ) { + return { outcome: 'none' } + } + return await withStaleOrcadActivationRecoveryLock(options, async (lock) => { + const transaction = await readOrcadActivationTransaction(options) + if (!transaction) { + // A release cut short after removing the journal; dropping the lock finishes it. + return { outcome: 'none' } + } + const result = await reconcileOrcadTransaction(options, transaction) + if (result.outcome !== 'recovered') { + lock.retain() + } + return result + }) + } catch (error) { + if (error instanceof RemoteInstallLockBusyError) { + return { + outcome: 'pending', + code: 'orcad_recovery_transaction_still_fresh', + reason: + 'The activation fence is held by a run that may still be working. Retry after the ' + + 'install lock recovery window.' + } + } + return { + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_unverifiable', + reason: + `The interrupted activation could not be reconciled safely: ${errorMessage(error)} ` + + 'The host remains fenced.' + } + } +} + +/** Runs under a held fence. Throws when a step is unverifiable. */ +export async function reconcileOrcadTransaction( + options: OrcadActivationRecoveryOptions, + transaction: OrcadActivationTransaction +): Promise { + const plan = planOrcadTransactionRecovery(transaction, await readOrcadActivationRecord(options)) + if (plan.action === 'refuse') { + return { outcome: 'refused', verdict: 'unverifiable', code: plan.code, reason: plan.reason } + } + if ( + plan.action === 'confirm-decommissioned' || + plan.action === 'resume-stop' || + plan.action === 'keep-serving' + ) { + return reconcileOrcadDecommission(options, plan) + } + if (plan.action === 'finish-commit') { + await writeOrcadActivationRecord(options, plan.record) + } + if (plan.action === 'stabilize-committed' || plan.action === 'finish-commit') { + const activeVersion = plan.action === 'finish-commit' ? plan.record.active : plan.activeVersion + if (!activeVersion) { + throw new Error('The committed activation record has no active version.') + } + const identity = await resolveOrcadSlotIdentity(options, activeVersion) + return { + outcome: 'recovered', + resolution: 'committed', + activeVersion, + readiness: await ensureOrcadSlotServing(options, identity) + } + } + const recovery = await recoverOrcadIncumbent(options, { + transactionStartedAt: transaction.startedAt, + launchedVersion: plan.launchedVersion, + incumbent: plan.activeVersion + ? await resolveOrcadSlotIdentity(options, plan.activeVersion) + : null, + restoreState: plan.restoreState, + launchedFromState: plan.launchedFromState + }) + return recovery.outcome === 'refused' + ? recovery + : { + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: plan.activeVersion, + readiness: recovery.readiness + } +} diff --git a/src/main/ssh/orcad-activation-transaction-schema.ts b/src/main/ssh/orcad-activation-transaction-schema.ts new file mode 100644 index 00000000000..14c776b3a66 --- /dev/null +++ b/src/main/ssh/orcad-activation-transaction-schema.ts @@ -0,0 +1,98 @@ +import { z } from 'zod' +import { OrcadManagedStopRequestSchema } from '../../shared/orcad-stop-request' +import { + ORCAD_INSTALL_MODEL, + remoteInstallDirName, + remoteInstallVersionDirRegex +} from './remote-install-model' + +export const ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION = 1 + +const RemoteVersionSchema = z + .string() + .refine( + (version) => + remoteInstallVersionDirRegex(ORCAD_INSTALL_MODEL).test( + remoteInstallDirName(ORCAD_INSTALL_MODEL, version) + ), + 'Expected a safe remote install version' + ) +const SafeSnapshotNameSchema = z + .string() + .min(1) + .max(255) + .regex(/^[A-Za-z0-9][A-Za-z0-9.+-]*$/u) +const SnapshotVerdictSchema = z.object({ + dirName: SafeSnapshotNameSchema, + state: z.enum(['pending', 'captured', 'empty']) +}) +const OrcadActivationTransactionCommonFields = { + schemaVersion: z.literal(ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION), + transactionId: z.uuid(), + startedAt: z.iso.datetime({ offset: true }), + updatedAt: z.iso.datetime({ offset: true }), + recordBefore: z.unknown() +} + +// Why strict operations: an older client reading a newer operation must keep the fence. +export const OrcadActivationTransactionSchema = z + .discriminatedUnion('operation', [ + z.object({ + ...OrcadActivationTransactionCommonFields, + operation: z.literal('activate'), + phase: z.enum(['prepared', 'incumbent-stopped', 'snapshot-captured', 'candidate-ready']), + candidateVersion: RemoteVersionSchema, + recordAfter: z.unknown().nullable(), + snapshot: SnapshotVerdictSchema + }), + z.object({ + ...OrcadActivationTransactionCommonFields, + operation: z.literal('rollback'), + phase: z.enum([ + 'prepared', + 'incumbent-stopped', + 'rescue-captured', + 'rollback-state-restored', + 'target-ready' + ]), + incumbentVersion: RemoteVersionSchema, + targetVersion: RemoteVersionSchema, + recordAfter: z.unknown(), + rescue: SnapshotVerdictSchema + }), + z.object({ + ...OrcadActivationTransactionCommonFields, + operation: z.literal('decommission'), + phase: z.enum(['prepared', 'stop-dispatched', 'process-exited']), + activeVersion: RemoteVersionSchema, + recordAfter: z.unknown(), + /** The instance-bound stop request; durable before it can reach the host. */ + request: OrcadManagedStopRequestSchema.nullable() + }) + ]) + .superRefine((transaction, context) => { + if (transaction.operation === 'decommission') { + if ((transaction.phase === 'prepared') !== (transaction.request === null)) { + context.addIssue({ code: 'custom', message: 'Stop request is inconsistent with phase' }) + } + if (transaction.request && transaction.request.transactionId !== transaction.transactionId) { + context.addIssue({ code: 'custom', message: 'Stop request names another transaction' }) + } + return + } + const beforeVerdict = + transaction.phase === 'prepared' || transaction.phase === 'incumbent-stopped' + const verdict = transaction.operation === 'activate' ? transaction.snapshot : transaction.rescue + if (beforeVerdict && verdict.state !== 'pending') { + context.addIssue({ code: 'custom', message: 'Snapshot state advanced before its phase' }) + } + if (!beforeVerdict && verdict.state === 'pending') { + context.addIssue({ code: 'custom', message: 'Snapshot phase has no durable verdict' }) + } + if ( + transaction.operation === 'activate' && + (transaction.phase === 'candidate-ready') !== (transaction.recordAfter !== null) + ) { + context.addIssue({ code: 'custom', message: 'Committed record is inconsistent with phase' }) + } + }) diff --git a/src/main/ssh/orcad-activation-transaction-store.ts b/src/main/ssh/orcad-activation-transaction-store.ts new file mode 100644 index 00000000000..fd770a2bf9d --- /dev/null +++ b/src/main/ssh/orcad-activation-transaction-store.ts @@ -0,0 +1,61 @@ +import { + ORCAD_ACTIVATION_TRANSACTION_FILENAME, + parseOrcadActivationTransaction, + serializeOrcadActivationTransaction, + type OrcadActivationTransaction +} from './orcad-activation-transaction' +import { orcadActivationTransactionRoot } from './orcad-activation-lock' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' +import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { currentOrcadFence } from './orcad-activation-fence-scope' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +const ORCAD_ACTIVATION_TRANSACTION_MAX_BYTES = 64 * 1024 + +type OrcadActivationTransactionStoreOptions = OrcadRemoteExecTarget & { remoteHome: string } + +export function orcadActivationTransactionPath( + host: RemoteHostPlatform, + remoteHome: string +): string { + return joinRemotePath( + host, + orcadActivationTransactionRoot(host, remoteHome), + ORCAD_ACTIVATION_TRANSACTION_FILENAME + ) +} + +/** `null` only on a verified absence; an unreadable journal keeps the host fenced. */ +export async function readOrcadActivationTransaction( + options: OrcadActivationTransactionStoreOptions +): Promise { + const read = await readBoundedOrcadRemoteRecord( + options, + orcadActivationTransactionPath(options.host, options.remoteHome), + ORCAD_ACTIVATION_TRANSACTION_MAX_BYTES + ) + const parsed = parseOrcadActivationTransaction(read.state === 'present' ? read.raw : null) + if (parsed.state === 'unreadable') { + throw new Error(`Cannot read this host's orcad activation transaction: ${parsed.reason}`) + } + return parsed.state === 'ok' ? parsed.transaction : null +} + +/** + * Stamped with the writing run's fence token, so a release only ever removes its own generation's + * journal (Astra pass 9); readers ignore the extra field. + */ +export function writeOrcadActivationTransaction( + options: OrcadActivationTransactionStoreOptions, + transaction: OrcadActivationTransaction +): Promise { + const fenceToken = currentOrcadFence()?.token + return writeAtomicOrcadRemoteRecord( + options, + orcadActivationTransactionPath(options.host, options.remoteHome), + serializeOrcadActivationTransaction(fenceToken ? { ...transaction, fenceToken } : transaction) + ) +} diff --git a/src/main/ssh/orcad-activation-transaction-transitions.ts b/src/main/ssh/orcad-activation-transaction-transitions.ts new file mode 100644 index 00000000000..95facd02daa --- /dev/null +++ b/src/main/ssh/orcad-activation-transaction-transitions.ts @@ -0,0 +1,102 @@ +import type { OrcadActivationRecord } from './orcad-activation-record' +import { ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION } from './orcad-activation-transaction-schema' +import type { + OrcadActivateTransaction, + OrcadRollbackTransaction +} from './orcad-activation-transaction' + +export function createOrcadActivationTransaction(options: { + transactionId: string + candidateVersion: string + recordBefore: OrcadActivationRecord + snapshotDirName: string + now: Date +}): OrcadActivateTransaction { + const timestamp = options.now.toISOString() + return { + schemaVersion: ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION, + transactionId: options.transactionId, + operation: 'activate', + phase: 'prepared', + startedAt: timestamp, + updatedAt: timestamp, + candidateVersion: options.candidateVersion, + recordBefore: options.recordBefore, + recordAfter: null, + snapshot: { dirName: options.snapshotDirName, state: 'pending' } + } +} + +export function withOrcadActivationIncumbentStopped( + transaction: OrcadActivateTransaction, + now: Date +): OrcadActivateTransaction { + return { ...transaction, phase: 'incumbent-stopped', updatedAt: now.toISOString() } +} + +export function withOrcadActivationSnapshot( + transaction: OrcadActivateTransaction, + state: 'captured' | 'empty', + now: Date +): OrcadActivateTransaction { + return { + ...transaction, + phase: 'snapshot-captured', + updatedAt: now.toISOString(), + snapshot: { dirName: transaction.snapshot.dirName, state } + } +} + +export function withOrcadActivationCandidateReady( + transaction: OrcadActivateTransaction, + recordAfter: OrcadActivationRecord, + now: Date +): OrcadActivateTransaction { + return { ...transaction, phase: 'candidate-ready', updatedAt: now.toISOString(), recordAfter } +} + +export function createOrcadRollbackTransaction(options: { + transactionId: string + incumbentVersion: string + targetVersion: string + recordBefore: OrcadActivationRecord + recordAfter: OrcadActivationRecord + rescueDirName: string + now: Date +}): OrcadRollbackTransaction { + const timestamp = options.now.toISOString() + return { + schemaVersion: ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION, + transactionId: options.transactionId, + operation: 'rollback', + phase: 'prepared', + startedAt: timestamp, + updatedAt: timestamp, + incumbentVersion: options.incumbentVersion, + targetVersion: options.targetVersion, + recordBefore: options.recordBefore, + recordAfter: options.recordAfter, + rescue: { dirName: options.rescueDirName, state: 'pending' } + } +} + +export function withOrcadRollbackPhase( + transaction: OrcadRollbackTransaction, + phase: 'incumbent-stopped' | 'rollback-state-restored' | 'target-ready', + now: Date +): OrcadRollbackTransaction { + return { ...transaction, phase, updatedAt: now.toISOString() } +} + +export function withOrcadRollbackRescue( + transaction: OrcadRollbackTransaction, + state: 'captured' | 'empty', + now: Date +): OrcadRollbackTransaction { + return { + ...transaction, + phase: 'rescue-captured', + updatedAt: now.toISOString(), + rescue: { dirName: transaction.rescue.dirName, state } + } +} diff --git a/src/main/ssh/orcad-activation-transaction.test.ts b/src/main/ssh/orcad-activation-transaction.test.ts new file mode 100644 index 00000000000..b8a5d8c5a0e --- /dev/null +++ b/src/main/ssh/orcad-activation-transaction.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn().mockResolvedValue('') +})) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock, RemoteInstallLockBusyError } from './ssh-relay-install-lock' +import { + parseOrcadActivationTransaction, + planOrcadTransactionRecovery, + serializeOrcadActivationTransaction, + type OrcadActivationTransaction +} from './orcad-activation-transaction' +import { + createOrcadActivationTransaction, + createOrcadRollbackTransaction, + withOrcadActivationCandidateReady, + withOrcadActivationIncumbentStopped, + withOrcadActivationSnapshot, + withOrcadRollbackPhase, + withOrcadRollbackRescue +} from './orcad-activation-transaction-transitions' +import { + resolveOrcadActivationReadinessTimeout, + withOrcadActivationLock, + withStaleOrcadActivationRecoveryLock +} from './orcad-activation-lock' +import { FakeOrcadHost, NEW, OLD } from './orcad-activation-host-test-harness' +import { withRolledBackVersion } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +const T = new Date('2026-02-02T00:00:00.000Z') +const ID = '7f1c2a7e-6c1b-4a8e-9f0e-0a1b2c3d4e5f' +const before = FakeOrcadHost.newRecord() + +function activation(): ReturnType { + return createOrcadActivationTransaction({ + transactionId: ID, + candidateVersion: '0.3.0+cc01', + recordBefore: before, + snapshotDirName: 'pre-0.3.0+cc01-1', + now: T + }) +} + +function rollback(): ReturnType { + return createOrcadRollbackTransaction({ + transactionId: ID, + incumbentVersion: NEW, + targetVersion: OLD, + recordBefore: before, + recordAfter: withRolledBackVersion(before, T), + rescueDirName: 'rollback-rescue-0.2.0+bb01-1', + now: T + }) +} + +const roundTrip = (transaction: OrcadActivationTransaction): unknown => + parseOrcadActivationTransaction(serializeOrcadActivationTransaction(transaction)) + +describe('parseOrcadActivationTransaction', () => { + it('round-trips every phase of both operations', () => { + const stopped = withOrcadActivationIncumbentStopped(activation(), T) + const captured = withOrcadActivationSnapshot(stopped, 'captured', T) + const ready = withOrcadActivationCandidateReady( + captured, + { ...before, active: '0.3.0+cc01' }, + T + ) + const rescued = withOrcadRollbackRescue( + withOrcadRollbackPhase(rollback(), 'incumbent-stopped', T), + 'empty', + T + ) + for (const transaction of [activation(), stopped, captured, ready, rollback(), rescued]) { + expect(roundTrip(transaction)).toEqual({ state: 'ok', transaction }) + } + }) + + it.each([ + ['an unknown operation, so an older client keeps a newer fence', { operation: 'decommission' }], + ['a snapshot verdict before its phase', { snapshot: { dirName: 'pre-x', state: 'captured' } }], + ['a committed record before candidate-ready', { recordAfter: before }], + ['an unsafe version', { candidateVersion: '../../etc' }], + ['an unsafe snapshot name', { snapshot: { dirName: '../x', state: 'pending' } }] + ])('reads %s as unreadable', (_name, patch) => { + const raw = JSON.stringify({ ...activation(), ...patch }) + expect(parseOrcadActivationTransaction(raw)).toMatchObject({ state: 'unreadable' }) + }) + + it('rejects a rollback whose versions disagree with its records', () => { + const raw = JSON.stringify({ ...rollback(), targetVersion: '0.0.9+dd01' }) + expect(parseOrcadActivationTransaction(raw)).toMatchObject({ state: 'unreadable' }) + }) +}) + +describe('planOrcadTransactionRecovery', () => { + const after = { ...before, active: '0.3.0+cc01' } + const captured = withOrcadActivationSnapshot( + withOrcadActivationIncumbentStopped(activation(), T), + 'captured', + T + ) + + it.each([ + ['prepared', activation(), null, null], + ['incumbent-stopped', withOrcadActivationIncumbentStopped(activation(), T), null, null], + ['snapshot-captured', captured, '0.3.0+cc01', 'captured'] + ] as const)('undoes an activation interrupted at %s', (_phase, transaction, launched, state) => { + expect(planOrcadTransactionRecovery(transaction, before)).toMatchObject({ + action: 'undo', + launchedVersion: launched, + activeVersion: NEW, + restoreState: state === null ? null : { state } + }) + }) + + it('finishes a commit whose record write was lost, and stabilizes one that landed', () => { + const ready = withOrcadActivationCandidateReady(captured, after, T) + expect(planOrcadTransactionRecovery(ready, before)).toEqual({ + action: 'finish-commit', + record: after + }) + expect(planOrcadTransactionRecovery(ready, after)).toEqual({ + action: 'stabilize-committed', + activeVersion: '0.3.0+cc01' + }) + }) + + it('restores the rescue once a rollback may have replaced the state', () => { + const rescued = withOrcadRollbackRescue(rollback(), 'captured', T) + expect(planOrcadTransactionRecovery(rescued, before)).toMatchObject({ + action: 'undo', + launchedVersion: null, + activeVersion: NEW, + restoreState: { state: 'captured' } + }) + expect( + planOrcadTransactionRecovery( + withOrcadRollbackPhase(rescued, 'rollback-state-restored', T), + before + ) + ).toMatchObject({ + launchedVersion: OLD, + // The target started from the restored snapshot; whether it changed that decides. + launchedFromState: { state: 'captured', dirName: before.snapshot?.dirName } + }) + }) + + it('refuses when the record matches neither side', () => { + expect(planOrcadTransactionRecovery(activation(), { ...before, previous: null })).toMatchObject( + { + action: 'refuse', + code: 'orcad_recovery_activation_record_changed' + } + ) + }) +}) + +describe('activation fence', () => { + const target = { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u' + } + const removals = (): string[] => + vi + .mocked(execCommand) + .mock.calls.map(([, command]) => command) + .filter((command) => command.includes('rm -')) + + const orphanings = (): string[] => + vi + .mocked(execCommand) + .mock.calls.map(([, command]) => command) + .filter((command) => command.includes('touch -m -t 200001010000')) + + const locked = (run: Parameters>[1]): Promise => + withOrcadActivationLock(target, run, () => { + throw new Error('fence held') + }) + + it('answers a fence still held after a short wait, running nothing', async () => { + vi.clearAllMocks() + vi.mocked(acquireInstallLock).mockRejectedValueOnce(new RemoteInstallLockBusyError('/l', 0)) + const run = vi.fn(async () => 'ran') + await expect(withOrcadActivationLock(target, run, () => 'held')).resolves.toBe('held') + expect(vi.mocked(acquireInstallLock).mock.calls[0]?.[3]).toMatchObject({ waitTimeoutMs: 5_000 }) + expect(run).not.toHaveBeenCalled() + expect(removals()).toEqual([]) + }) + + it('never takes over a held fence by age, and removes the journal before the lock', async () => { + vi.clearAllMocks() + await locked(async () => undefined) + expect(vi.mocked(acquireInstallLock).mock.calls[0]?.[3]).toMatchObject({ + allowStaleTakeover: false, + relayGcClaim: false + }) + const [release] = removals() + // Only while the lock still carries this run's token; the journal goes before the lock. + expect(release).toMatch( + /^\[ "\$\(cat '[^']*\.orca-fence-owner' 2>\/dev\/null\)" = '[^']+' \] \|\| \{ echo SUPERSEDED; exit 0; \};/u + ) + expect(release?.indexOf('transaction.json')).toBeLessThan( + release?.indexOf("mv '/home/u/.orca-remote/.orcad-activation-transaction/.install-lock'") ?? + -1 + ) + }) + + it('keeps the fence after an unconfirmed termination, a retained error, or retain()', async () => { + vi.clearAllMocks() + const lost = Object.assign(new Error('lost'), { sshChannelCloseConfirmed: false }) + await expect(locked(() => Promise.reject(lost))).rejects.toBe(lost) + await expect( + locked(async (lock) => { + lock.retainOnError() + throw new Error('mid-transaction') + }) + ).rejects.toThrow('mid-transaction') + await locked(async (lock) => lock.retain()) + expect(removals()).toEqual([]) + // A finished run's fence is ownerless; one an unconfirmed command may still use stays fresh. + expect(orphanings()).toHaveLength(2) + }) + + it('marks a fence recovery retained as ownerless, so the next recovery need not wait', async () => { + vi.clearAllMocks() + await withStaleOrcadActivationRecoveryLock(target, async (lock) => lock.retain()) + await expect( + withStaleOrcadActivationRecoveryLock(target, () => Promise.reject(new Error('refused'))) + ).rejects.toThrow('refused') + expect(orphanings()).toHaveLength(2) + expect(removals()).toEqual([]) + }) + + it('releases after a recovered failure even though the run throws', async () => { + vi.clearAllMocks() + await expect( + locked(async (lock) => { + lock.retainOnError() + lock.recovered() + throw new Error('snapshot failed; incumbent restarted') + }) + ).rejects.toThrow('incumbent restarted') + expect(removals()).toHaveLength(1) + }) + + it('lets recovery take over only stale fences, without waiting', async () => { + vi.clearAllMocks() + await withStaleOrcadActivationRecoveryLock(target, async () => undefined) + expect(vi.mocked(acquireInstallLock).mock.calls[0]?.[3]).toMatchObject({ + allowStaleTakeover: true, + waitTimeoutMs: 0 + }) + }) + + it.each([0, -1, 5 * 60_000 + 1, 1.5])('rejects readiness timeout %s', (timeout) => { + expect(() => resolveOrcadActivationReadinessTimeout(timeout, 1)).toThrow() + }) +}) diff --git a/src/main/ssh/orcad-activation-transaction.ts b/src/main/ssh/orcad-activation-transaction.ts new file mode 100644 index 00000000000..405d0b440cf --- /dev/null +++ b/src/main/ssh/orcad-activation-transaction.ts @@ -0,0 +1,271 @@ +/** + * The host-side journal that makes an orcad activation, rollback or decommission crash-safe. + * + * Written before the first mutation and kept until the host is proven to serve exactly one + * slot again. The activation record stays the commit point: a journal whose `recordAfter` + * matches the record committed; one whose `recordBefore` matches did not, and recovery puts + * the pre-transaction slot and state back. Anything else keeps the fence for an operator. + */ +import { + parseOrcadActivationRecord, + coreOrcadActivationRecord, + serializeOrcadActivationRecord, + type OrcadActivationRecord +} from './orcad-activation-record' +import { + type ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION, + OrcadActivationTransactionSchema +} from './orcad-activation-transaction-schema' +import { + orcadDecommissionTransactionDefect, + planOrcadDecommissionRecovery, + type OrcadDecommissionRecoveryPlan, + type OrcadDecommissionTransaction +} from './orcad-decommission-transaction' +import { errorMessage } from '../../shared/error-message' + +export const ORCAD_ACTIVATION_TRANSACTION_FILENAME = 'transaction.json' +export const ORCAD_ACTIVATION_TRANSACTION_DIRNAME = '.orcad-activation-transaction' + +export type OrcadSnapshotVerdict = { dirName: string; state: 'pending' | 'captured' | 'empty' } + +export type OrcadActivateTransaction = { + schemaVersion: typeof ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION + transactionId: string + operation: 'activate' + phase: 'prepared' | 'incumbent-stopped' | 'snapshot-captured' | 'candidate-ready' + startedAt: string + updatedAt: string + candidateVersion: string + recordBefore: OrcadActivationRecord + recordAfter: OrcadActivationRecord | null + snapshot: OrcadSnapshotVerdict +} + +export type OrcadRollbackTransaction = { + schemaVersion: typeof ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION + transactionId: string + operation: 'rollback' + phase: + | 'prepared' + | 'incumbent-stopped' + | 'rescue-captured' + | 'rollback-state-restored' + | 'target-ready' + startedAt: string + updatedAt: string + incumbentVersion: string + targetVersion: string + recordBefore: OrcadActivationRecord + recordAfter: OrcadActivationRecord + rescue: OrcadSnapshotVerdict +} + +export type OrcadActivationTransaction = + | OrcadActivateTransaction + | OrcadRollbackTransaction + | OrcadDecommissionTransaction + +export type OrcadActivationTransactionReadResult = + | { state: 'absent' } + | { state: 'ok'; transaction: OrcadActivationTransaction } + | { state: 'unreadable'; reason: string } + +/** What recovery must do; `undo` lists the state to put back once the new slot is quiescent. */ +export type OrcadTransactionRecoveryPlan = + | { action: 'stabilize-committed'; activeVersion: string | null } + /** The new slot passed its gate and the journal holds the record; only the write was lost. */ + | { action: 'finish-commit'; record: OrcadActivationRecord } + | { + action: 'undo' + /** The slot that may have run after the state changed hands. */ + launchedVersion: string | null + activeVersion: string | null + restoreState: OrcadSnapshotVerdict | null + launchedFromState?: OrcadSnapshotVerdict | null + } + | OrcadDecommissionRecoveryPlan + | { action: 'refuse'; code: string; reason: string } + +export function parseOrcadActivationTransaction( + raw: string | null +): OrcadActivationTransactionReadResult { + if (raw === null || raw.trim() === '') { + return { state: 'absent' } + } + let json: unknown + try { + json = JSON.parse(raw) + } catch (error) { + return unreadable(`transaction is not JSON: ${errorMessage(error)}`) + } + const parsed = OrcadActivationTransactionSchema.safeParse(json) + if (!parsed.success) { + const issue = parsed.error.issues[0] + const path = issue?.path.length ? issue.path.join('.') : 'transaction' + return unreadable(`${path} is invalid: ${issue?.message ?? 'unknown shape'}`) + } + const recordBefore = parseNestedRecord(parsed.data.recordBefore, 'recordBefore') + if (recordBefore.state === 'unreadable') { + return recordBefore + } + if (parsed.data.operation === 'decommission') { + const after = parseNestedRecord(parsed.data.recordAfter, 'recordAfter') + if (after.state === 'unreadable') { + return after + } + const transaction = { + ...parsed.data, + recordBefore: recordBefore.record, + recordAfter: after.record + } + const defect = orcadDecommissionTransactionDefect(transaction) + return defect ? unreadable(defect) : { state: 'ok', transaction } + } + if (parsed.data.operation === 'activate') { + const recordAfter = + parsed.data.recordAfter === null + ? null + : parseNestedRecord(parsed.data.recordAfter, 'recordAfter') + if (recordAfter?.state === 'unreadable') { + return recordAfter + } + if (recordAfter && recordAfter.record.active !== parsed.data.candidateVersion) { + return unreadable('recordAfter does not activate candidateVersion') + } + return { + state: 'ok', + transaction: { + ...parsed.data, + recordBefore: recordBefore.record, + recordAfter: recordAfter?.record ?? null + } + } + } + const recordAfter = parseNestedRecord(parsed.data.recordAfter, 'recordAfter') + if (recordAfter.state === 'unreadable') { + return recordAfter + } + if ( + recordBefore.record.active !== parsed.data.incumbentVersion || + recordBefore.record.previous !== parsed.data.targetVersion + ) { + return unreadable('rollback versions do not match recordBefore') + } + if ( + recordAfter.record.active !== parsed.data.targetVersion || + recordAfter.record.previous !== null || + recordAfter.record.snapshot !== null + ) { + return unreadable('recordAfter is not a completed rollback record') + } + return { + state: 'ok', + transaction: { + ...parsed.data, + recordBefore: recordBefore.record, + recordAfter: recordAfter.record + } + } +} + +export function serializeOrcadActivationTransaction( + transaction: OrcadActivationTransaction & { fenceToken?: string } +): string { + return `${JSON.stringify(transaction, null, 2)}\n` +} + +export function planOrcadTransactionRecovery( + transaction: OrcadActivationTransaction, + currentRecord: OrcadActivationRecord +): OrcadTransactionRecoveryPlan { + if (transaction.operation === 'decommission') { + const committed = sameOrcadActivationRecord(currentRecord, transaction.recordAfter) + return committed || sameOrcadActivationRecord(currentRecord, transaction.recordBefore) + ? planOrcadDecommissionRecovery(transaction, committed) + : recordChangedRefusal(transaction) + } + if ( + transaction.recordAfter && + sameOrcadActivationRecord(currentRecord, transaction.recordAfter) + ) { + return { action: 'stabilize-committed', activeVersion: transaction.recordAfter.active } + } + if (!sameOrcadActivationRecord(currentRecord, transaction.recordBefore)) { + return recordChangedRefusal(transaction) + } + if (transaction.phase === 'candidate-ready' || transaction.phase === 'target-ready') { + return { action: 'finish-commit', record: transaction.recordAfter ?? neverRecord() } + } + if (transaction.operation === 'activate') { + // The candidate launches only after the snapshot verdict is durable. + const launched = transaction.phase === 'snapshot-captured' + return { + action: 'undo', + launchedVersion: launched ? transaction.candidateVersion : null, + activeVersion: transaction.recordBefore.active, + restoreState: launched ? transaction.snapshot : null + } + } + // The rescue is the incumbent's state; it only needs restoring once the target's replaced it. + const replaced = transaction.phase === 'rollback-state-restored' + const rescued = replaced || transaction.phase === 'rescue-captured' + return { + action: 'undo', + launchedVersion: replaced ? transaction.targetVersion : null, + activeVersion: transaction.incumbentVersion, + // A crash mid-restore leaves the phase at rescue-captured with the root half replaced. + restoreState: rescued ? transaction.rescue : null, + launchedFromState: replaced ? rollbackStartingState(transaction) : null + } +} + +/** The pre-activation snapshot a rollback restored before launching its target. */ +export function rollbackStartingState( + transaction: Pick +): OrcadSnapshotVerdict | null { + const snapshot = transaction.recordBefore.snapshot + return snapshot ? { dirName: snapshot.dirName, state: 'captured' } : null +} + +export function sameOrcadActivationRecord( + left: OrcadActivationRecord, + right: OrcadActivationRecord +): boolean { + return ( + serializeOrcadActivationRecord(coreOrcadActivationRecord(left)) === + serializeOrcadActivationRecord(coreOrcadActivationRecord(right)) + ) +} + +function parseNestedRecord( + value: unknown, + field: string +): { state: 'ok'; record: OrcadActivationRecord } | { state: 'unreadable'; reason: string } { + const parsed = parseOrcadActivationRecord(JSON.stringify(value)) + return parsed.state === 'ok' + ? { state: 'ok', record: parsed.record } + : unreadable( + `${field} is invalid: ${parsed.state === 'absent' ? 'record is absent' : parsed.reason}` + ) +} + +function recordChangedRefusal( + transaction: OrcadActivationTransaction +): Extract { + return { + action: 'refuse', + code: 'orcad_recovery_activation_record_changed', + reason: + `The activation record matches neither side of the interrupted ${transaction.operation}. ` + + 'Preserving the activation fence for operator inspection.' + } +} + +function neverRecord(): never { + throw new Error('A committed phase must carry its record; the schema enforces this.') +} + +function unreadable(reason: string): { state: 'unreadable'; reason: string } { + return { state: 'unreadable', reason } +} diff --git a/src/main/ssh/orcad-active-readiness.ts b/src/main/ssh/orcad-active-readiness.ts new file mode 100644 index 00000000000..fc5858c85a8 --- /dev/null +++ b/src/main/ssh/orcad-active-readiness.ts @@ -0,0 +1,129 @@ +/** Proving that the orcad an activation record names is the one actually serving. */ +import { evaluateOrcadActivation, type OrcadActivationExpectation } from './orcad-activation-gate' +import { + orcadLivenessProbeCommand, + parseOrcadLiveness, + type OrcadLaunchSpec, + type OrcadReadinessParse +} from './orcad-remote-launch' +import { + execOrcadRemote, + launchOrcadAndAwaitReadiness, + type OrcadRemoteExecTarget +} from './orcad-remote-runtime-control' +import { + parseOrcadReadinessWaitOutput, + readOrcadReadinessNowCommand +} from './orcad-remote-readiness-wait' +import type { ServeReadiness } from '../server/serve-readiness' +import { withOrcadLogTail } from './orcad-remote-log-tail' + +/** `exited` is proven absence; `unverifiable` means the host could not say, which is not death. */ +export type OrcadReadinessFailureVerdict = 'exited' | 'unverifiable' | 'rejected' + +export class OrcadActiveReadinessError extends Error { + constructor( + readonly verdict: OrcadReadinessFailureVerdict, + message: string + ) { + super(message) + this.name = 'OrcadActiveReadinessError' + } +} + +function gatedReadiness( + parsed: OrcadReadinessParse, + expectation: OrcadActivationExpectation, + label: string +): ServeReadiness { + const readiness = parsed.state === 'ready' ? parsed.readiness : null + const verdict = evaluateOrcadActivation(readiness, expectation) + if (verdict.decision === 'reject') { + throw new OrcadActiveReadinessError( + 'rejected', + `${label} failed its readiness check: ${verdict.reason}` + ) + } + if (!readiness) { + throw new OrcadActiveReadinessError( + 'rejected', + `${label} passed activation without a readiness payload.` + ) + } + const coverage = orcadDaemonCoverageRefusal(readiness) + if (coverage) { + throw new OrcadActiveReadinessError('rejected', `${label} ${coverage}`) + } + return readiness +} + +/** + * A slot proves terminals only when its daemon's self-test spawned a PTY, or completed the + * handshake on a platform whose daemon never spawn-probes. A build that predates the coverage + * field keeps the identity gate alone, so an older slot is not stranded. + */ +export function orcadDaemonCoverageRefusal(readiness: ServeReadiness): string | null { + const health = readiness.health + const coverage = health?.terminalDaemon?.selfTest?.coverage + if (!health || coverage === undefined || coverage === 'pty-spawn') { + return null + } + if (coverage === 'handshake' && health.platform === 'win32') { + return null + } + return ( + `reported terminal-daemon coverage '${String(coverage)}', which does not prove a PTY can ` + + `be created on ${health.platform}.` + ) +} + +/** Checks a recorded-active slot without starting anything. */ +export async function probeActiveOrcadReadiness( + target: OrcadRemoteExecTarget & { remoteInstallDir: string }, + expectation: OrcadActivationExpectation +): Promise { + const liveness = parseOrcadLiveness( + await execOrcadRemote(target, orcadLivenessProbeCommand(target.host, target.remoteInstallDir)) + ) + if (liveness === 'DEAD') { + throw new OrcadActiveReadinessError( + 'exited', + `orcad ${expectation.fullVersion} is recorded active but its process has exited.` + ) + } + if (liveness !== 'LIVE') { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${expectation.fullVersion} process state is unverifiable.` + ) + } + const parsed = parseOrcadReadinessWaitOutput( + target.host, + await execOrcadRemote( + target, + readOrcadReadinessNowCommand(target.host, target.remoteInstallDir) + ) + ) + return gatedReadiness(parsed, expectation, 'The active orcad') +} + +/** Starts a slot (recovery or rollback) and accepts it only if it proves the expected build. */ +export async function launchOrcadSlotAndAwaitReadiness( + target: OrcadRemoteExecTarget & { + readinessTimeoutMs?: number + sleep?: (ms: number) => Promise + }, + spec: OrcadLaunchSpec, + expectation: OrcadActivationExpectation +): Promise { + const parsed = await launchOrcadAndAwaitReadiness(target, spec) + try { + return gatedReadiness(parsed, expectation, `orcad ${spec.fullVersion}`) + } catch (error) { + if (!(error instanceof OrcadActiveReadinessError)) { + throw error + } + const message = await withOrcadLogTail(target, spec.remoteInstallDir, error.message) + throw new OrcadActiveReadinessError(error.verdict, message) + } +} diff --git a/src/main/ssh/orcad-artifact-materializer.test.ts b/src/main/ssh/orcad-artifact-materializer.test.ts index 4dd1998e09c..301d023b668 100644 --- a/src/main/ssh/orcad-artifact-materializer.test.ts +++ b/src/main/ssh/orcad-artifact-materializer.test.ts @@ -7,11 +7,13 @@ import { renameSync, rmSync, statSync, + utimesSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { basename, dirname, join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' +import { pruneOrcadArtifactCache } from '../orcad/orcad-artifact-cache-retention' import { z } from 'zod' import { NODE_RUNTIME_ASSETS, @@ -133,6 +135,29 @@ function rewriteManifest( } describe('assembleOrcadArtifact', () => { + it('reuses the same version from cache after retention evicts other versions', async () => { + const fixture = createTemplate() + const first = await assembleOrcadArtifact({ ...fixture, target: TARGET }) + const entry = statSync(join(first, 'orcad.js')) + // Older versions of the same target, used before this one. + const older = [1, 2, 3].map((age) => { + const dir = join(fixture.cacheRoot, TARGET, `0.0.${age}+old`) + write(join(dir, 'orcad.js'), 'old') + const at = new Date(Date.now() - age * 60_000) + utimesSync(dir, at, at) + return dir + }) + const removed = await pruneOrcadArtifactCache(fixture.cacheRoot, { + inUseVersions: new Set([basename(first)]) + }) + + expect(removed).toEqual([older[2]]) + const second = await assembleOrcadArtifact({ ...fixture, target: TARGET }) + expect(second).toBe(first) + // A verified hit: the same files, not a re-copy. + expect(statSync(join(second, 'orcad.js')).ino).toBe(entry.ino) + }) + it.skipIf(process.platform === 'win32')( 'restores executable modes from a template copied without them', async () => { @@ -163,6 +188,10 @@ describe('assembleOrcadArtifact', () => { expect( readFileSync(join(artifactDir, 'node_modules/node-pty/build/Release/pty.node'), 'utf8') ).toBe(`${target}:node_modules/node-pty/build/Release/pty.node`) + // The host-side preflight hashes it the same way, so managed orcad can run it. + expect(await readOrcadArtifactIdentity(artifactDir)).toBe( + readFileSync(join(artifactDir, ORCAD_VERSION_FILENAME), 'utf8').trim() + ) }) it('refuses a compat slot that names the default runtime', async () => { diff --git a/src/main/ssh/orcad-artifact-materializer.ts b/src/main/ssh/orcad-artifact-materializer.ts index 5ae3f0985ed..f9b95c38c2b 100644 --- a/src/main/ssh/orcad-artifact-materializer.ts +++ b/src/main/ssh/orcad-artifact-materializer.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from 'node:crypto' import { createReadStream, existsSync } from 'node:fs' -import { chmod, copyFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { chmod, copyFile, mkdir, readFile, rename, rm, utimes, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { z } from 'zod' import { getAppEnvironment } from '../../shared/app-environment' @@ -23,6 +23,7 @@ import { verifyFileSha256, type PinnedRuntimeMaterializeOptions } from './pinned-runtime-materializer' +import { OrcadArtifactsUnavailableError, OrcadHostUnsupportedError } from './orcad-host-unavailable' const Sha256Schema = z.string().regex(/^[a-f0-9]{64}$/u) const TemplateTargetSchema = z @@ -49,6 +50,12 @@ type MaterializeOptions = PinnedRuntimeMaterializeOptions & { } const materializations = new Map>() +// Slots handed out this session: a deploy may still be reading one, so cache retention keeps them. +const materializedVersions = new Set() + +export function materializedOrcadArtifactVersions(): ReadonlySet { + return materializedVersions +} /** A verified slot directory for `target`; its runtime is referenced, not included (design D2). */ export async function materializeOrcadArtifact( @@ -88,7 +95,11 @@ export async function assembleOrcadArtifact(args: { (path) => isCompleteArtifact(path, fullVersion, sources, sourceHashes) ) const targetDir = cached.path + materializedVersions.add(fullVersion) if (cached.verified) { + // Retention evicts by recency, so a reused slot counts as recently used. + const now = new Date() + await utimes(targetDir, now, now).catch(() => undefined) return targetDir } await mkdir(targetRoot, { recursive: true }) @@ -133,7 +144,7 @@ function artifactSources( const targetDir = join(templateDir, ORCAD_TEMPLATE_TARGETS_DIR, target) const targetManifest = manifest.targets[target] if (!targetManifest) { - throw new Error(`Packaged orcad template does not support ${target}`) + throw new OrcadHostUnsupportedError(`Packaged orcad template does not support ${target}`) } const targetFiles = new Set(orcadTemplateTargetFilenames(target)) const required = orcadArtifactFilenames(target).map((filename) => ({ @@ -209,7 +220,7 @@ async function verifyTemplate( ): Promise { const targetManifest = manifest.targets[target] if (!targetManifest) { - throw new Error(`Packaged orcad template does not support ${target}`) + throw new OrcadHostUnsupportedError(`Packaged orcad template does not support ${target}`) } for (const filename of orcadTemplateCommonFilenames()) { const expected = manifest.commonSha256[filename] @@ -262,10 +273,15 @@ export function getOrcadTemplateCandidates(): string[] { return [...new Set(candidates)] } +/** Whether this build carries an orcad template at all; dev builds usually don't. */ +export function hasOrcadTemplate(): boolean { + return getOrcadTemplateCandidates().some((candidate) => existsSync(candidate)) +} + function resolveOrcadTemplateDir(): string { const found = getOrcadTemplateCandidates().find((candidate) => existsSync(candidate)) if (!found) { - throw new Error('The packaged orcad deployment template is missing') + throw new OrcadArtifactsUnavailableError('The packaged orcad deployment template is missing') } return found } diff --git a/src/main/ssh/orcad-candidate-launch-verdict.ts b/src/main/ssh/orcad-candidate-launch-verdict.ts new file mode 100644 index 00000000000..f99cc8ff8fb --- /dev/null +++ b/src/main/ssh/orcad-candidate-launch-verdict.ts @@ -0,0 +1,50 @@ +import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' +import { orcadActivationTransactionRoot } from './orcad-activation-lock' +import type { OrcadReadinessParse } from './orcad-remote-launch' +import { + launchOrcadAndAwaitReadiness, + type OrcadRemoteExecTarget +} from './orcad-remote-runtime-control' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' + +type CandidateLaunchTarget = Parameters[0] & + OrcadRemoteExecTarget & { + remoteHome: string + nodePath: string + userDataDir: string + bindHost: string + port: number + } + +/** + * Starts one slot and judges its readiness. An unconfirmed SSH termination rethrows: it says + * nothing about the candidate, so it must never read as a rejected one. + */ +export async function launchAndJudgeOrcadSlot( + options: CandidateLaunchTarget, + slot: { remoteInstallDir: string; fullVersion: string; buildHash: string } +): Promise<{ verdict: OrcadActivationVerdict; launchError: unknown }> { + let parsed: OrcadReadinessParse | null = null + let launchError: unknown + try { + parsed = await launchOrcadAndAwaitReadiness(options, { + remoteInstallDir: slot.remoteInstallDir, + nodePath: options.nodePath, + fullVersion: slot.fullVersion, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port, + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) + }) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + launchError = error + } + const verdict = evaluateOrcadActivation(parsed?.state === 'ready' ? parsed.readiness : null, { + buildHash: slot.buildHash, + fullVersion: slot.fullVersion + }) + return { verdict, launchError } +} diff --git a/src/main/ssh/orcad-catalog-durable-mutation.ts b/src/main/ssh/orcad-catalog-durable-mutation.ts new file mode 100644 index 00000000000..f392fc74c99 --- /dev/null +++ b/src/main/ssh/orcad-catalog-durable-mutation.ts @@ -0,0 +1,23 @@ +import type { OrcadMigrationCatalogState } from '../../shared/orcad-migration-manifest' + +export async function resolveDurableOrcadCatalogMutation( + mutate: () => Promise, + read: () => Promise, + accepted: (state: OrcadMigrationCatalogState) => boolean +) { + try { + return await mutate() + } catch (mutationError) { + let observed: OrcadMigrationCatalogState + try { + observed = await read() + } catch { + throw mutationError + } + if (!accepted(observed)) { + throw mutationError + } + } + // State reads may reflect unflushed mutations; an idempotent acknowledgment proves durability. + return mutate() +} diff --git a/src/main/ssh/orcad-conversion-abandon.ts b/src/main/ssh/orcad-conversion-abandon.ts new file mode 100644 index 00000000000..f7d64d0ede6 --- /dev/null +++ b/src/main/ssh/orcad-conversion-abandon.ts @@ -0,0 +1,49 @@ +/** + * Gives a host whose conversion stopped short back to the relay. A destination that was never + * registered held nothing; a registered one must prove through an abort that it holds nothing, + * and is unregistered before the fence goes so no later start re-fences the host to it. + */ +import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { closeOrcadManagedTunnel } from './orcad-managed-tunnel' +import { + abortOrcadMigrationCutover, + type OrcadMigrationAbortResult, + type OrcadMigrationCutoverContext, + type OrcadMigrationDestinationCatalog +} from './orcad-migration-cutover-coordinator' +import { findOrcadMigrationSourceCutoverForTarget } from './orcad-migration-cutover-journal' +import { + releaseOrcadMigrationFence, + releaseUndeployedMigrationFence +} from './orcad-migration-source-fence' + +export async function abandonOrcadConversion(args: { + userDataPath: string + store: OrcadMigrationCutoverContext['store'] + claims: OrcadMigrationCutoverContext['claims'] + targetId: string + destinationFor: (environment: KnownRuntimeEnvironment) => OrcadMigrationDestinationCatalog +}): Promise { + const cutover = findOrcadMigrationSourceCutoverForTarget(args.userDataPath, args.targetId) + // A delta move keeps its committed chain; it resolves through the delta flow, never here. + if (!cutover || cutover.supersedesMigrationId) { + return 'none' + } + const isRegistered = (id: string): boolean => + listEnvironments(args.userDataPath).some((entry) => entry.id === id) + const environment = listEnvironments(args.userDataPath).find( + (entry) => entry.id === cutover.destinationEnvironmentId + ) + if (!environment) { + await releaseUndeployedMigrationFence({ ...args, isDestinationRegistered: isRegistered }) + return 'none' + } + const context = { ...args, destination: args.destinationFor(environment) } + return abortOrcadMigrationCutover(context, cutover.migrationId, async (aborted) => { + await closeOrcadManagedTunnel(environment.id).catch(() => undefined) + removeManagedOrcadEnvironment(args.userDataPath, environment.id) + await releaseOrcadMigrationFence(context, aborted) + }) +} diff --git a/src/main/ssh/orcad-daemon-protocol-crossing.ts b/src/main/ssh/orcad-daemon-protocol-crossing.ts new file mode 100644 index 00000000000..9a2506d0896 --- /dev/null +++ b/src/main/ssh/orcad-daemon-protocol-crossing.ts @@ -0,0 +1,56 @@ +/** + * D7: whether live terminals survive an orcad restart onto another build. + * + * The daemon outlives every orcad restart, so after the swap the incoming build must route + * sessions owned by a daemon that may speak another protocol. It can only when it speaks that + * protocol or lists it as previous — the same rule `config/scripts/daemon-protocol-facts.mjs` + * (`canAttach`) applies to release pairs in CI. + */ +import { + PREVIOUS_DAEMON_PROTOCOL_VERSIONS, + PROTOCOL_VERSION +} from '../daemon/daemon-protocol-version' +import type { OrcadTerminalCensus } from './orcad-update-plan' + +export type OrcadDaemonProtocolFacts = { + protocolVersion: number + previousProtocolVersions: readonly number[] +} + +/** This client's build, which is also the orcad bundle it deploys. */ +export const CURRENT_ORCAD_DAEMON_PROTOCOL: OrcadDaemonProtocolFacts = { + protocolVersion: PROTOCOL_VERSION, + previousProtocolVersions: PREVIOUS_DAEMON_PROTOCOL_VERSIONS +} + +export function orcadBuildCanAttachDaemon( + reader: OrcadDaemonProtocolFacts, + ownerProtocolVersion: number +): boolean { + return ( + reader.protocolVersion === ownerProtocolVersion || + reader.previousProtocolVersions.includes(ownerProtocolVersion) + ) +} + +export type OrcadLiveDaemonCrossing = + | 'no-live-terminals' + | 'attachable' + | 'strands-live-terminals' + | 'unverifiable' + +/** An unknown session count or daemon protocol is planned for as live and unattachable. */ +export function assessOrcadLiveDaemonCrossing( + census: OrcadTerminalCensus, + incoming: OrcadDaemonProtocolFacts +): OrcadLiveDaemonCrossing { + if (census.liveSessions === 0) { + return 'no-live-terminals' + } + if (census.daemonProtocolVersion === null) { + return 'unverifiable' + } + return orcadBuildCanAttachDaemon(incoming, census.daemonProtocolVersion) + ? 'attachable' + : 'strands-live-terminals' +} diff --git a/src/main/ssh/orcad-decommission-recovery.ts b/src/main/ssh/orcad-decommission-recovery.ts new file mode 100644 index 00000000000..4c1fecbf698 --- /dev/null +++ b/src/main/ssh/orcad-decommission-recovery.ts @@ -0,0 +1,70 @@ +/** Recovering an interrupted decommission from its journal entry; see orcad-decommission-transaction. */ +import { writeOrcadActivationRecord } from './orcad-activation-record-store' +import type { OrcadActivationRecoveryResult } from './orcad-activation-recovery' +import type { OrcadDecommissionRecoveryPlan } from './orcad-decommission-transaction' +import { settleOrcadDecommissionStop } from './orcad-decommission-stop' +import { orcadLivenessProbeCommand, parseOrcadLiveness } from './orcad-remote-launch' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { + ensureOrcadSlotServing, + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotOptions +} from './orcad-recovery-slot' + +export async function reconcileOrcadDecommission( + options: OrcadSlotOptions, + plan: OrcadDecommissionRecoveryPlan +): Promise { + if (plan.action === 'keep-serving') { + // Nothing was sent, so the recorded version must still be the one serving. + const identity = await resolveOrcadSlotIdentity(options, plan.version) + return { + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: plan.version, + readiness: await ensureOrcadSlotServing(options, identity) + } + } + if (plan.action === 'resume-stop') { + const settlement = await settleOrcadDecommissionStop(options, plan.request) + if (settlement.state === 'unsettled') { + return { + outcome: 'refused', + verdict: settlement.verdict, + code: 'orcad_recovery_decommission_unsettled', + reason: settlement.reason + } + } + if (settlement.state === 'withdrawn') { + return reconcileOrcadDecommission(options, { + action: 'keep-serving', + version: plan.request.version + }) + } + return reconcileOrcadDecommission(options, { + action: 'confirm-decommissioned', + version: plan.request.version, + record: plan.record + }) + } + // Only proven exit commits; a slot that is live or unanswering keeps the fence. + const liveness = parseOrcadLiveness( + await execOrcadRemote( + options, + orcadLivenessProbeCommand(options.host, orcadSlotDir(options, plan.version)) + ) + ) + if (liveness !== 'DEAD') { + return { + outcome: 'refused', + verdict: liveness === 'LIVE' ? 'live' : 'unverifiable', + code: 'orcad_recovery_decommissioned_slot_not_exited', + reason: `orcad ${plan.version} is recorded as stopped but is ${liveness.toLowerCase()}.` + } + } + if (plan.record) { + await writeOrcadActivationRecord(options, plan.record) + } + return { outcome: 'recovered', resolution: 'committed', activeVersion: null, readiness: null } +} diff --git a/src/main/ssh/orcad-decommission-stop.ts b/src/main/ssh/orcad-decommission-stop.ts new file mode 100644 index 00000000000..69b3e85e01d --- /dev/null +++ b/src/main/ssh/orcad-decommission-stop.ts @@ -0,0 +1,72 @@ +/** + * Settling a dispatched decommission stop: proven exit, a clean cancellation, or a fence. + * + * Used by the decommission and by its crash recovery, so both read the host the same way. A + * lost answer is `unverifiable`, never exit; cancelling is how a stop that did not finish is + * withdrawn, and only an orcad-confirmed cancellation lets the fence go. + */ +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { + cancelRemoteOrcadManagedStop, + completeRemoteOrcadManagedStop +} from './orcad-managed-remote-stop' +import type { OrcadSlotOptions } from './orcad-recovery-slot' +import type { + OrcadDaemonRetirementVerdict, + OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' +import { errorMessage } from '../../shared/error-message' + +export type OrcadDecommissionStopSettlement = + | { state: 'exited'; retirement: OrcadDaemonRetirementVerdict } + /** orcad never acted on the request and keeps serving; the verdict says why it was withdrawn. */ + | { state: 'withdrawn'; verdict: 'live' | 'unverifiable'; reason: string } + /** orcad began stopping, or the host could not say: the fence must stay. */ + | { state: 'unsettled'; verdict: 'live' | 'unverifiable'; reason: string } + +export async function settleOrcadDecommissionStop( + options: OrcadSlotOptions, + request: OrcadManagedStopRequest +): Promise { + let verdict: 'live' | 'unverifiable' + let reason: string + try { + const completion = await completeRemoteOrcadManagedStop(options, request) + if (completion.verdict === 'exited') { + // A completion without a recorded outcome proves exit but not retirement. + return { state: 'exited', retirement: completion.retirement ?? 'unverifiable' } + } + verdict = completion.verdict + reason = `orcad ${request.version} did not exit (${completion.verdict}).` + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + verdict = 'unverifiable' + reason = `The host gave no verifiable stop verdict: ${errorMessage(error)}` + } + try { + const cancellation = await cancelRemoteOrcadManagedStop(options, request) + if (cancellation.outcome === 'canceled') { + return { + state: 'withdrawn', + verdict, + reason: `${reason} The stop request was withdrawn; orcad keeps serving.` + } + } + return { + state: 'unsettled', + verdict: 'live', + reason: `${reason} orcad already acted on the request and is still stopping.` + } + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return { + state: 'unsettled', + verdict: 'unverifiable', + reason: `${reason} Withdrawing the request gave no verifiable answer: ${errorMessage(error)}` + } + } +} diff --git a/src/main/ssh/orcad-decommission-transaction.test.ts b/src/main/ssh/orcad-decommission-transaction.test.ts new file mode 100644 index 00000000000..cb49f77a0d5 --- /dev/null +++ b/src/main/ssh/orcad-decommission-transaction.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' +import { + parseOrcadActivationTransaction, + planOrcadTransactionRecovery, + serializeOrcadActivationTransaction +} from './orcad-activation-transaction' +import { + createOrcadDecommissionTransaction, + withOrcadDecommissionProcessExited, + withOrcadDecommissionStopDispatched +} from './orcad-decommission-transaction' +import { FakeOrcadHost, NEW } from './orcad-activation-host-test-harness' +import type { OrcadManagedStopRequest } from '../../shared/orcad-stop-request' + +const T = new Date('2026-02-02T00:00:00.000Z') +const ID = '7f1c2a7e-6c1b-4a8e-9f0e-0a1b2c3d4e5f' +const before = { ...FakeOrcadHost.newRecord(), active: NEW } +const prepared = createOrcadDecommissionTransaction({ + transactionId: ID, + recordBefore: before, + now: T +}) +const request: OrcadManagedStopRequest = { + schemaVersion: 1, + transactionId: ID, + version: NEW, + runtimeId: 'runtime-1', + instance: { pid: 42, startedAtMs: 5, nonce: 'nonce', lockPath: '/home/u/.orca/orcad.lock' }, + retireIdleDaemon: true +} +const dispatched = withOrcadDecommissionStopDispatched(prepared, request, T) +const exited = withOrcadDecommissionProcessExited(dispatched, T) + +describe('the decommission journal entry', () => { + it('deactivates the active version and keeps it as previous', () => { + expect(prepared.recordAfter).toMatchObject({ active: null, previous: NEW, snapshot: null }) + }) + + it('round-trips every phase', () => { + for (const transaction of [prepared, dispatched, exited]) { + expect( + parseOrcadActivationTransaction(serializeOrcadActivationTransaction(transaction)) + ).toEqual({ state: 'ok', transaction }) + } + }) + + it.each([ + ['a dispatched phase without its request', { ...dispatched, request: null }], + ['a request before dispatch', { ...prepared, request }], + [ + 'a request for another transaction', + { + ...dispatched, + request: { ...request, transactionId: '5a7e1f0c-3b2d-4e6f-9a8b-7c6d5e4f3a2b' } + } + ], + [ + 'a request for another version', + { ...dispatched, request: { ...request, version: '0.9.0+ff01' } } + ], + ['a recordAfter that still serves', { ...prepared, recordAfter: before }] + ])('reads %s as unreadable, keeping the fence', (_name, transaction) => { + expect(parseOrcadActivationTransaction(JSON.stringify(transaction))).toMatchObject({ + state: 'unreadable' + }) + }) + + it('plans recovery from what the host is known to have done', () => { + const after = prepared.recordAfter + expect(planOrcadTransactionRecovery(prepared, before)).toEqual({ + action: 'keep-serving', + version: NEW + }) + expect(planOrcadTransactionRecovery(dispatched, before)).toEqual({ + action: 'resume-stop', + request, + record: after + }) + expect(planOrcadTransactionRecovery(exited, before)).toEqual({ + action: 'confirm-decommissioned', + version: NEW, + record: after + }) + expect(planOrcadTransactionRecovery(dispatched, after)).toEqual({ + action: 'confirm-decommissioned', + version: NEW, + record: null + }) + expect(planOrcadTransactionRecovery(dispatched, { ...after, previous: null })).toMatchObject({ + action: 'refuse' + }) + }) +}) diff --git a/src/main/ssh/orcad-decommission-transaction.ts b/src/main/ssh/orcad-decommission-transaction.ts new file mode 100644 index 00000000000..4852cb965fe --- /dev/null +++ b/src/main/ssh/orcad-decommission-transaction.ts @@ -0,0 +1,114 @@ +/** + * The decommission entry of the activation journal: stop the active orcad with an + * instance-bound request and record that nothing serves, or put it back. + * + * Phases: `prepared` (nothing sent), `stop-dispatched` (the request may have reached the + * host), `process-exited` (exit proven by a completed-stop receipt). Only the last may commit. + */ +import { + serializeOrcadActivationRecord, + withDeactivatedVersion, + type OrcadActivationRecord +} from './orcad-activation-record' +import { ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION } from './orcad-activation-transaction-schema' +import type { OrcadManagedStopRequest } from '../../shared/orcad-stop-request' + +export type OrcadDecommissionTransaction = { + schemaVersion: typeof ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION + transactionId: string + operation: 'decommission' + phase: 'prepared' | 'stop-dispatched' | 'process-exited' + startedAt: string + updatedAt: string + activeVersion: string + recordBefore: OrcadActivationRecord + recordAfter: OrcadActivationRecord + request: OrcadManagedStopRequest | null +} + +export type OrcadDecommissionRecoveryPlan = + /** Exit was proven; write the deactivated record (if it is not there yet) and confirm. */ + | { action: 'confirm-decommissioned'; version: string; record: OrcadActivationRecord | null } + /** The request may have reached orcad; its completion or cancellation decides. */ + | { action: 'resume-stop'; request: OrcadManagedStopRequest; record: OrcadActivationRecord } + /** Nothing was sent; the active version must still be serving. */ + | { action: 'keep-serving'; version: string } + +export function createOrcadDecommissionTransaction(options: { + transactionId: string + recordBefore: OrcadActivationRecord & { active: string } + now: Date +}): OrcadDecommissionTransaction { + const timestamp = options.now.toISOString() + return { + schemaVersion: ORCAD_ACTIVATION_TRANSACTION_SCHEMA_VERSION, + transactionId: options.transactionId, + operation: 'decommission', + phase: 'prepared', + startedAt: timestamp, + updatedAt: timestamp, + activeVersion: options.recordBefore.active, + recordBefore: options.recordBefore, + recordAfter: withDeactivatedVersion(options.recordBefore), + request: null + } +} + +export function withOrcadDecommissionStopDispatched( + transaction: OrcadDecommissionTransaction, + request: OrcadManagedStopRequest, + now: Date +): OrcadDecommissionTransaction { + return { ...transaction, phase: 'stop-dispatched', updatedAt: now.toISOString(), request } +} + +export function withOrcadDecommissionProcessExited( + transaction: OrcadDecommissionTransaction, + now: Date +): OrcadDecommissionTransaction { + return { ...transaction, phase: 'process-exited', updatedAt: now.toISOString() } +} + +/** A reason when the parsed journal is not a coherent decommission; otherwise `null`. */ +export function orcadDecommissionTransactionDefect( + transaction: OrcadDecommissionTransaction +): string | null { + if (transaction.recordBefore.active !== transaction.activeVersion) { + return 'decommission version does not match recordBefore' + } + if ( + serializeOrcadActivationRecord(transaction.recordAfter) !== + serializeOrcadActivationRecord(withDeactivatedVersion(transaction.recordBefore)) + ) { + return 'recordAfter is not the deactivated recordBefore' + } + if (transaction.request && transaction.request.version !== transaction.activeVersion) { + return 'stop request names another version' + } + return null +} + +/** Called once the current record matched one side of the transaction. */ +export function planOrcadDecommissionRecovery( + transaction: OrcadDecommissionTransaction, + committed: boolean +): OrcadDecommissionRecoveryPlan { + if (committed) { + return { action: 'confirm-decommissioned', version: transaction.activeVersion, record: null } + } + if (transaction.phase === 'process-exited') { + return { + action: 'confirm-decommissioned', + version: transaction.activeVersion, + record: transaction.recordAfter + } + } + if (transaction.phase === 'stop-dispatched' && transaction.request) { + return { + action: 'resume-stop', + request: transaction.request, + record: transaction.recordAfter + } + } + return { action: 'keep-serving', version: transaction.activeVersion } +} diff --git a/src/main/ssh/orcad-deployment-target.test.ts b/src/main/ssh/orcad-deployment-target.test.ts index 2293078ae33..cdf54d438e0 100644 --- a/src/main/ssh/orcad-deployment-target.test.ts +++ b/src/main/ssh/orcad-deployment-target.test.ts @@ -2,7 +2,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { parseGlibcVersion, parseOrcadLinuxLibc, - resolveOrcadDeploymentTarget, resolveOrcadDeploymentTargetFacts } from './orcad-deployment-target' import { SshConnection } from './ssh-connection' @@ -11,6 +10,12 @@ import { execCommand } from './ssh-relay-deploy-helpers' import { getRemoteHostPlatform } from './ssh-remote-platform' vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn() })) + +async function resolveOrcadDeploymentTarget( + options: Parameters[0] +): Promise { + return (await resolveOrcadDeploymentTargetFacts(options)).target +} beforeEach(() => vi.mocked(execCommand).mockReset()) describe('deployment C library selection', () => { diff --git a/src/main/ssh/orcad-deployment-target.ts b/src/main/ssh/orcad-deployment-target.ts index c0207e7e458..b943c1dfa84 100644 --- a/src/main/ssh/orcad-deployment-target.ts +++ b/src/main/ssh/orcad-deployment-target.ts @@ -1,6 +1,10 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' -import { SERVER_TARGETS, type ServerTarget } from '../../shared/node-runtime-pin' +import { + isNodeRuntimeTarget, + type NodeRuntimeTarget, + type ServerTarget +} from '../../shared/node-runtime-pin' import { ORCAD_SERVER_TARGET_FILENAME } from '../../shared/orcad-artifacts' import type { SshConnection } from './ssh-connection' import { execCommand } from './ssh-relay-deploy-helpers' @@ -39,15 +43,6 @@ export function parseGlibcVersion(output: string): GlibcVersion | null { return match ? { major: Number(match[1]), minor: Number(match[2]) } : null } -export async function resolveOrcadDeploymentTarget(options: { - conn: SshConnection - host: RemoteHostPlatform - signal?: AbortSignal - exec?: (command: string) => Promise -}): Promise { - return (await resolveOrcadDeploymentTargetFacts(options)).target -} - export async function resolveOrcadDeploymentTargetFacts(options: { conn: SshConnection host: RemoteHostPlatform @@ -81,12 +76,11 @@ function linuxTargetFacts(host: RemoteHostPlatform, output: string): OrcadDeploy } } -/** The server target an assembled bundle was built for. */ -export function readOrcadBundleTarget(localOrcadDir: string): ServerTarget { +/** The runtime target an assembled bundle was built for, a compat one included. */ +export function readOrcadBundleTarget(localOrcadDir: string): NodeRuntimeTarget { const recorded = readFileSync(join(localOrcadDir, ORCAD_SERVER_TARGET_FILENAME), 'utf8').trim() - const target = SERVER_TARGETS.find((candidate) => candidate === recorded) - if (!target) { + if (!isNodeRuntimeTarget(recorded)) { throw new Error(`The orcad bundle names no known server target: ${recorded}`) } - return target + return recorded } diff --git a/src/main/ssh/orcad-exited-own-fence-mutation-race.test.ts b/src/main/ssh/orcad-exited-own-fence-mutation-race.test.ts new file mode 100644 index 00000000000..4da20635802 --- /dev/null +++ b/src/main/ssh/orcad-exited-own-fence-mutation-race.test.ts @@ -0,0 +1,234 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { hostname, tmpdir, uptime } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +// The proof's remote commands run in a real local shell. +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => { + const { runProcess: run } = await import('../../shared/child-process/run-process') + return { + ...(await importOriginal()), + execCommand: async (_conn: unknown, command: string) => + (await run({ program: '/bin/sh', args: ['-c', command] })).stdout + } +}) + +const { exitedOwnLockProof } = await import('./orcad-exited-own-lock') +const { initOrcadHeldFenceTokenFile, ORCAD_HELD_FENCE_TOKENS_FILE_NAME } = + await import('./orcad-held-fence-tokens') +const { tryStealInstallLockCommand } = await import('./ssh-relay-install-lock-commands') +const { serializedStateMutationCommand } = await import('./orcad-state-snapshot') +const { ORCAD_FENCE_LOST_EXIT, ORCAD_FENCE_LOST_MARKER } = + await import('./orcad-activation-fence-scope') +const { getRemoteHostPlatform } = await import('./ssh-remote-platform') + +const OWNER = '.orca-fence-owner' +// Above every Linux and macOS pid_max, so no process can hold it. +const EXITED_PID = 4_194_304 + 1 +const quote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'` +const dirs: string[] = [] +afterEach(() => { + for (const dir of dirs.splice(0)) { + writeFileSync(join(dir, 'resume'), '') + rmSync(dir, { recursive: true, force: true }) + } +}) + +/** A quiet fence an exited desktop process left, and the steal a relaunch would run on it. */ +async function fenceOfExitedHolder() { + const dir = mkdtempSync(join(tmpdir(), 'orcad-fence-mutation-race-')) + dirs.push(dir) + const lock = join(dir, '.orcad-activation-transaction', '.install-lock') + mkdirSync(lock, { recursive: true }) + writeFileSync(join(lock, OWNER), 'old-token') + const quietSince = new Date(Date.now() - 10 * 60_000) + utimesSync(lock, quietSince, quietSince) + mkdirSync(join(dir, 'data')) + initOrcadHeldFenceTokenFile(join(dir, 'data', 'orca-data.json')) + writeFileSync( + join(dir, 'data', ORCAD_HELD_FENCE_TOKENS_FILE_NAME), + JSON.stringify([ + { + token: 'old-token', + pid: EXITED_PID, + host: hostname(), + bootedAt: Date.now() - uptime() * 1000, + at: Date.now() + } + ]) + ) + const host = getRemoteHostPlatform('linux-x64') + const proof = exitedOwnLockProof( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + { conn: {} as never, host }, + { baseDir: dir, guardsStateMutation: true } + ) + const token = await proof.find(lock) + expect(token).toBe('old-token') + const steal = tryStealInstallLockCommand( + host, + lock, + 20 * 60, + { fileName: OWNER, token: 'new-token' }, + { + fileName: OWNER, + token: 'old-token', + quietSeconds: proof.quietSeconds, + mutationLock: proof.mutationLock + } + ) + // A surviving mutation of the exited run: it passed its outer fence check as 'old-token'. + const ownerSeen = join(dir, 'owner-seen-by-mutation') + const mutation = serializedStateMutationCommand( + dir, + `: > ${quote(join(dir, 'ready'))}; while [ ! -e ${quote(join(dir, 'resume'))} ]; do sleep 0.01; done; cat ${quote(join(lock, OWNER))} > ${quote(ownerSeen)}`, + 0.05, + { lockDir: lock, token: 'old-token' } + ) + writeFileSync(join(dir, 'mutation.sh'), mutation) + const bin = join(dir, 'bin') + mkdirSync(bin) + return { dir, lock, steal, ownerSeen, bin } +} + +async function waitFor(file: string): Promise { + await expect.poll(() => existsSync(file), { timeout: 10_000 }).toBe(true) +} + +// Astra 26087 round 2: a mutation of the exited run must never keep running under a fence a +// relaunch replaced, whether it is admitted before, during or after the steal. +describe.skipIf(process.platform === 'win32')( + 'an exited holder’s fence and a late mutation', + () => { + it('keeps the fence while a mutation admitted after the proof waits for its first heartbeat', async () => { + const { dir, lock, steal, ownerSeen, bin } = await fenceOfExitedHolder() + const paused = join(dir, 'paused') + writeFileSync( + join(bin, 'touch'), + `#!/bin/sh\nif [ ! -e ${quote(paused)} ]; then : > ${quote(paused)}; while [ ! -e ${quote(join(dir, 'resume'))} ]; do sleep 0.01; done; fi\nPATH=${quote(process.env.PATH ?? '')} exec touch "$@"\n`, + { mode: 0o755 } + ) + const child = spawnProcess({ + program: '/bin/sh', + args: [join(dir, 'mutation.sh')], + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } + }) + const exited = new Promise((resolve) => child.on('exit', resolve)) + await waitFor(paused) + expect((await runProcess({ program: '/bin/sh', args: ['-c', steal] })).stdout.trim()).toBe( + 'BUSY' + ) + writeFileSync(join(dir, 'resume'), '') + await exited + expect(readFileSync(ownerSeen, 'utf8').trim()).toBe('old-token') + expect(readFileSync(join(lock, OWNER), 'utf8')).toBe('old-token') + }) + + it('stops a mutation that takes its lock only after the steal replaced the fence', async () => { + const { dir, lock, steal, ownerSeen, bin } = await fenceOfExitedHolder() + const mutationLock = join(dir, 'orcad-state-mutation.lock') + const paused = join(dir, 'paused') + // The mutation passed its outer fence check and pauses just before taking its lock. + writeFileSync( + join(bin, 'mkdir'), + `#!/bin/sh\nif [ "$1" = ${quote(mutationLock)} ] && [ ! -e ${quote(paused)} ]; then : > ${quote(paused)}; while [ ! -e ${quote(join(dir, 'resume'))} ]; do sleep 0.01; done; fi\nPATH=${quote(process.env.PATH ?? '')} exec mkdir "$@"\n`, + { mode: 0o755 } + ) + const mutation = runProcess({ + program: '/bin/sh', + args: [join(dir, 'mutation.sh')], + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } + }) + await waitFor(paused) + expect((await runProcess({ program: '/bin/sh', args: ['-c', steal] })).stdout.trim()).toBe( + 'EXITED_OWNER_OK' + ) + writeFileSync(join(dir, 'resume'), '') + const result = await mutation + expect(result.code).toBe(ORCAD_FENCE_LOST_EXIT) + expect(result.stdout.trim().split('\n').at(-1)).toBe(ORCAD_FENCE_LOST_MARKER) + expect(existsSync(ownerSeen)).toBe(false) + expect(existsSync(mutationLock)).toBe(false) + expect(readFileSync(join(lock, OWNER), 'utf8')).toBe('new-token') + }) + + // Astra 26087 r3: a steal stalled past the ownerless-lock age still holds the mutation lock, + // because it records its pid as a mutation holder does. Backdating stands in for the stall. + it('keeps a suspended steal’s mutation lock against a mutation however long it stalls', async () => { + const { dir, lock, steal, ownerSeen, bin } = await fenceOfExitedHolder() + const mutationLock = join(dir, 'orcad-state-mutation.lock') + const paused = join(dir, 'paused') + // Stalls the steal right before it moves the fence, after every check it makes. + writeFileSync( + join(bin, 'mv'), + `#!/bin/sh\nif [ ! -e ${quote(paused)} ]; then : > ${quote(paused)}; while [ ! -e ${quote(join(dir, 'resume'))} ]; do sleep 0.01; done; fi\nPATH=${quote(process.env.PATH ?? '')} exec mv "$@"\n`, + { mode: 0o755 } + ) + const stealing = runProcess({ + program: '/bin/sh', + args: ['-c', steal], + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } + }) + await waitFor(paused) + expect(readFileSync(join(mutationLock, 'pid'), 'utf8').trim()).toMatch(/^\d+$/u) + const longAgo = new Date(Date.now() - 10 * 60_000) + utimesSync(mutationLock, longAgo, longAgo) + const mutation = await runProcess({ program: '/bin/sh', args: [join(dir, 'mutation.sh')] }) + expect(mutation.stdout).toContain('STATE_MUTATION_BUSY') + expect(existsSync(ownerSeen)).toBe(false) + writeFileSync(join(dir, 'resume'), '') + expect((await stealing).stdout.trim()).toBe('EXITED_OWNER_OK') + expect(readFileSync(join(lock, OWNER), 'utf8')).toBe('new-token') + expect(existsSync(mutationLock)).toBe(false) + }) + + it.each([ + ['before the steal holds the mutation lock', 2, 'BUSY'], + ['while the steal holds the mutation lock', 3, 'EXITED_OWNER_OK'] + ])( + 'never overlaps a mutation that starts inside the steal claim %s', + async (_when, ownerRead, verdict) => { + const { dir, lock, steal, ownerSeen, bin } = await fenceOfExitedHolder() + const reads = join(dir, 'owner-reads') + const finished = join(dir, 'finished') + writeFileSync( + join(bin, 'cat'), + `#!/bin/sh\nif [ "$1" = ${quote(join(lock, OWNER))} ]; then\nprintf x >> ${quote(reads)}\n` + + `if [ "$(wc -c < ${quote(reads)})" -eq ${ownerRead} ]; then\n` + + `( PATH=${quote(process.env.PATH ?? '')} /bin/sh ${quote(join(dir, 'mutation.sh'))} > ${quote(join(dir, 'mutation-out'))}; : > ${quote(finished)} ) /dev/null 2>&1 &\n` + + `while [ ! -e ${quote(join(dir, 'ready'))} ] && [ ! -e ${quote(finished)} ]; do sleep 0.01; done\nfi\nfi\n` + + `PATH=${quote(process.env.PATH ?? '')} exec cat "$@"\n`, + { mode: 0o755 } + ) + const result = await runProcess({ + program: '/bin/sh', + args: ['-c', steal], + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } + }) + expect(result.stdout.trim()).toBe(verdict) + writeFileSync(join(dir, 'resume'), '') + await waitFor(finished) + if (verdict === 'BUSY') { + // The mutation held its lock first, so it ran under the fence it was admitted under. + expect(readFileSync(ownerSeen, 'utf8').trim()).toBe('old-token') + expect(readFileSync(join(lock, OWNER), 'utf8')).toBe('old-token') + } else { + // The steal held the mutation lock, so the mutation never ran. + expect(readFileSync(join(dir, 'mutation-out'), 'utf8')).toContain('STATE_MUTATION_BUSY') + expect(existsSync(ownerSeen)).toBe(false) + expect(readFileSync(join(lock, OWNER), 'utf8')).toBe('new-token') + } + } + ) + } +) diff --git a/src/main/ssh/orcad-exited-own-fence.test.ts b/src/main/ssh/orcad-exited-own-fence.test.ts new file mode 100644 index 00000000000..261b34dd906 --- /dev/null +++ b/src/main/ssh/orcad-exited-own-fence.test.ts @@ -0,0 +1,229 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { hostname, tmpdir, uptime } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +// Every remote command runs in a real local shell, so the host-side checks are the real ones. +const shell = vi.hoisted((): { env: NodeJS.ProcessEnv | undefined } => ({ env: undefined })) +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => { + const { runProcess } = await import('../../shared/child-process/run-process') + const { sshCommandExitError } = await import('./ssh-relay-exec-command') + return { + ...(await importOriginal()), + execCommand: async (_conn: unknown, command: string) => { + const result = await runProcess({ program: '/bin/sh', args: ['-c', command], env: shell.env }) + if (result.code !== 0) { + throw sshCommandExitError(command, result.code ?? 1, result.stdout) + } + return result.stdout + } + } +}) + +const { orcadActivationFenceRefusal } = await import('./orcad-activation-fence-hold') +const { withOrcadActivationLock } = await import('./orcad-activation-lock') +const { initOrcadHeldFenceTokenFile, ORCAD_HELD_FENCE_TOKENS_FILE_NAME } = + await import('./orcad-held-fence-tokens') +const { getRemoteHostPlatform } = await import('./ssh-remote-platform') + +// Above every Linux and macOS pid_max, so no process can hold it. +const EXITED_PID = 4_194_304 + 1 +const homes: string[] = [] +afterEach(() => { + shell.env = undefined + for (const home of homes.splice(0)) { + rmSync(home, { recursive: true, force: true }) + } +}) + +function entry(token: string, pid: number, host = hostname()) { + return { token, pid, host, bootedAt: Date.now() - uptime() * 1000, at: Date.now() } +} + +/** A fence quiet for ten minutes, owned by `owner`, with this desktop holding `held`. */ +function hostWithFence(owner: string, held: ReturnType[]) { + const home = mkdtempSync(join(tmpdir(), 'orcad-exited-fence-')) + homes.push(home) + const fence = join(home, '.orca-remote', '.orcad-activation-transaction', '.install-lock') + mkdirSync(fence, { recursive: true }) + writeFileSync(join(fence, '.orca-fence-owner'), owner) + const quietSince = new Date(Date.now() - 10 * 60_000) + utimesSync(fence, quietSince, quietSince) + mkdirSync(join(home, 'data')) + initOrcadHeldFenceTokenFile(join(home, 'data', 'orca-data.json')) + const store = join(home, 'data', ORCAD_HELD_FENCE_TOKENS_FILE_NAME) + writeFileSync(store, JSON.stringify(held)) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + const options = { conn: {} as never, host: getRemoteHostPlatform('linux-x64'), remoteHome: home } + return { home, fence, store, options } +} + +// BUG-23: a quit mid-update left this desktop's own fence, and the next launch waited 20 minutes. +describe.skipIf(process.platform === 'win32')('a fence this desktop’s exited process left', () => { + it('is cleared at once when quiet and no state mutation is live', async () => { + const { fence, store, options } = hostWithFence('t-exited', [entry('t-exited', EXITED_PID)]) + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + cleared: true + }) + expect(existsSync(fence)).toBe(false) + expect(readFileSync(store, 'utf-8')).not.toContain('t-exited') + }) + + it('is kept while a state mutation of that run may still be running', async () => { + const { home, fence, options } = hostWithFence('t-exited', [entry('t-exited', EXITED_PID)]) + // sshd kept the pty-less restore running after the desktop quit. + const mutation = join(home, '.orca-remote', 'orcad-state-mutation.lock') + mkdirSync(mutation) + writeFileSync(join(mutation, 'pid'), String(process.pid)) + const refusal = await orcadActivationFenceRefusal(options, 'update') + expect(refusal.cleared).toBeUndefined() + expect(refusal.code).toBe('orcad_activation_fence_busy') + expect(existsSync(fence)).toBe(true) + }) + + it('is kept while it is not yet quiet', async () => { + const { fence, options } = hostWithFence('t-exited', [entry('t-exited', EXITED_PID)]) + utimesSync(fence, new Date(), new Date()) + expect((await orcadActivationFenceRefusal(options, 'update')).cleared).toBeUndefined() + expect(existsSync(fence)).toBe(true) + }) + + it('never touches a fence another desktop holds', async () => { + const { fence, options } = hostWithFence('t-foreign', [entry('t-ours', EXITED_PID)]) + const refusal = await orcadActivationFenceRefusal(options, 'update') + expect(refusal.cleared).toBeUndefined() + expect(refusal.code).toBe('orcad_activation_fence_busy') + expect(existsSync(fence)).toBe(true) + }) + + it('never touches a fence a live process of this desktop holds', async () => { + const { fence, options } = hostWithFence('t-live', [entry('t-live', process.pid)]) + expect((await orcadActivationFenceRefusal(options, 'update')).cleared).toBeUndefined() + expect(existsSync(fence)).toBe(true) + }) + + it('never trusts a pid recorded on another machine sharing the profile', async () => { + const { fence, options } = hostWithFence('t-exited', [ + entry('t-exited', EXITED_PID, 'another-machine') + ]) + expect((await orcadActivationFenceRefusal(options, 'update')).cleared).toBeUndefined() + expect(existsSync(fence)).toBe(true) + }) + + it('hands a fence over a journal to Recover and keeps the journal', async () => { + const { home, fence, options } = hostWithFence('t-exited', [entry('t-exited', EXITED_PID)]) + const journal = join(home, '.orca-remote', '.orcad-activation-transaction', 'transaction.json') + writeFileSync(journal, '{"schemaVersion":1}') + await expect(orcadActivationFenceRefusal(options, 'update')).resolves.toMatchObject({ + code: 'orcad_activation_recovery_required' + }) + expect(existsSync(journal)).toBe(true) + expect(existsSync(fence)).toBe(true) + }) + + // Astra 26087: a live successor that replaces the fence after the proof is never aged or taken. + it('leaves a successor that replaced the fence after the proof alone', async () => { + const { home, fence, store, options } = hostWithFence('t-exited', [ + entry('t-exited', EXITED_PID) + ]) + // The successor's takeover lands right after the quiet check proved the exited holder's fence. + const bin = join(home, 'bin') + mkdirSync(bin) + writeFileSync( + join(bin, 'find'), + `#!/bin/sh\nPATH='${process.env.PATH}' find "$@"\n` + + `if [ ! -e '${home}/replaced' ]; then touch '${home}/replaced'; rm -rf '${fence}'; ` + + `mkdir '${fence}'; printf live-successor > '${fence}/.orca-fence-owner'; fi\n`, + { mode: 0o755 } + ) + shell.env = { ...process.env, PATH: `${bin}:${process.env.PATH}` } + const refusal = await orcadActivationFenceRefusal(options, 'update') + expect(existsSync(join(home, 'replaced'))).toBe(true) + expect(refusal.cleared).toBeUndefined() + expect(readFileSync(join(fence, '.orca-fence-owner'), 'utf-8')).toBe('live-successor') + expect(Date.now() - statSync(fence).mtimeMs).toBeLessThan(60_000) + expect(readFileSync(store, 'utf-8')).toContain('t-exited') + }) +}) + +describe.skipIf(process.platform === 'win32')('the held fence token record', () => { + function held(store: string): string { + return readFileSync(store, 'utf-8') + } + + it('holds a token while its run works and drops it on release', async () => { + const { home, store, options } = hostWithFence('unused', []) + rmSync(join(home, '.orca-remote'), { recursive: true }) + const during = await withOrcadActivationLock( + options, + async () => held(store), + () => '', + 't-run' + ) + expect(during).toContain('t-run') + expect(held(store)).not.toContain('t-run') + }) + + it('drops a token a successor superseded, and keeps one a lost connection left', async () => { + const { home, fence, store, options } = hostWithFence('unused', []) + rmSync(join(home, '.orca-remote'), { recursive: true }) + await withOrcadActivationLock( + options, + async () => writeFileSync(join(fence, '.orca-fence-owner'), 'successor'), + () => undefined, + 't-superseded' + ) + expect(held(store)).not.toContain('t-superseded') + rmSync(join(home, '.orca-remote'), { recursive: true }) + + const lost = Object.assign(new Error('lost'), { sshChannelCloseConfirmed: false }) + await expect( + withOrcadActivationLock( + options, + () => Promise.reject(lost), + () => undefined, + 't-lost' + ) + ).rejects.toBe(lost) + expect(held(store)).toContain('t-lost') + }) + + it('drops a token whose lock was never taken', async () => { + const { fence, store, options } = hostWithFence('t-foreign', []) + utimesSync(fence, new Date(), new Date()) + expect( + await withOrcadActivationLock( + options, + async () => 'ran', + () => 'held', + 't-busy' + ) + ).toBe('held') + expect(held(store)).not.toContain('t-busy') + }, 20_000) + + it('never lets an unwritable record fail a fence operation', async () => { + const { home, store, options } = hostWithFence('unused', []) + rmSync(join(home, '.orca-remote'), { recursive: true }) + rmSync(store) + mkdirSync(store) + expect( + await withOrcadActivationLock( + options, + async () => 'ran', + () => 'held', + 't-x' + ) + ).toBe('ran') + }) +}) diff --git a/src/main/ssh/orcad-exited-own-install-lock.test.ts b/src/main/ssh/orcad-exited-own-install-lock.test.ts new file mode 100644 index 00000000000..6df375af814 --- /dev/null +++ b/src/main/ssh/orcad-exited-own-install-lock.test.ts @@ -0,0 +1,128 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { hostname, tmpdir, uptime } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +// Every remote command runs in a real local shell, so the host-side checks are the real ones. +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => { + const { runProcess } = await import('../../shared/child-process/run-process') + const { sshCommandExitError } = await import('./ssh-relay-exec-command') + return { + ...(await importOriginal()), + execCommand: async (_conn: unknown, command: string) => { + const result = await runProcess({ program: '/bin/sh', args: ['-c', command] }) + if (result.code !== 0) { + throw sshCommandExitError(command, result.code ?? 1, result.stdout) + } + return result.stdout + } + } +}) + +const { acquireInstallLock } = await import('./ssh-relay-install-lock') +const { exitedOwnLockProof } = await import('./orcad-exited-own-lock') +const { initOrcadHeldFenceTokenFile, ORCAD_HELD_FENCE_TOKENS_FILE_NAME } = + await import('./orcad-held-fence-tokens') +const { getRemoteHostPlatform } = await import('./ssh-remote-platform') + +// Above every Linux and macOS pid_max, so no process can hold it. +const EXITED_PID = 4_194_304 + 1 +const homes: string[] = [] +afterEach(() => { + for (const home of homes.splice(0)) { + rmSync(home, { recursive: true, force: true }) + } +}) + +/** A version dir whose install lock a quit left mid-upload, quiet for ten minutes. */ +function versionDirWithLock(owner: string, heldToken: string) { + const home = mkdtempSync(join(tmpdir(), 'orcad-exited-install-')) + homes.push(home) + const dir = join(home, '.orca-remote', 'orcad-0.1.0+aa01') + const lock = join(dir, '.install-lock') + mkdirSync(lock, { recursive: true }) + writeFileSync(join(lock, '.orca-fence-owner'), owner) + const quietSince = new Date(Date.now() - 10 * 60_000) + utimesSync(lock, quietSince, quietSince) + mkdirSync(join(home, 'data')) + initOrcadHeldFenceTokenFile(join(home, 'data', 'orca-data.json')) + const bootedAt = Date.now() - uptime() * 1000 + writeFileSync( + join(home, 'data', ORCAD_HELD_FENCE_TOKENS_FILE_NAME), + JSON.stringify([ + { token: heldToken, pid: EXITED_PID, host: hostname(), bootedAt, at: Date.now() } + ]) + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked to a local shell, so the connection is never used. + const target = { conn: {} as never, host: getRemoteHostPlatform('linux-x64') } + const proof = exitedOwnLockProof(target, { + baseDir: join(home, '.orca-remote'), + guardsStateMutation: false + }) + const store = join(home, 'data', ORCAD_HELD_FENCE_TOKENS_FILE_NAME) + const acquire = (waitTimeoutMs: number, exitedOwner = proof) => + acquireInstallLock(target.conn, dir, target.host, { + waitTimeoutMs, + owner: { fileName: '.orca-fence-owner', token: 't-relaunch' }, + exitedOwner + }) + return { lock, store, proof, acquire } +} + +// BUG-23: a quit mid-upload left the version dir's install lock, and the relaunch waited 20 minutes. +describe.skipIf(process.platform === 'win32')( + 'an install lock this desktop’s exited process left', + () => { + it('is taken over at once', async () => { + const { lock, store, acquire } = versionDirWithLock('t-exited', 't-exited') + const started = Date.now() + await acquire(5_000) + expect(Date.now() - started).toBeLessThan(5_000) + expect(readFileSync(join(lock, '.orca-fence-owner'), 'utf-8')).toBe('t-relaunch') + expect(readFileSync(store, 'utf-8')).not.toContain('t-exited') + }) + + it('never writes to the lock while proving its holder exited', async () => { + const { lock, proof } = versionDirWithLock('t-exited', 't-exited') + const before = statSync(lock).mtimeMs + await expect(proof.find(lock)).resolves.toBe('t-exited') + expect(statSync(lock).mtimeMs).toBe(before) + }) + + // Astra 26087: a live successor that replaces the lock after the proof must never be aged or taken. + it('leaves a successor that replaced the lock after the proof alone', async () => { + const { lock, store, proof, acquire } = versionDirWithLock('t-exited', 't-exited') + const replacing = { + ...proof, + find: async (lockDir: string) => { + const token = await proof.find(lockDir) + rmSync(lock, { recursive: true }) + mkdirSync(lock) + writeFileSync(join(lock, '.orca-fence-owner'), 'live-successor') + return token + } + } + await expect(acquire(1_500, replacing)).rejects.toThrow() + expect(readFileSync(join(lock, '.orca-fence-owner'), 'utf-8')).toBe('live-successor') + expect(Date.now() - statSync(lock).mtimeMs).toBeLessThan(60_000) + expect(readFileSync(store, 'utf-8')).toContain('t-exited') + }) + + it('is left to the stale window when another desktop holds it', async () => { + const { lock, acquire } = versionDirWithLock('t-foreign', 't-exited') + await expect(acquire(1_500)).rejects.toThrow() + expect(existsSync(lock)).toBe(true) + expect(readFileSync(join(lock, '.orca-fence-owner'), 'utf-8')).toBe('t-foreign') + }) + } +) diff --git a/src/main/ssh/orcad-exited-own-lock-windows.test.ts b/src/main/ssh/orcad-exited-own-lock-windows.test.ts new file mode 100644 index 00000000000..1643ae9cd79 --- /dev/null +++ b/src/main/ssh/orcad-exited-own-lock-windows.test.ts @@ -0,0 +1,102 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { hostname, tmpdir, uptime } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' + +// The host script's own checks are covered against the real script; here only the client's part. +const remote = vi.hoisted((): { commands: string[]; reply: string } => ({ + commands: [], + reply: '' +})) +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: async (_conn: unknown, command: string) => { + remote.commands.push(command) + return command.includes('script-present') ? 'ORCAD_HOST_SCRIPT_PRESENT\n' : remote.reply + } +})) + +const { findExitedOwnLockToken } = await import('./orcad-exited-own-lock') +const { initOrcadHeldFenceTokenFile, ORCAD_HELD_FENCE_TOKENS_FILE_NAME } = + await import('./orcad-held-fence-tokens') +const { getRemoteHostPlatform } = await import('./ssh-remote-platform') + +// Above every Linux and macOS pid_max, so no process can hold it. +const EXITED_PID = 4_194_304 + 1 +const host = getRemoteHostPlatform('win32-x64') +const baseDir = 'C:/Users/me/.orca-remote' +const lockDir = `${baseDir}/.orcad-activation-transaction/.install-lock` +let store = '' +let home = '' + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'orcad-exited-win-')) + mkdirSync(join(home, 'data')) + initOrcadHeldFenceTokenFile(join(home, 'data', 'orca-data.json')) + store = join(home, 'data', ORCAD_HELD_FENCE_TOKENS_FILE_NAME) + remote.commands = [] +}) +afterEach(() => rmSync(home, { recursive: true, force: true })) + +function hold(...entries: { token: string; pid: number }[]): void { + const bootedAt = Date.now() - uptime() * 1000 + writeFileSync( + store, + JSON.stringify(entries.map((e) => ({ ...e, host: hostname(), bootedAt, at: Date.now() }))) + ) +} + +function find(guardsStateMutation: boolean): Promise { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the connection is never used. + const target = { conn: {} as never, host } + return findExitedOwnLockToken(target, lockDir, { baseDir, guardsStateMutation }) +} + +const checkOps = (): string[] => remote.commands.filter((c) => c.includes('fence-exited-owner')) + +describe('reclaiming this desktop’s exited lock on a Windows host', () => { + it('asks the host script about exited holders only', async () => { + hold({ token: 't-exited', pid: EXITED_PID }, { token: 't-live', pid: process.pid }) + remote.reply = 'EXITED_OWNER t-exited\n' + await expect(find(true)).resolves.toBe('t-exited') + const [op] = checkOps() + expect(op).toContain('t-exited') + expect(op).not.toContain('t-live') + expect(op).toMatch(/fence-exited-owner"? "?[^ ]*\.install-lock"? "?1"?/u) + }) + + it('finds nothing when the host kept the lock', async () => { + hold({ token: 't-exited', pid: EXITED_PID }) + remote.reply = 'KEPT\n' + await expect(find(false)).resolves.toBeNull() + expect(checkOps()).toHaveLength(1) + }) + + it('ignores an answer naming a token it did not offer', async () => { + hold({ token: 't-exited', pid: EXITED_PID }) + remote.reply = 'EXITED_OWNER t-foreign\n' + await expect(find(false)).resolves.toBeNull() + }) + + it('asks the host nothing while no holder is proven exited', async () => { + hold({ token: 't-live', pid: process.pid }) + await expect(find(true)).resolves.toBeNull() + expect(remote.commands).toEqual([]) + }) + + it('falls back to the stale window when the check would not fit cmd.exe', async () => { + hold({ token: 't-exited', pid: EXITED_PID }) + remote.reply = 'EXITED_OWNER t-exited\n' + const deep = `C:/Users/me/${'nested-folder/'.repeat(200)}.orca-remote` + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so the connection is never used. + const target = { conn: {} as never, host } + await expect( + findExitedOwnLockToken(target, `${deep}/.install-lock`, { + baseDir: deep, + guardsStateMutation: true + }) + ).resolves.toBeNull() + expect(remote.commands).toEqual([]) + }) +}) diff --git a/src/main/ssh/orcad-exited-own-lock.ts b/src/main/ssh/orcad-exited-own-lock.ts new file mode 100644 index 00000000000..d491c7c4ab3 --- /dev/null +++ b/src/main/ssh/orcad-exited-own-lock.ts @@ -0,0 +1,135 @@ +/** + * A lock this desktop's own earlier process took and exited without releasing (BUG-23): the + * activation fence, or a version dir's install lock a quit left mid-upload. The steal takes it + * without the 20-minute wait, but only through its own arbitration and only while the lock is the + * same instance and still names the exited holder's token. Process exit alone is not enough: sshd + * keeps pty-less steps running, so the lock must also be quiet for three heartbeats and, for the + * fence, there must be no state-mutation lock at all; the steal then holds that lock across the + * takeover, and a mutation rechecks its fence once it holds it, so the two never overlap. + */ +import { + ORCAD_FENCE_OWNER_FILENAME, + posixOrcadFenceOwnedTest +} from './orcad-activation-fence-scope' +import { + forgetHeldOrcadFence, + orcadFenceTokensHeldByExitedProcesses +} from './orcad-held-fence-tokens' +import { readBoundedOrcadRemoteRecord } from './orcad-remote-record-file' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { + installOrcadWindowsHostScript, + orcadWindowsHostOpCommand +} from './orcad-remote-windows-node' +import { + ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS, + ORCAD_STATE_MUTATION_LOCK_DIRNAME +} from './orcad-state-snapshot-members' +import { CMD_EXE_COMMAND_LINE_MAX_CHARS } from '../providers/windows-shell-args' +import { shellEscape } from './ssh-connection-utils' +import type { InstallLockExitedOwnerProof } from './ssh-relay-install-lock' +import { isWindowsRemoteHost, joinRemotePath } from './ssh-remote-platform' + +// Keeps the Windows command line short; an older token left out only waits out the stale window. +const MAX_WINDOWS_CANDIDATES = 16 + +/** `baseDir` is `~/.orca-remote`; `guardsStateMutation` means a live state mutation there keeps the lock. */ +export type ExitedOwnLockScope = { baseDir: string; guardsStateMutation: boolean } + +export function exitedOwnLockProof( + target: OrcadRemoteExecTarget, + scope: ExitedOwnLockScope +): InstallLockExitedOwnerProof { + return { + find: (lockDir) => findExitedOwnLockToken(target, lockDir, scope), + reclaimed: forgetHeldOrcadFence, + quietSeconds: ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS, + mutationLock: mutationLockOf(target, scope) ?? undefined + } +} + +/** Read-only: the token of an exited holder of this desktop the lock still names, or null. */ +export async function findExitedOwnLockToken( + target: OrcadRemoteExecTarget, + lockDir: string, + scope: ExitedOwnLockScope +): Promise { + const exited = orcadFenceTokensHeldByExitedProcesses() + if (exited.length === 0) { + return null + } + try { + const token = isWindowsRemoteHost(target.host) + ? await findOnWindows(target, lockDir, scope, exited.slice(-MAX_WINDOWS_CANDIDATES)) + : await findOnPosix(target, lockDir, scope, exited) + return token !== null && exited.includes(token) ? token : null + } catch { + // Unanswered: the stale window still applies. + return null + } +} + +function mutationLockOf(target: OrcadRemoteExecTarget, scope: ExitedOwnLockScope): string | null { + return scope.guardsStateMutation + ? joinRemotePath(target.host, scope.baseDir, ORCAD_STATE_MUTATION_LOCK_DIRNAME) + : null +} + +async function findOnPosix( + target: OrcadRemoteExecTarget, + lockDir: string, + scope: ExitedOwnLockScope, + exited: string[] +): Promise { + const owner = await readBoundedOrcadRemoteRecord( + target, + joinRemotePath(target.host, lockDir, ORCAD_FENCE_OWNER_FILENAME), + 64 + ) + const token = owner.state === 'present' ? owner.raw.trim() : '' + if (!exited.includes(token)) { + return null + } + const command = exitedOwnLockCheckCommand({ lockDir, token }, mutationLockOf(target, scope)) + return (await execOrcadRemote(target, command)).trim() === 'EXITED_OWNER' ? token : null +} + +/** The host script reads the owner itself, so one node.exe answers with the token it found. */ +async function findOnWindows( + target: OrcadRemoteExecTarget, + lockDir: string, + scope: ExitedOwnLockScope, + exited: string[] +): Promise { + const command = orcadWindowsHostOpCommand(target.host, scope.baseDir, 'fence-exited-owner', [ + lockDir, + scope.guardsStateMutation ? '1' : '0', + ...exited + ]) + // Too long for sshd's cmd.exe: no proof, so the stale window applies. + if (command.length > CMD_EXE_COMMAND_LINE_MAX_CHARS) { + return null + } + await installOrcadWindowsHostScript(target, scope.baseDir) + const output = await execOrcadRemote(target, command) + const match = /^EXITED_OWNER (\S+)$/u.exec(output.trim().split(/\r?\n/u).at(-1) ?? '') + return match ? match[1] : null +} + +function exitedOwnLockCheckCommand( + lock: { lockDir: string; token: string }, + mutationLock: string | null +): string { + const quiet = (path: string): string => + `[ -n "$(find ${path} -maxdepth 0 -mmin +${ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS / 60} 2>/dev/null)" ]` + return [ + `${posixOrcadFenceOwnedTest(lock)} && ${quiet(shellEscape(lock.lockDir))} || exit 0;`, + ...(mutationLock + ? [ + // Any mutation lock refuses, as the steal does: it can only take an absent one. + `[ -e ${shellEscape(mutationLock)} ] && exit 0;` + ] + : []), + 'echo EXITED_OWNER' + ].join(' ') +} diff --git a/src/main/ssh/orcad-fence-lost-classification.test.ts b/src/main/ssh/orcad-fence-lost-classification.test.ts new file mode 100644 index 00000000000..18665f40d5d --- /dev/null +++ b/src/main/ssh/orcad-fence-lost-classification.test.ts @@ -0,0 +1,146 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ClientChannel } from 'ssh2' +import { execOrcadStateMutation } from './orcad-state-mutation-exec' +import { execOrcadRemote, execOrcadRemoteOr } from './orcad-remote-runtime-control' +import { + ORCAD_FENCE_LOST_MARKER, + OrcadFenceLostError, + isOrcadFenceLost, + posixOrcadFenceGuard, + runWithOrcadFence +} from './orcad-activation-fence-scope' +import { isUnconfirmedSshCommandTermination, sshCommandExitError } from './ssh-relay-exec-command' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +/** An ssh2 exec channel; the host never acknowledges a close, as when the link is down. */ +class HostChannel extends EventEmitter { + readonly stderr = Object.assign(new EventEmitter(), { resume: () => {} }) + readonly stdin = this + close(): void {} + resume(): void {} + exit(code: number, stdout: string): void { + this.emit('data', Buffer.from(stdout)) + this.emit('close', code) + } +} + +let channel: HostChannel +const commands: string[] = [] +const conn = { + exec: async (command: string) => { + commands.push(command) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand reads only the channel events, close() and resume() HostChannel implements. + return channel as unknown as ClientChannel + }, + usesSystemSshTransport: () => false +} +const target = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only exec and usesSystemSshTransport are read. + conn: conn as unknown as SshConnection, + host: getRemoteHostPlatform('linux-x64') +} +const fence = { + lockDir: '/home/u/.orca-remote/.orcad-activation-transaction/.install-lock', + token: 't1' +} + +// As serializedStateMutationCommand builds it: the guard leads the mutation. +const mutation = `${posixOrcadFenceGuard(fence)} restore` + +function fenced(run: () => Promise): Promise { + return runWithOrcadFence(fence, run) +} + +async function settledAfter(step: Promise, answer: () => void): Promise { + const settled = step.catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + answer() + return settled +} + +beforeEach(() => { + vi.useFakeTimers() + channel = new HostChannel() + commands.length = 0 +}) +afterEach(() => { + vi.useRealTimers() +}) + +// Round 13: the real exec error quotes the command, and every fenced command carries the marker. +describe('a fenced step through the real exec', () => { + it('stays an ordinary failure when the step itself exits nonzero', async () => { + const error = await settledAfter( + fenced(() => execOrcadRemote(target, 'false')), + () => channel.exit(1, 'boom\n') + ) + expect(commands[0]).toContain(ORCAD_FENCE_LOST_MARKER) + expect(error).not.toBeInstanceOf(OrcadFenceLostError) + expect(error).toMatchObject({ exitCode: 1, stdout: 'boom\n' }) + }) + + it('lets a failed step fall back', async () => { + const answer = await settledAfter( + fenced(() => execOrcadRemoteOr(target, 'false', 'FALLBACK')), + () => channel.exit(1, '') + ) + expect(answer).toBe('FALLBACK') + }) + + it('keeps a timed-out step unconfirmed, so the fence is not released under it', async () => { + const settled = fenced(() => execOrcadRemoteOr(target, 'sleep 99', 'FALLBACK')).catch( + (error: unknown) => error + ) + await vi.advanceTimersByTimeAsync(60_000) + const error = await settled + expect(error).not.toBeInstanceOf(OrcadFenceLostError) + expect(isUnconfirmedSshCommandTermination(error)).toBe(true) + }) + + it('reads the guard’s own exit as a lost fence', async () => { + const error = await settledAfter( + fenced(() => execOrcadRemote(target, 'true')), + () => channel.exit(75, `${ORCAD_FENCE_LOST_MARKER}\n`) + ) + expect(error).toBeInstanceOf(OrcadFenceLostError) + }) + + it('does not read a marker-free exit 75 as a lost fence', async () => { + const error = await settledAfter( + fenced(() => execOrcadRemote(target, 'true')), + () => channel.exit(75, 'other\n') + ) + expect(error).not.toBeInstanceOf(OrcadFenceLostError) + }) +}) + +describe('a fenced state mutation through the real exec', () => { + it('stays an ordinary failure when the mutation exits nonzero', async () => { + const error = await settledAfter( + fenced(() => execOrcadStateMutation(target, mutation)), + () => channel.exit(2, 'no snapshot\n') + ) + expect(error).not.toBeInstanceOf(OrcadFenceLostError) + expect(isUnconfirmedSshCommandTermination(error)).toBe(false) + }) + + it('reads the guard’s own exit as a lost fence', async () => { + const error = await settledAfter( + fenced(() => execOrcadStateMutation(target, mutation)), + () => channel.exit(75, `${ORCAD_FENCE_LOST_MARKER}\n`) + ) + expect(error).toBeInstanceOf(OrcadFenceLostError) + }) +}) + +describe('a Windows host op whose exit PowerShell flattened to 1', () => { + it('is a lost fence only when the host script printed the marker', () => { + const op = `powershell.exe -Command "& 'node.exe' 'host.js' --fence d t ${ORCAD_FENCE_LOST_MARKER}"` + expect(isOrcadFenceLost(sshCommandExitError(op, 1, `${ORCAD_FENCE_LOST_MARKER}\r\n`))).toBe( + true + ) + expect(isOrcadFenceLost(sshCommandExitError(op, 1, 'ENOENT\r\n'))).toBe(false) + }) +}) diff --git a/src/main/ssh/orcad-gc-transaction-pins.ts b/src/main/ssh/orcad-gc-transaction-pins.ts new file mode 100644 index 00000000000..4c90e6be434 --- /dev/null +++ b/src/main/ssh/orcad-gc-transaction-pins.ts @@ -0,0 +1,55 @@ +/** + * What an in-flight activation, rollback or decommission still needs from GC. + * + * The journal names every slot the transaction may restart or settle; GC must keep them all. + * A held fence without a journal, or a journal this client cannot read, means a transaction + * this client cannot see into, so GC keeps everything rather than guess. + */ +import { remoteInstallDirName, ORCAD_INSTALL_MODEL } from './remote-install-model' +import type { OrcadActivationTransaction } from './orcad-activation-transaction' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { orcadActivationFenceExists } from './orcad-activation-lock' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control' + +export type OrcadGcTransactionPins = { state: 'pinned'; dirNames: string[] } | { state: 'keep-all' } + +function transactionVersions(transaction: OrcadActivationTransaction): (string | null)[] { + const records = [transaction.recordBefore, transaction.recordAfter] + const fromRecords = records.flatMap((record) => (record ? [record.active, record.previous] : [])) + if (transaction.operation === 'activate') { + return [...fromRecords, transaction.candidateVersion] + } + // Any other operation names its slots through its records. + return transaction.operation === 'rollback' + ? [...fromRecords, transaction.incumbentVersion, transaction.targetVersion] + : fromRecords +} + +export async function readOrcadGcTransactionPins( + target: OrcadRemoteExecTarget & { remoteHome: string } +): Promise { + try { + const transaction = await readOrcadActivationTransaction(target) + if (!transaction) { + // A fence without a journal is a transaction starting or a release cut short. + return (await orcadActivationFenceExists(target)) + ? { state: 'keep-all' } + : { state: 'pinned', dirNames: [] } + } + const versions = transactionVersions(transaction).filter( + (version): version is string => typeof version === 'string' && version.length > 0 + ) + return { + state: 'pinned', + dirNames: [...new Set(versions)].map((version) => + remoteInstallDirName(ORCAD_INSTALL_MODEL, version) + ) + } + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return { state: 'keep-all' } + } +} diff --git a/src/main/ssh/orcad-held-fence-tokens.ts b/src/main/ssh/orcad-held-fence-tokens.ts new file mode 100644 index 00000000000..de029b23e82 --- /dev/null +++ b/src/main/ssh/orcad-held-fence-tokens.ts @@ -0,0 +1,112 @@ +/** + * The activation fence tokens this desktop's processes hold, kept beside the profile so a later + * launch can tell a fence its own quit or crash left from one another desktop holds (BUG-23). + * Best effort: a lost entry only costs the early reclaim, never a fence operation. + */ +import { readFileSync } from 'node:fs' +import { hostname, uptime } from 'node:os' +import { dirname, join } from 'node:path' +import { writeDurableSecureJsonFile } from '../../shared/secure-file' + +export const ORCAD_HELD_FENCE_TOKENS_FILE_NAME = 'orcad-held-fence-tokens.json' +// Older entries are leaks from releases never confirmed; the fence went stale long ago anyway. +const MAX_AGE_MS = 24 * 60 * 60_000 +// Boot time from uptime drifts by the clock's resolution; a reboot moves it far more. +const BOOT_TOLERANCE_MS = 60_000 + +/** `host` and `bootedAt` pin the pid to one machine and one boot: a shared profile dir is not. */ +type HeldFence = { token: string; pid: number; host: string; bootedAt: number; at: number } + +let file: string | null = null + +export function initOrcadHeldFenceTokenFile(dataFile: string): void { + file = join(dirname(dataFile), ORCAD_HELD_FENCE_TOKENS_FILE_NAME) +} + +function bootedAt(): number { + return Date.now() - uptime() * 1000 +} + +function readHeld(): HeldFence[] { + try { + const parsed: unknown = JSON.parse(readFileSync(file ?? '', 'utf-8')) + const fresh = Date.now() - MAX_AGE_MS + return Array.isArray(parsed) + ? parsed.filter((entry) => isHeldFence(entry) && entry.at > fresh) + : [] + } catch { + return [] + } +} + +function isHeldFence(entry: unknown): entry is HeldFence { + return ( + typeof entry === 'object' && + entry !== null && + 'token' in entry && + typeof entry.token === 'string' && + 'pid' in entry && + Number.isSafeInteger(entry.pid) && + 'host' in entry && + typeof entry.host === 'string' && + 'bootedAt' in entry && + typeof entry.bootedAt === 'number' && + 'at' in entry && + typeof entry.at === 'number' + ) +} + +function updateHeld(change: (held: HeldFence[]) => HeldFence[] | null): void { + if (!file) { + return + } + try { + const next = change(readHeld()) + if (next) { + writeDurableSecureJsonFile(file, next) + } + } catch (error) { + console.warn(`[orcad] Could not update the held fence token file: ${String(error)}`) + } +} + +/** Before the lock command: a reply lost after the lock landed still leaves a provable token. */ +export function rememberHeldOrcadFence(token: string): void { + updateHeld((held) => [ + ...held, + { token, pid: process.pid, host: hostname(), bootedAt: bootedAt(), at: Date.now() } + ]) +} + +export function forgetHeldOrcadFence(token: string): void { + updateHeld((held) => + held.some((entry) => entry.token === token) + ? held.filter((entry) => entry.token !== token) + : null + ) +} + +/** Tokens only positively exited earlier processes of this machine and boot held; any other may live. */ +export function orcadFenceTokensHeldByExitedProcesses(): string[] { + return file + ? readHeld() + .filter(heldByExitedProcess) + .map((entry) => entry.token) + : [] +} + +function heldByExitedProcess(entry: HeldFence): boolean { + if ( + entry.pid === process.pid || + entry.host !== hostname() || + Math.abs(entry.bootedAt - bootedAt()) > BOOT_TOLERANCE_MS + ) { + return false + } + try { + process.kill(entry.pid, 0) + return false + } catch (error) { + return error instanceof Error && 'code' in error && error.code === 'ESRCH' + } +} diff --git a/src/main/ssh/orcad-host-unavailable.test.ts b/src/main/ssh/orcad-host-unavailable.test.ts new file mode 100644 index 00000000000..752019ff014 --- /dev/null +++ b/src/main/ssh/orcad-host-unavailable.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { + classifyOrcadHostUnavailable, + orcadCandidateLaunchFailureCode +} from './orcad-host-unavailable' +import { OrcadWindowsLaunchRefusedError } from './orcad-remote-launch-windows' +import { OrcadWindowsCommandLineError } from './orcad-remote-windows-node' + +describe('classifyOrcadHostUnavailable', () => { + it('sends a Windows host that cannot launch orcad back to the relay', () => { + const refused = new OrcadWindowsLaunchRefusedError('breakaway denied') + const unsafe = new OrcadWindowsCommandLineError('C:\\Users\\%x%') + expect(classifyOrcadHostUnavailable(refused)).toBe('unsupported_host') + expect(classifyOrcadHostUnavailable(unsafe)).toBe('unsupported_host') + expect(classifyOrcadHostUnavailable({ code: refused.code })).toBe('unsupported_host') + expect(classifyOrcadHostUnavailable({ code: unsafe.code })).toBe('unsupported_host') + }) + + it('keeps an ordinary launch failure retryable', () => { + expect(orcadCandidateLaunchFailureCode(new Error('spawn failed'))).toBe( + 'orcad_candidate_launch_failed' + ) + expect( + classifyOrcadHostUnavailable({ + code: orcadCandidateLaunchFailureCode(new Error('spawn failed')) + }) + ).toBeNull() + expect(orcadCandidateLaunchFailureCode(new OrcadWindowsLaunchRefusedError('x'))).toBe( + 'orcad_windows_launch_refused' + ) + }) +}) diff --git a/src/main/ssh/orcad-host-unavailable.ts b/src/main/ssh/orcad-host-unavailable.ts new file mode 100644 index 00000000000..4a49fdf466b --- /dev/null +++ b/src/main/ssh/orcad-host-unavailable.ts @@ -0,0 +1,82 @@ +/** + * Deploy failures that mean managed orcad can't run on this host at all, as opposed to failures + * a later connect may not hit again. Only these send a host back to the pinned-relay ladder. + */ + +/** No orcad build exists for this host's platform or target. */ +export class OrcadHostUnsupportedError extends Error { + constructor(message: string) { + super(message) + this.name = 'OrcadHostUnsupportedError' + } +} + +/** This build carries no orcad template (dev builds): retried once the app version changes. */ +export class OrcadArtifactsUnavailableError extends Error { + constructor(message: string) { + super(message) + this.name = 'OrcadArtifactsUnavailableError' + } +} + +/** The host refuses port forwarding and can't run the stdio bridge either. */ +export class OrcadStdioBridgeUnavailableError extends Error { + constructor(message: string) { + super(message) + this.name = 'OrcadStdioBridgeUnavailableError' + } +} + +export const ORCAD_TUNNEL_UNAVAILABLE_REASON = 'ssh_tunnel_unavailable' +export const ORCAD_WINDOWS_LAUNCH_REFUSED_CODE = 'orcad_windows_launch_refused' +export const ORCAD_WINDOWS_COMMAND_LINE_UNSAFE_CODE = 'orcad_windows_command_line_unsafe' + +export type OrcadHostUnavailableReason = + | 'unsupported_host' + | 'artifacts_unavailable' + | 'libc_unidentified' + | 'runtime_self_test' + | 'security_software' + | 'native_preflight' + | typeof ORCAD_TUNNEL_UNAVAILABLE_REASON + +const UNAVAILABLE_BY_ERROR_NAME: Record = { + OrcadHostUnsupportedError: 'unsupported_host', + OrcadArtifactsUnavailableError: 'artifacts_unavailable', + OrcadRemoteLaunchUnsupportedError: 'unsupported_host', + UnidentifiedHostLibcError: 'libc_unidentified', + RemoteNodeRuntimeSelfTestError: 'runtime_self_test', + RemoteNodeRuntimeSecurityModifiedError: 'security_software', + OrcadStdioBridgeUnavailableError: ORCAD_TUNNEL_UNAVAILABLE_REASON, + OrcadWindowsLaunchRefusedError: 'unsupported_host', + OrcadWindowsCommandLineError: 'unsupported_host' +} + +// Why only these deferrals: the candidate's native preflight (libc floor, missing libraries), its +// PTY self-test and a Windows launch refusal fail the same way on every retry; races don't. +const UNAVAILABLE_BY_DEFERRAL_CODE: Record = { + orcad_candidate_preflight_failed: 'native_preflight', + orcad_activation_pty_self_test_failed: 'native_preflight', + [ORCAD_WINDOWS_LAUNCH_REFUSED_CODE]: 'unsupported_host', + [ORCAD_WINDOWS_COMMAND_LINE_UNSAFE_CODE]: 'unsupported_host' +} + +/** The deferral code for a candidate launch failure, keeping the permanent Windows refusals. */ +export function orcadCandidateLaunchFailureCode(error: unknown): string { + const code = error instanceof Error && 'code' in error ? error.code : undefined + return code === ORCAD_WINDOWS_LAUNCH_REFUSED_CODE || + code === ORCAD_WINDOWS_COMMAND_LINE_UNSAFE_CODE + ? code + : 'orcad_candidate_launch_failed' +} + +export function classifyOrcadHostUnavailable(failure: unknown): OrcadHostUnavailableReason | null { + if (failure instanceof Error) { + return UNAVAILABLE_BY_ERROR_NAME[failure.name] ?? null + } + if (failure && typeof failure === 'object' && 'code' in failure) { + const code = failure.code + return typeof code === 'string' ? (UNAVAILABLE_BY_DEFERRAL_CODE[code] ?? null) : null + } + return null +} diff --git a/src/main/ssh/orcad-incumbent-recovery.ts b/src/main/ssh/orcad-incumbent-recovery.ts new file mode 100644 index 00000000000..ea8e4709f2f --- /dev/null +++ b/src/main/ssh/orcad-incumbent-recovery.ts @@ -0,0 +1,153 @@ +/** + * Putting a host back to the slot and state it had before an activation or rollback. + * + * Shared by the in-flight failure paths and by crash recovery, so both apply one rule: state + * a launched slot may have changed is replaced only after that slot is proven exited, and only + * when an operator accepts it — the slot exposed RPC, and with it stopped nothing on the host + * can count the terminals it started, so the snapshot may no longer describe them. + */ +import { execOrcadStateMutation } from './orcad-state-mutation-exec' +import { orcadRemoteBaseDir } from './orcad-remote-windows-node' +import type { ServeReadiness } from '../server/serve-readiness' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { ORCAD_STATE_SNAPSHOT_DIR } from './orcad-activation-record' +import type { OrcadSnapshotVerdict } from './orcad-activation-transaction' +import { + clearOrcadStateSnapshotMembersCommand, + compareOrcadStateSnapshotCommand, + orcadSnapshotIsUnchanged, + parseOrcadSnapshotRestore, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { + ensureOrcadSlotServing, + launchOrcadSlot, + quiesceInterruptedOrcadSlot, + type OrcadSlotIdentity, + type OrcadSlotOptions +} from './orcad-recovery-slot' +import { joinRemotePath } from './ssh-remote-platform' +import { ORCAD_RECOVERY_CHANGED_STATE_CODE } from '../../shared/orcad-managed-runtime' + +export type OrcadIncumbentRecoveryOptions = OrcadSlotOptions & { + /** The operator accepted restoring the snapshot over state a launched build changed. */ + acceptChangedState?: boolean +} + +export type OrcadIncumbentRecovery = + | { outcome: 'restored'; readiness: ServeReadiness | null } + | { outcome: 'refused'; verdict: 'live' | 'unverifiable'; code: string; reason: string } + +export function orcadSnapshotPath(options: OrcadSlotOptions, dirName: string): string { + return joinRemotePath( + options.host, + options.remoteHome, + RELAY_REMOTE_DIR, + ORCAD_STATE_SNAPSHOT_DIR, + dirName + ) +} + +/** Throws when a step cannot be verified; the caller keeps the fence. */ +export async function recoverOrcadIncumbent( + options: OrcadIncumbentRecoveryOptions, + input: { + transactionStartedAt: string + launchedVersion: string | null + incumbent: OrcadSlotIdentity | null + restoreState: OrcadSnapshotVerdict | null + /** The state the launched slot started from, when that is not `restoreState` (a rollback). */ + launchedFromState?: OrcadSnapshotVerdict | null + /** This run itself proved both slots exited, so no fresh liveness probe is needed. */ + slotsProvenExited?: boolean + } +): Promise { + const quiescence = + input.slotsProvenExited || !input.restoreState + ? 'exited' + : await quiesceInterruptedOrcadSlot(options, input.launchedVersion, input.incumbent) + // With no incumbent there is no older reader to protect; the record names nothing to serve. + if (quiescence === 'exited' && input.restoreState && input.incumbent) { + const decision = input.launchedVersion + ? await decideChangedStateRestore(options, input.launchedFromState ?? input.restoreState) + : 'restore' + if (decision !== 'restore' && decision !== 'unchanged') { + return decision + } + // A launched slot that left its starting state untouched still sits on a root to replace. + if (decision === 'restore' || input.launchedFromState) { + await restoreState(options, input.restoreState) + } + } + if (!input.incumbent) { + return { outcome: 'restored', readiness: null } + } + return { + outcome: 'restored', + readiness: input.slotsProvenExited + ? await launchOrcadSlot(options, input.incumbent) + : await ensureOrcadSlotServing(options, input.incumbent) + } +} + +async function decideChangedStateRestore( + options: OrcadIncumbentRecoveryOptions, + state: OrcadSnapshotVerdict +): Promise<'unchanged' | 'restore' | Extract> { + if (state.state === 'captured') { + const snapshotDir = orcadSnapshotPath(options, state.dirName) + // Read-only, so a lost answer is just "changed". + const comparison = await execOrcadRemote( + options, + compareOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + snapshotDir, + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + ).catch(() => '') + if (orcadSnapshotIsUnchanged(comparison)) { + return 'unchanged' + } + } + if (options.acceptChangedState) { + return 'restore' + } + const retained = + state.state === 'captured' ? ` at ${orcadSnapshotPath(options, state.dirName)}` : '' + return { + outcome: 'refused', + verdict: 'unverifiable', + code: ORCAD_RECOVERY_CHANGED_STATE_CODE, + reason: + 'The launched build is stopped, but it changed profile state (or the change could not be ' + + 'checked), so the previous build was not restarted against it and this host serves ' + + 'nothing. Recover to restore the prelaunch snapshot' + + `${retained} and restart the previous build; terminals the launched build started keep ` + + 'running but drop out of the restored state.' + } +} + +async function restoreState(options: OrcadSlotOptions, state: OrcadSnapshotVerdict): Promise { + if (state.state === 'pending') { + throw new Error('The interrupted transaction has no durable snapshot verdict.') + } + const command = + state.state === 'captured' + ? restoreOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + orcadSnapshotPath(options, state.dirName), + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + : clearOrcadStateSnapshotMembersCommand( + options.host, + options.userDataDir, + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + const restored = parseOrcadSnapshotRestore(await execOrcadStateMutation(options, command)) + if (restored !== 'restored') { + throw new Error(`The prelaunch state could not be restored (${restored}).`) + } +} diff --git a/src/main/ssh/orcad-initial-activation-admission.ts b/src/main/ssh/orcad-initial-activation-admission.ts new file mode 100644 index 00000000000..abf94751853 --- /dev/null +++ b/src/main/ssh/orcad-initial-activation-admission.ts @@ -0,0 +1,102 @@ +/** + * Before the first managed activation, prove no unmanaged Orca runtime owns the data root. + * + * With no active record there is no incumbent to stop, but a hand-started orcad or headless + * Orca may still hold the shared root. Its owner records name a PID; a live, unreadable or + * unexpected record defers rather than starting a second owner beside it. + */ +import { ORCAD_LOCK_FILE_NAME } from '../orcad/orcad-instance-lock' +import { PRIMARY_RUNTIME_METADATA_FILE } from '../../shared/runtime-bootstrap' +import { shellEscape } from './ssh-connection-utils' +import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { orcadWindowsBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' + +const OWNER_RECORD_MAX_BYTES = 64 * 1024 + +export type OrcadInitialActivationAdmission = + | { decision: 'proceed' } + | { decision: 'defer'; code: string; reason: string } + +/** Runs on the candidate slot's own runtime, so the probe needs no host Node. */ +export function initialOrcadActivationAdmissionCommand( + host: RemoteHostPlatform, + userDataDir: string, + remoteInstallDir: string, + legacyNodePath: string +): string { + if (isWindowsRemoteHost(host)) { + // Windows has no O_NOFOLLOW; the host script refuses links by lstat instead. + return orcadWindowsHostOpCommand( + host, + orcadWindowsBaseDir(host, remoteInstallDir), + 'owner-admission', + [userDataDir] + ) + } + const owners = [ORCAD_LOCK_FILE_NAME, PRIMARY_RUNTIME_METADATA_FILE].map((name) => ({ + name, + path: joinRemotePath(host, userDataDir, name) + })) + const script = [ + 'const fs=require("node:fs");', + `const limit=${OWNER_RECORD_MAX_BYTES};`, + 'const owners=JSON.parse(process.argv[1]??"[]");', + 'const invalid=(owner)=>`UNVERIFIABLE ${owner.name}`;', + 'function probe(owner){let fd;', + 'try{const noFollow=fs.constants.O_NOFOLLOW;', + 'if(typeof noFollow!=="number")return invalid(owner);', + 'fd=fs.openSync(owner.path,fs.constants.O_RDONLY|noFollow);', + 'const before=fs.fstatSync(fd);', + 'if(!before.isFile()||before.size>limit)return invalid(owner);', + 'const buffer=Buffer.alloc(limit+1);let bytes=0;', + 'while(byteslimit||bytes!==after.size||before.size!==after.size||before.mtimeMs!==after.mtimeMs)', + 'return invalid(owner);', + 'const record=JSON.parse(buffer.subarray(0,bytes).toString("utf8"));', + 'const pid=record?.pid;', + 'if(!Number.isSafeInteger(pid)||pid<=0)return invalid(owner);', + 'try{process.kill(pid,0);return `LIVE ${owner.name} ${pid}`;}', + 'catch(error){if(error?.code==="ESRCH")return null;', + 'if(error?.code==="EPERM")return `LIVE ${owner.name} ${pid}`;', + 'return invalid(owner);}}', + 'catch(error){return error?.code==="ENOENT"?null:invalid(owner);}', + 'finally{if(fd!==undefined){try{fs.closeSync(fd);}catch{}}}}', + 'for(const owner of owners){const result=probe(owner);', + 'if(result){console.log(result);process.exit(0);}}console.log("CLEAR");' + ].join('') + // The subshell turns the selector's `exit 78` into silence, which parses as unverifiable. + return ( + `(${selectOrcadSlotRuntimeCommand(host, remoteInstallDir, legacyNodePath)}; ` + + `"$orcad_runtime" -e ${shellEscape(script)} ${shellEscape(JSON.stringify(owners))})` + ) +} + +export function parseInitialOrcadActivationAdmission( + output: string +): OrcadInitialActivationAdmission { + const result = output.trim().split(/\r?\n/u).pop()?.trim() ?? '' + if (result === 'CLEAR') { + return { decision: 'proceed' } + } + const live = /^LIVE ([^ ]+) ([1-9][0-9]*)$/u.exec(result) + if (live) { + return { + decision: 'defer', + code: 'orcad_initial_runtime_live', + reason: + `An unmanaged runtime owner is still live according to ${live[1]} (pid ${live[2]}). ` + + 'Stop that Orca runtime before converting this data root to managed orcad.' + } + } + const record = /^UNVERIFIABLE ([^ ]+)$/u.exec(result)?.[1] ?? 'owner record' + return { + decision: 'defer', + code: 'orcad_initial_runtime_unverifiable', + reason: + `The host could not safely interpret ${record}, so it cannot prove the shared data root ` + + 'is quiescent. Preserve the file, verify its owner on the host, and retry after the owner exits.' + } +} diff --git a/src/main/ssh/orcad-installed-activation.ts b/src/main/ssh/orcad-installed-activation.ts new file mode 100644 index 00000000000..48d8cbfb8fa --- /dev/null +++ b/src/main/ssh/orcad-installed-activation.ts @@ -0,0 +1,222 @@ +/** + * The locked half of a deploy: stop, snapshot, start and commit, journaled at every step. + * + * The journal is durable before the first mutation, so a crash at any point leaves enough on + * the host for `recoverInterruptedOrcadActivation` to finish or undo it. A run that ends with + * the host provably back on one slot drops the fence; one that cannot prove it keeps it. + */ +import { orcadRemoteBaseDir } from './orcad-remote-windows-node' +import { randomUUID } from 'node:crypto' +import type { OrcadDeployOptions, OrcadDeployResult } from './orcad-remote-deploy' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { withActivatedVersion, type OrcadStateSnapshot } from './orcad-activation-record' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { launchAndJudgeOrcadSlot } from './orcad-candidate-launch-verdict' +import { planOrcadUpdate } from './orcad-update-plan' +import { CURRENT_ORCAD_DAEMON_PROTOCOL } from './orcad-daemon-protocol-crossing' +import { ORCAD_LOG_FILENAME } from './orcad-remote-launch' +import { + captureOrcadStateSnapshotCommand, + orcadSnapshotDirName, + parseOrcadSnapshotCapture +} from './orcad-state-snapshot' +import { joinRemotePath } from './ssh-remote-platform' +import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' +import { preflightInstalledOrcad } from './orcad-remote-preflight' +import type { OrcadActivationLockControl } from './orcad-activation-lock' +import type { OrcadActivateTransaction } from './orcad-activation-transaction' +import { + createOrcadActivationTransaction, + withOrcadActivationCandidateReady, + withOrcadActivationIncumbentStopped, + withOrcadActivationSnapshot +} from './orcad-activation-transaction-transitions' +import { writeOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { execOrcadRemoteOr, withoutAbortSignal } from './orcad-remote-runtime-control' +import { execOrcadStateMutationOr } from './orcad-state-mutation-exec' +import { + initialOrcadActivationAdmissionCommand, + parseInitialOrcadActivationAdmission +} from './orcad-initial-activation-admission' +import { + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotIdentity +} from './orcad-recovery-slot' +import { orcadSnapshotPath } from './orcad-incumbent-recovery' +import { + restartAfterSnapshotFailure, + restoreAfterRejectedCandidate, + stopTransactionIncumbent +} from './orcad-transaction-incumbent' +import { withOrcadLogTail } from './orcad-remote-log-tail' +import { orcadCandidateLaunchFailureCode } from './orcad-host-unavailable' +import { errorMessage } from '../../shared/error-message' + +type Outcome = Extract + +export async function activateInstalledOrcad( + options: OrcadDeployOptions & { localOrcadDir: string }, + fullVersion: string, + remoteDir: string, + lock: OrcadActivationLockControl +): Promise { + const now = options.now ?? ((): Date => new Date()) + const notActivated = (code: string, reason: string): Outcome => ({ + outcome: 'installed-not-activated', + fullVersion, + code, + reason + }) + const record = await readOrcadActivationRecord(options) + const plan = planOrcadUpdate({ + record, + candidateVersion: fullVersion, + census: options.census, + candidateDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL, + ...(options.force !== undefined ? { force: options.force } : {}) + }) + if (plan.action === 'noop') { + return { outcome: 'already-active', fullVersion } + } + if (plan.action === 'defer') { + return notActivated(plan.code, plan.reason) + } + + try { + await preflightInstalledOrcad({ ...options, remoteInstallDir: remoteDir, fullVersion }) + } catch (error) { + options.signal?.throwIfAborted() + return notActivated( + 'orcad_candidate_preflight_failed', + `Candidate profile preflight failed; the incumbent was not stopped: ${errorMessage(error)}` + ) + } + + let incumbent: OrcadSlotIdentity | null = null + if (record.active) { + try { + incumbent = await resolveOrcadSlotIdentity(options, record.active) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return notActivated( + 'orcad_incumbent_identity_unverifiable', + `The active orcad ${record.active} build identity could not be verified before ` + + `stopping it: ${errorMessage(error)} Nothing was stopped.` + ) + } + } else { + const admission = parseInitialOrcadActivationAdmission( + await execOrcadRemoteOr( + options, + initialOrcadActivationAdmissionCommand( + options.host, + options.userDataDir, + remoteDir, + options.nodePath + ) + ) + ) + if (admission.decision === 'defer') { + return notActivated(admission.code, admission.reason) + } + } + + const startedAt = now() + let transaction: OrcadActivateTransaction = createOrcadActivationTransaction({ + transactionId: randomUUID(), + candidateVersion: fullVersion, + recordBefore: record, + snapshotDirName: orcadSnapshotDirName(fullVersion, startedAt.getTime()), + now: startedAt + }) + await writeOrcadActivationTransaction(options, transaction) + lock.retainOnError() + // Past the first mutation a cancel would strand a stopped host, so the run finishes or rolls back. + options = withoutAbortSignal(options) + + const unstopped = incumbent ? await stopTransactionIncumbent(options, incumbent, lock) : null + if (unstopped) { + return notActivated( + 'orcad_outgoing_stop_incomplete', + `${unstopped} No snapshot was taken and the candidate was not started. Orca requires ` + + 'matching runtime readiness before signaling an incumbent and confirmed exit before ' + + 'snapshotting.' + ) + } + transaction = withOrcadActivationIncumbentStopped(transaction, now()) + await writeOrcadActivationTransaction(options, transaction) + + // A live SQLite WAL is not a backup boundary, so the snapshot waits for confirmed exit. + const snapshotDir = orcadSnapshotPath(options, transaction.snapshot.dirName) + const capture = parseOrcadSnapshotCapture( + await execOrcadStateMutationOr( + options, + captureOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + snapshotDir, + orcadRemoteBaseDir(options.host, options.remoteHome) + ), + 'FAILED' + ) + ) + if (capture === 'failed') { + const restarted = incumbent + ? ` The incumbent was stopped before snapshotting; ${await restartAfterSnapshotFailure(options, incumbent, lock)}` + : '' + throw new Error( + `Could not snapshot ${options.userDataDir} before activating ${fullVersion}. Orca's ` + + 'persisted state carries no schema version, so without a snapshot a rollback has no ' + + `way back. Refusing to activate.${restarted}` + ) + } + const snapshot: OrcadStateSnapshot | null = + capture === 'captured' + ? { + dirName: transaction.snapshot.dirName, + takenBeforeVersion: fullVersion, + readableByVersion: record.active, + takenAt: startedAt.toISOString() + } + : null + transaction = withOrcadActivationSnapshot(transaction, capture, now()) + await writeOrcadActivationTransaction(options, transaction) + + const { verdict, launchError } = await launchAndJudgeOrcadSlot(options, { + remoteInstallDir: remoteDir, + fullVersion, + buildHash: computeLocalOrcadBuildHash(options.localOrcadDir) + }) + if (verdict.decision === 'reject') { + const [code, reason] = + launchError === undefined + ? [verdict.code, verdict.reason] + : [ + orcadCandidateLaunchFailureCode(launchError), + `The candidate failed while starting: ${errorMessage(launchError)}` + ] + const restored = await restoreAfterRejectedCandidate(options, lock, { + launchedDir: orcadSlotDir(options, transaction.candidateVersion), + launchedVersion: transaction.candidateVersion, + transactionStartedAt: transaction.startedAt, + incumbent, + restoreState: transaction.snapshot + }) + const located = + `${reason} Candidate stderr is at ` + + `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` + return notActivated(code, await withOrcadLogTail(options, remoteDir, located)) + } + + const recordAfter = withActivatedVersion(record, fullVersion, snapshot, now(), options.appVersion) + transaction = withOrcadActivationCandidateReady(transaction, recordAfter, now()) + await writeOrcadActivationTransaction(options, transaction) + await writeOrcadActivationRecord(options, recordAfter) + return { outcome: 'installed-and-activated', fullVersion, verdict } +} diff --git a/src/main/ssh/orcad-managed-auto-update.test.ts b/src/main/ssh/orcad-managed-auto-update.test.ts new file mode 100644 index 00000000000..e781c078e5b --- /dev/null +++ b/src/main/ssh/orcad-managed-auto-update.test.ts @@ -0,0 +1,172 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record' +import { OrcadHostUnsupportedError } from './orcad-host-unavailable' + +const mocks = vi.hoisted(() => { + const state: { record: OrcadActivationRecord | null } = { record: null } + return { + state, + materialize: vi.fn(async (_target: string) => '/cache/orcad'), + runUpdate: vi.fn(), + migrating: vi.fn((_userData: string, _environmentId: string): unknown => null) + } +}) + +vi.mock('./orcad-runtime-maintenance', () => ({ + withManagedOrcadLifecycle: ( + _userData: string, + _selector: string, + run: (managed: unknown) => Promise + ) => run({ environment: { id: 'env-1' }, deployment: {} }), + runManagedOrcadUpdate: mocks.runUpdate +})) +vi.mock('./orcad-managed-runtime-context', () => ({ + resolveLinkedOrcadContext: async () => ({ + activationRecord: mocks.state.record, + serverTarget: 'linux-x64' + }) +})) +vi.mock('./orcad-managed-migration-status', () => ({ + findIncompleteManagedOrcadMigration: mocks.migrating +})) +vi.mock('./orcad-artifact-materializer', () => ({ materializeOrcadArtifact: mocks.materialize })) +vi.mock('./ssh-relay-versioned-install', () => ({ readLocalFullVersion: () => '0.1.0+new' })) + +import { + autoUpdateManagedOrcadEnvironment, + planManagedOrcadAutoUpdate, + resetBundledOrcadVersionsForTests +} from './orcad-managed-auto-update' + +function record(overrides: Partial): OrcadActivationRecord { + return { ...emptyOrcadActivationRecord(), active: '0.1.0+old', ...overrides } +} + +const plan = (r: OrcadActivationRecord, extra: { failedBefore?: boolean } = {}) => + planManagedOrcadAutoUpdate({ + record: r, + candidateVersion: '0.1.0+new', + appVersion: '1.5.0', + failedBefore: extra.failedBefore ?? false + }) + +describe('whether a connect updates its managed orcad', () => { + it('skips a host already on this build', () => { + expect(plan(record({ active: '0.1.0+new' }))).toEqual({ action: 'skip', reason: 'current' }) + }) + + it('updates a host an older or pre-tracking Orca activated', () => { + expect(plan(record({ activeAppVersion: '1.4.0' }))).toEqual({ action: 'update' }) + expect(plan(record({}))).toEqual({ action: 'update' }) + }) + + it('never downgrades a host a newer Orca activated', () => { + expect(plan(record({ activeAppVersion: '1.6.0' }))).toEqual({ + action: 'skip', + reason: 'host-newer' + }) + }) + + it('leaves alone a build an explicit rollback moved away from', () => { + expect(plan(record({ rolledBackFrom: '0.1.0+new' }))).toEqual({ + action: 'skip', + reason: 'rolled-back' + }) + }) + + it('does not retry an update that already failed for this app version', () => { + expect(plan(record({}), { failedBefore: true })).toEqual({ + action: 'skip', + reason: 'failed-before' + }) + }) + + it('skips when the template carries no build for the host', () => { + expect( + planManagedOrcadAutoUpdate({ + record: record({}), + candidateVersion: null, + appVersion: '1.5.0', + failedBefore: false + }) + ).toEqual({ action: 'skip', reason: 'no-template' }) + }) +}) + +describe('updating a managed orcad on connect', () => { + const run = () => + autoUpdateManagedOrcadEnvironment('/user-data', { + environmentId: 'env-1', + appVersion: '1.5.0', + failedBefore: false, + onUpdating: vi.fn() + }) + + beforeEach(() => { + resetBundledOrcadVersionsForTests() + mocks.materialize.mockReset().mockResolvedValue('/cache/orcad') + mocks.runUpdate.mockReset() + mocks.migrating.mockReset().mockReturnValue(null) + mocks.state.record = record({ activeAppVersion: '1.4.0' }) + }) + + it('runs the Managed servers update, never forced past running terminals', async () => { + mocks.runUpdate.mockResolvedValue({ outcome: 'updated', activeVersion: '0.1.0+new' }) + await expect(run()).resolves.toEqual({ outcome: 'updated', activeVersion: '0.1.0+new' }) + expect(mocks.runUpdate).toHaveBeenCalledWith( + '/user-data', + expect.anything(), + expect.anything(), + {} + ) + }) + + it('reports live terminals as a wait, and a rejected candidate as a failure', async () => { + mocks.runUpdate.mockResolvedValueOnce({ + outcome: 'deferred', + code: 'orcad_update_terminals_running', + reason: '1 terminal is running on this host.' + }) + await expect(run()).resolves.toMatchObject({ outcome: 'deferred' }) + + // A fence another run holds briefly is retried later, never recorded as a failed update. + mocks.runUpdate.mockResolvedValueOnce({ + outcome: 'deferred', + code: 'orcad_activation_fence_busy', + reason: 'Another run is changing this host.' + }) + await expect(run()).resolves.toMatchObject({ outcome: 'deferred' }) + + mocks.runUpdate.mockResolvedValueOnce({ + outcome: 'deferred', + code: 'orcad_candidate_launch_failed', + reason: 'The candidate failed while starting. orcad 0.1.0+old was restarted.' + }) + await expect(run()).resolves.toEqual({ + outcome: 'failed', + reason: 'The candidate failed while starting. orcad 0.1.0+old was restarted.' + }) + + mocks.runUpdate.mockRejectedValueOnce(new Error('Could not snapshot state')) + await expect(run()).resolves.toEqual({ outcome: 'failed', reason: 'Could not snapshot state' }) + }) + + it('reads the bundled build once per session', async () => { + mocks.state.record = record({ active: '0.1.0+new' }) + await run() + await run() + expect(mocks.materialize).toHaveBeenCalledTimes(1) + expect(mocks.runUpdate).not.toHaveBeenCalled() + }) + + it('leaves a server alone while a migration into it is still running', async () => { + mocks.migrating.mockReturnValue({ migrationId: 'm', phase: 'staged' }) + await expect(run()).resolves.toEqual({ outcome: 'skipped', reason: 'migrating' }) + expect(mocks.runUpdate).not.toHaveBeenCalled() + }) + + it('skips a host whose target the template does not carry', async () => { + mocks.materialize.mockRejectedValue(new OrcadHostUnsupportedError('no linux-x64')) + await expect(run()).resolves.toEqual({ outcome: 'skipped', reason: 'no-template' }) + }) +}) diff --git a/src/main/ssh/orcad-managed-auto-update.ts b/src/main/ssh/orcad-managed-auto-update.ts new file mode 100644 index 00000000000..584046646bf --- /dev/null +++ b/src/main/ssh/orcad-managed-auto-update.ts @@ -0,0 +1,151 @@ +/** + * Updating a managed orcad on connect, through the same update the Managed servers action runs. + * + * The connect only picks whether to try: the update planner still defers over live or uncounted + * terminals, and a rejected candidate is restored through the activation journal, so the old + * version keeps serving. A host a newer Orca activated is never downgraded. + */ +import { compareAppVersions } from '../../shared/app-version' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { materializeOrcadArtifact } from './orcad-artifact-materializer' +import { OrcadArtifactsUnavailableError, OrcadHostUnsupportedError } from './orcad-host-unavailable' +import { findIncompleteManagedOrcadMigration } from './orcad-managed-migration-status' +import { ORCAD_ACTIVATION_FENCE_BUSY_CODE } from './orcad-activation-fence-hold' +import { resolveLinkedOrcadContext } from './orcad-managed-runtime-context' +import { runManagedOrcadUpdate, withManagedOrcadLifecycle } from './orcad-runtime-maintenance' +import type { OrcadUpdateDeferCode } from './orcad-update-plan' +import { readLocalFullVersion } from './ssh-relay-versioned-install' + +export type ManagedOrcadAutoUpdateSkip = + | 'current' + | 'no-template' + | 'host-newer' + | 'rolled-back' + | 'failed-before' + | 'migrating' + +export type ManagedOrcadAutoUpdatePlan = + | { action: 'update' } + | { action: 'skip'; reason: ManagedOrcadAutoUpdateSkip } + +export type ManagedOrcadAutoUpdateOutcome = + | { outcome: 'skipped'; reason: ManagedOrcadAutoUpdateSkip } + | { outcome: 'updated'; activeVersion: string } + /** The update planner chose to wait, e.g. for terminals to close; a later connect retries. */ + | { outcome: 'deferred'; code: string; reason: string } + /** The candidate was rejected or the update threw; the incumbent keeps serving. */ + | { outcome: 'failed'; reason: string } + +export function planManagedOrcadAutoUpdate(input: { + record: OrcadActivationRecord + /** This app's bundled build for the host's target; null when the template lacks it. */ + candidateVersion: string | null + appVersion: string + /** An update to this app version already failed on this host. */ + failedBefore: boolean +}): ManagedOrcadAutoUpdatePlan { + const { record, candidateVersion } = input + if (!candidateVersion) { + return { action: 'skip', reason: 'no-template' } + } + if (record.active === candidateVersion) { + return { action: 'skip', reason: 'current' } + } + if (record.rolledBackFrom === candidateVersion) { + return { action: 'skip', reason: 'rolled-back' } + } + // Why absent counts as older: only builds that predate the field omit it. + if ( + record.activeAppVersion && + compareAppVersions(record.activeAppVersion, input.appVersion) > 0 + ) { + return { action: 'skip', reason: 'host-newer' } + } + return input.failedBefore ? { action: 'skip', reason: 'failed-before' } : { action: 'update' } +} + +const WAITING_CODES: ReadonlySet = new Set< + OrcadUpdateDeferCode | typeof ORCAD_ACTIVATION_FENCE_BUSY_CODE +>([ + ORCAD_ACTIVATION_FENCE_BUSY_CODE, + 'orcad_update_terminals_running', + 'orcad_update_terminal_census_unavailable', + 'orcad_update_strands_live_terminals', + 'orcad_update_daemon_protocol_unverifiable', + 'orcad_update_ends_in_process_terminals' +]) + +/** + * Deferrals that mean "not now", as opposed to a candidate that was tried and rejected. An + * interrupted activation is not one: no later connect clears it, so it surfaces as a failure. + */ +export function isWaitingOrcadUpdateDeferral(code: string): boolean { + return WAITING_CODES.has(code) +} + +export function autoUpdateManagedOrcadEnvironment( + userDataPath: string, + args: { + environmentId: string + appVersion: string + failedBefore: boolean + onUpdating: () => void + } +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.environmentId, async (managed) => { + // Why: a restart mid-migration would race the staging the cutover journal is driving. + if (findIncompleteManagedOrcadMigration(userDataPath, managed.environment.id)) { + return { outcome: 'skipped', reason: 'migrating' } + } + const context = await resolveLinkedOrcadContext(managed.environment, managed.deployment) + const plan = planManagedOrcadAutoUpdate({ + record: context.activationRecord, + candidateVersion: await bundledOrcadVersion(context.serverTarget), + appVersion: args.appVersion, + failedBefore: args.failedBefore + }) + if (plan.action === 'skip') { + return { outcome: 'skipped', reason: plan.reason } + } + args.onUpdating() + try { + const result = await runManagedOrcadUpdate(userDataPath, managed, context, {}) + if (result.outcome !== 'deferred') { + return { outcome: 'updated', activeVersion: result.activeVersion } + } + return isWaitingOrcadUpdateDeferral(result.code) + ? { outcome: 'deferred', code: result.code, reason: result.reason } + : { outcome: 'failed', reason: result.reason } + } catch (error) { + return { outcome: 'failed', reason: error instanceof Error ? error.message : String(error) } + } + }) +} + +type OrcadTarget = Parameters[0] + +// Why per session: the packaged template can't change while the app runs, and hashing it costs a read. +const bundledVersions = new Map>() + +function bundledOrcadVersion(target: OrcadTarget): Promise { + let version = bundledVersions.get(target) + if (!version) { + version = materializeOrcadArtifact(target).then(readLocalFullVersion, (error: unknown) => { + if ( + error instanceof OrcadHostUnsupportedError || + error instanceof OrcadArtifactsUnavailableError + ) { + return null + } + throw error + }) + // A transient local failure must not stick for the session. + version.catch(() => bundledVersions.delete(target)) + bundledVersions.set(target, version) + } + return version +} + +export function resetBundledOrcadVersionsForTests(): void { + bundledVersions.clear() +} diff --git a/src/main/ssh/orcad-managed-bound-port.test.ts b/src/main/ssh/orcad-managed-bound-port.test.ts new file mode 100644 index 00000000000..8125a256006 --- /dev/null +++ b/src/main/ssh/orcad-managed-bound-port.test.ts @@ -0,0 +1,112 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshConnection } from './ssh-connection' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const execOrcadRemote = vi.hoisted(() => vi.fn()) +vi.mock('./orcad-remote-runtime-control', () => ({ execOrcadRemote })) +const resolveOrcadRemoteContext = vi.hoisted(() => vi.fn()) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext })) + +import { + orcadBoundPort, + readManagedOrcadBoundPort, + resolveManagedOrcadTunnelPort +} from './orcad-managed-bound-port' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' + +function readinessLine(boundEndpoint: string | null): string { + return `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'runtime-1', boundEndpoint })}\n` +} + +function slot() { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: exec is mocked; the connection is never used. + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteInstallDir: '/home/deploy/.orca-orcad/1.0.0' + } +} + +describe('orcadBoundPort', () => { + it('reads the port orcad actually bound', () => { + expect(orcadBoundPort({ boundEndpoint: 'ws://127.0.0.1:58520' })).toBe(58_520) + }) + + it.each([null, 'not a url', 'ws://127.0.0.1'])('returns null for %s', (boundEndpoint) => { + expect(orcadBoundPort({ boundEndpoint })).toBeNull() + }) +}) + +describe('readManagedOrcadBoundPort', () => { + beforeEach(() => { + execOrcadRemote.mockReset() + }) + + it('follows orcad off a preferred port another runtime holds', async () => { + execOrcadRemote.mockResolvedValue(readinessLine('ws://127.0.0.1:58520')) + await expect(readManagedOrcadBoundPort(slot(), 6_768)).resolves.toBe(58_520) + }) + + it.each([ + ['no readiness file', ''], + ['a readiness without an endpoint', readinessLine(null)] + ])('falls back to the preferred port for %s (older builds)', async (_label, output) => { + execOrcadRemote.mockResolvedValue(output) + await expect(readManagedOrcadBoundPort(slot(), 6_768)).resolves.toBe(6_768) + }) + + it('surfaces a failed host read instead of guessing a port', async () => { + execOrcadRemote.mockImplementation(async () => { + throw new Error('channel closed') + }) + await expect(readManagedOrcadBoundPort(slot(), 6_768)).rejects.toThrow('channel closed') + }) +}) + +describe('resolveManagedOrcadTunnelPort', () => { + const link = { + sshTargetId: 'ssh-1', + sshTargetGeneration: 1, + localPort: 46_768, + remotePort: 6_768 + } + function input(kind: 'orcadDeployment' | 'sshAccess') { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the resolver reads only the link fields. + environment: { [kind]: link } as unknown as KnownRuntimeEnvironment, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: passed through to the mocked context. + target: { id: 'ssh-1' } as SshTarget, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: passed through to the mocked context. + connection: {} as SshConnection + } + } + + beforeEach(() => { + execOrcadRemote.mockReset() + resolveOrcadRemoteContext.mockReset() + }) + + it('reads the active slot of a managed server', async () => { + resolveOrcadRemoteContext.mockResolvedValue({ + activationRecord: { active: '1.0.0+abc' }, + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/deploy' + }) + execOrcadRemote.mockResolvedValue(readinessLine('ws://127.0.0.1:58520')) + await expect(resolveManagedOrcadTunnelPort(input('orcadDeployment'))).resolves.toBe(58_520) + }) + + it('keeps the configured port for independent SSH access without touching the host', async () => { + await expect(resolveManagedOrcadTunnelPort(input('sshAccess'))).resolves.toBe(6_768) + expect(resolveOrcadRemoteContext).not.toHaveBeenCalled() + }) + + it('tries the preferred port when the host cannot be read; the identity check still guards it', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + resolveOrcadRemoteContext.mockRejectedValue(new Error('platform probe failed')) + await expect(resolveManagedOrcadTunnelPort(input('orcadDeployment'))).resolves.toBe(6_768) + warn.mockRestore() + }) +}) diff --git a/src/main/ssh/orcad-managed-bound-port.ts b/src/main/ssh/orcad-managed-bound-port.ts new file mode 100644 index 00000000000..f192fd2910b --- /dev/null +++ b/src/main/ssh/orcad-managed-bound-port.ts @@ -0,0 +1,86 @@ +/** + * The port a managed orcad actually listens on. orcad asks for its preferred port but moves to a + * free one when another runtime (a desktop Orca, `orca serve`) holds it, so a tunnel that assumed + * the preferred port would reach that other runtime instead. The slot's readiness says which. + */ +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import type { ServeReadiness } from '../server/serve-readiness' +import { managedOrcadInstallDir } from './orcad-managed-runtime-context' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { + parseOrcadReadinessWaitOutput, + readOrcadReadinessNowCommand +} from './orcad-remote-readiness-wait' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import type { SshConnection } from './ssh-connection' + +export type OrcadTunnelPortInput = { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection +} + +export function orcadBoundPort(readiness: Pick): number | null { + if (!readiness.boundEndpoint) { + return null + } + try { + const port = Number(new URL(readiness.boundEndpoint).port) + return Number.isInteger(port) && port > 0 && port <= 65_535 ? port : null + } catch { + return null + } +} + +/** Falls back to the preferred port only when the slot published no endpoint (older builds). */ +export async function readManagedOrcadBoundPort( + target: OrcadRemoteExecTarget & { remoteInstallDir: string }, + preferredPort: number +): Promise { + const parsed = parseOrcadReadinessWaitOutput( + target.host, + await execOrcadRemote( + target, + readOrcadReadinessNowCommand(target.host, target.remoteInstallDir) + ) + ) + return (parsed.state === 'ready' ? orcadBoundPort(parsed.readiness) : null) ?? preferredPort +} + +/** Where a tunnel to this environment must point; independent SSH access keeps its own port. */ +export async function resolveManagedOrcadTunnelPort(input: OrcadTunnelPortInput): Promise { + const access = getRuntimeSshAccess(input.environment) + if (!access) { + throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') + } + if (!input.environment.orcadDeployment) { + return access.remotePort + } + try { + const context = await resolveOrcadRemoteContext(input.target, input.connection) + const active = context.activationRecord.active + if (!active) { + return access.remotePort + } + return await readManagedOrcadBoundPort( + { + conn: context.connection, + host: context.host, + remoteHome: context.remoteHome, + remoteInstallDir: managedOrcadInstallDir(context, active) + }, + access.remotePort + ) + } catch (error) { + // Why fall back: the identity check after the forward still catches a wrong server. + console.warn( + '[ssh] Could not read the managed Orca server port; trying the preferred one:', + error + ) + return access.remotePort + } +} diff --git a/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts b/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts new file mode 100644 index 00000000000..764acc35d56 --- /dev/null +++ b/src/main/ssh/orcad-managed-lifecycle-test-fixture.ts @@ -0,0 +1,102 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import type { SshTarget } from '../../shared/ssh-types' +import type { ServeReadiness } from '../server/serve-readiness' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { emptyDependentStateStore } from './ssh-target-orcad-dependents-fixture' + +export const MANAGED_VERSION = '0.2.0+abc' +export const MANAGED_PREVIOUS_VERSION = '0.1.0+def' +export const MANAGED_LOCAL_PORT = 46_768 + +/** A managed server registered in a temp profile, with its claimed SSH target. */ +export function createManagedLifecycleHarness() { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-managed-lifecycle-')) + let target: SshTarget = { + id: 'ssh-1', + label: 'Builder', + host: 'builder', + port: 22, + username: 'dev', + generation: 4, + orcadFence: { environmentId: 'environment-1' } + } + const environment = addManagedOrcadEnvironment(userDataPath, { + id: 'environment-1', + name: 'Managed', + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: `ws://127.0.0.1:${MANAGED_LOCAL_PORT}/`, + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: 'ssh-1', + sshTargetGeneration: 4, + localPort: MANAGED_LOCAL_PORT, + remotePort: 6_768 + } + }) + const flushes: number[] = [] + const claims = new SshTargetOrcadClaims({ + ...emptyDependentStateStore(), + allocateSshTargetGeneration: () => 5, + flushPendingOrThrowAsync: async () => { + flushes.push(flushes.length) + }, + getFolderWorkspaces: () => [], + getRepos: () => [], + getSshTarget: (id) => (id === target.id ? target : undefined), + getSshTargets: () => [target], + updateSshTarget: (_id, updates) => (target = { ...target, ...updates }) + }) + const targetStore = { + getTarget: (id: string) => (id === target.id ? target : undefined), + getOrcadRuntimeClaims: () => claims + } + const context = (record: Record = {}) => ({ + activationRecord: { + active: MANAGED_VERSION, + previous: MANAGED_PREVIOUS_VERSION, + activatedAt: '2026-01-01T00:00:00.000Z', + snapshot: null, + ...record + }, + serverTarget: 'linux-x64-glibc', + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target, + userDataDir: '/home/dev/.orca' + }) + return { userDataPath, environment, targetStore, context, flushes, current: () => target } +} + +/** Readiness whose pairing offer, once tunneled, matches the harness server's saved one. */ +export function managedReadiness(deviceToken = 'device-token'): ServeReadiness { + const endpoint = 'ws://127.0.0.1:6768' + return { + runtimeId: 'runtime-1', + boundEndpoint: endpoint, + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { + available: true, + url: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint, + deviceToken, + publicKeyB64: 'public-key' + }), + endpoint, + deviceId: 'device-1', + webClientUrl: null, + scope: 'runtime', + qr: null + } + } +} diff --git a/src/main/ssh/orcad-managed-migration-status.ts b/src/main/ssh/orcad-managed-migration-status.ts new file mode 100644 index 00000000000..86a5f2c1d0e --- /dev/null +++ b/src/main/ssh/orcad-managed-migration-status.ts @@ -0,0 +1,62 @@ +/** Which dormant migrations into managed servers have not finished, for status and resume. */ +import { + isRetainedOrcadMigrationSourceCutover, + type OrcadMigrationSourceCutover +} from '../../shared/orcad-migration-source-cutover' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' + +export type OrcadManagedPendingMigration = { + migrationId: string + environmentId: string + name: string + sshTargetId: string + phase: OrcadMigrationSourceCutover['phase'] + startedAt: string +} + +/** Throws on an unreadable journal: a migration we cannot read is not "none". */ +export function listPendingManagedOrcadMigrations( + userDataPath: string +): OrcadManagedPendingMigration[] { + return ( + listOrcadMigrationSourceCutovers(userDataPath) + // Why retained too: committed is finished; keeping source rows is downgrade insurance, not work. + .filter( + (cutover) => + cutover.phase !== 'source-retired' && !isRetainedOrcadMigrationSourceCutover(cutover) + ) + .map((cutover) => ({ + migrationId: cutover.migrationId, + environmentId: cutover.destinationEnvironmentId, + name: cutover.destinationName, + sshTargetId: cutover.sshTargetId, + phase: cutover.phase, + startedAt: cutover.startedAt + })) + ) +} + +export function findIncompleteManagedOrcadMigration( + userDataPath: string, + environmentId: string +): OrcadManagedPendingMigration | null { + return ( + listPendingManagedOrcadMigrations(userDataPath).find( + (migration) => migration.environmentId === environmentId + ) ?? null + ) +} + +/** The registered server is the one the journal deploys into, on the journaled registration. */ +export function environmentMatchesManagedOrcadCutover( + environment: KnownRuntimeEnvironment, + cutover: OrcadMigrationSourceCutover +): boolean { + return ( + environment.id === cutover.destinationEnvironmentId && + environment.name === cutover.destinationName && + environment.orcadDeployment?.sshTargetId === cutover.sshTargetId && + environment.orcadDeployment.sshTargetGeneration === cutover.sshTargetGeneration + ) +} diff --git a/src/main/ssh/orcad-managed-remote-stop.ts b/src/main/ssh/orcad-managed-remote-stop.ts new file mode 100644 index 00000000000..0c76dfc405a --- /dev/null +++ b/src/main/ssh/orcad-managed-remote-stop.ts @@ -0,0 +1,191 @@ +/** + * Stopping one remote orcad instance through its instance-bound request, never a signal. + * + * The client names the instance from two host records that must agree: the slot's readiness + * payload (runtime ID, PID, capability) and the data root's instance lock (PID, start time, + * nonce). The slot's own `orcad.js` then writes the request and proves exit on the host. + */ +import { randomUUID } from 'node:crypto' +import { shellEscape } from './ssh-connection-utils' +import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' +import { + parseOrcadReadinessWaitOutput, + readOrcadReadinessNowCommand +} from './orcad-remote-readiness-wait' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { readBoundedOrcadRemoteRecord } from './orcad-remote-record-file' +import { orcadSlotDir, type OrcadSlotOptions } from './orcad-recovery-slot' +import { isWindowsRemoteHost, joinRemotePath } from './ssh-remote-platform' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + orcadWindowsNodeCommandLine, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_RUNTIME_MARKER } from './orcad-windows-host-script' +import { ORCAD_LOCK_FILE_NAME, parseOrcadInstanceLockRecord } from '../orcad/orcad-instance-lock' +import { + ORCAD_CANCEL_MANAGED_STOP_FLAG, + ORCAD_COMPLETE_MANAGED_STOP_FLAG, + ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG, + ORCAD_STOP_REQUESTS_CAPABILITY, + OrcadManagedStopCancellationSchema, + OrcadManagedStopCompletionSchema, + type OrcadManagedStopCancellation, + type OrcadManagedStopCompletion, + type OrcadManagedStopContext, + type OrcadManagedStopRequest +} from '../../shared/orcad-stop-request' + +const LOCK_RECORD_MAX_BYTES = 64 * 1024 + +export type OrcadManagedStopTarget = + | { state: 'ready'; context: OrcadManagedStopContext } + | { state: 'refused'; verdict: 'unverifiable'; code: string; reason: string } + +function refused(code: string, reason: string): OrcadManagedStopTarget { + return { state: 'refused', verdict: 'unverifiable', code, reason } +} + +/** Names the running instance of `version`, or says why the host could not prove which it is. */ +export async function readRemoteOrcadManagedStopTarget( + options: OrcadSlotOptions, + version: string +): Promise { + const slotDir = orcadSlotDir(options, version) + const readiness = parseOrcadReadinessWaitOutput( + options.host, + await execOrcadRemote(options, readOrcadReadinessNowCommand(options.host, slotDir)) + ) + if (readiness.state !== 'ready' || !readiness.readiness.health) { + return refused( + 'orcad_managed_stop_readiness_unverifiable', + `orcad ${version} has no readable readiness record, so the running instance is unknown.` + ) + } + const { health, runtimeId } = readiness.readiness + if (health.stopRequests !== ORCAD_STOP_REQUESTS_CAPABILITY) { + return refused( + 'orcad_managed_stop_unsupported', + `orcad ${version} predates managed stop requests; it can only be stopped by signal.` + ) + } + const lockPath = joinRemotePath(options.host, options.userDataDir, ORCAD_LOCK_FILE_NAME) + const lockRead = await readBoundedOrcadRemoteRecord(options, lockPath, LOCK_RECORD_MAX_BYTES) + const lock = lockRead.state === 'present' ? parseOrcadInstanceLockRecord(lockRead.raw) : null + if (!lock || lock.pid !== health.pid || !runtimeId) { + return refused( + 'orcad_managed_stop_instance_unverifiable', + `The instance lock does not name the orcad ${version} that published readiness.` + ) + } + return { + state: 'ready', + context: { + version, + runtimeId, + instance: { pid: lock.pid, startedAtMs: lock.startedAtMs, nonce: lock.nonce, lockPath } + } + } +} + +export async function completeRemoteOrcadManagedStop( + options: OrcadSlotOptions, + request: OrcadManagedStopRequest +): Promise { + return OrcadManagedStopCompletionSchema.parse( + await runSlotCommand(options, request, ORCAD_COMPLETE_MANAGED_STOP_FLAG) + ) +} + +export async function cancelRemoteOrcadManagedStop( + options: OrcadSlotOptions, + request: OrcadManagedStopRequest +): Promise { + return OrcadManagedStopCancellationSchema.parse( + await runSlotCommand(options, request, ORCAD_CANCEL_MANAGED_STOP_FLAG) + ) +} + +/** Runs the slot's own build, which owns the request format it is asked to write. */ +async function runSlotCommand( + options: OrcadSlotOptions, + request: OrcadManagedStopRequest, + flag: string +): Promise { + const slotDir = orcadSlotDir(options, request.version) + const entry = joinRemotePath(options.host, slotDir, 'orcad.js') + const output = isWindowsRemoteHost(options.host) + ? await runWindowsSlotCommand(options, slotDir, entry, request, flag) + : await execOrcadRemote( + options, + `${selectOrcadSlotRuntimeCommand(options.host, slotDir, options.nodePath)} && ` + + `"$orcad_runtime" ${shellEscape(entry)} ${flag} ${shellEscape(JSON.stringify(request))}` + ) + const line = output + .trim() + .split('\n') + .findLast((candidate) => candidate.trim().startsWith('{')) + if (!line) { + throw new Error('orcad managed stop command returned no verifiable answer') + } + const parsed: unknown = JSON.parse(line) + if (typeof parsed !== 'object' || parsed === null) { + throw new Error('orcad managed stop command returned no verifiable answer') + } + // The answer must be about this exact request, not another transaction's. + if (!('transactionId' in parsed) || parsed.transactionId !== request.transactionId) { + throw new Error('orcad managed stop command answered another transaction') + } + return parsed +} + +/** Windows passes the request as a staged file: JSON on a command line meets two quoting layers. */ +async function runWindowsSlotCommand( + options: OrcadSlotOptions, + slotDir: string, + entry: string, + request: OrcadManagedStopRequest, + flag: string +): Promise { + const baseDir = orcadWindowsBaseDir(options.host, slotDir) + const runtime = readOrcadWindowsEncodedAnswer( + await execOrcadRemote( + options, + orcadWindowsHostOpCommand(options.host, baseDir, 'slot-runtime', [slotDir]) + ), + ORCAD_WINDOWS_RUNTIME_MARKER + ) + if (!runtime) { + throw new Error('The Windows host did not name the runtime this orcad slot needs.') + } + const staged = joinRemotePath(options.host, slotDir, `.orcad-stop-command-${randomUUID()}.json`) + let removeStaged = true + try { + await options.conn.writeFile(staged, JSON.stringify(request), { + hostPlatform: options.host, + signal: options.signal + }) + return await execOrcadRemote( + options, + orcadWindowsNodeCommandLine(runtime, [ + entry, + flag, + ORCAD_MANAGED_STOP_REQUEST_FILE_FLAG, + staged + ]) + ) + } catch (error) { + // The command may still be reading it. + removeStaged = !isUnconfirmedSshCommandTermination(error) + throw error + } finally { + if (removeStaged) { + await execOrcadRemote( + options, + orcadWindowsHostOpCommand(options.host, baseDir, 'remove-file', [staged]) + ).catch(() => {}) + } + } +} diff --git a/src/main/ssh/orcad-managed-runtime-context.ts b/src/main/ssh/orcad-managed-runtime-context.ts new file mode 100644 index 00000000000..6195028ab75 --- /dev/null +++ b/src/main/ssh/orcad-managed-runtime-context.ts @@ -0,0 +1,113 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { + KnownRuntimeEnvironment, + OrcadDeploymentLink +} from '../../shared/runtime-environments' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { ServeReadiness } from '../server/serve-readiness' +import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' +import { probeActiveOrcadReadiness } from './orcad-active-readiness' +import { resolveOrcadRemoteContext, type OrcadRemoteContext } from './orcad-remote-context' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir } from './ssh-relay-versioned-install' +import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' + +export const ORCAD_BIND_HOST = '127.0.0.1' + +// Why empty: managed slots always carry their pinned runtime marker, so a marker-less slot +// fails to launch instead of silently running on whatever Node the host has. +export const MANAGED_ORCAD_LEGACY_NODE_PATH = '' + +export function requireManagedOrcadInfrastructure() { + const targetStore = requireManagedOrcadTargetStore() + const connectionManager = getSshConnectionManager() + if (!connectionManager) { + throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') + } + return { connectionManager, targetStore, claims: targetStore.getOrcadRuntimeClaims() } +} + +export function requireManagedOrcadTargetStore() { + const targetStore = getSshTargetRegistryStore() + if (!targetStore) { + throw new Error('SSH target state is unavailable; the managed Orca server is unverifiable.') + } + return targetStore +} + +export function requireManagedOrcadEnvironment( + userDataPath: string, + selector: string +): { environment: KnownRuntimeEnvironment; deployment: OrcadDeploymentLink } { + const environment = resolveEnvironment(userDataPath, selector) + const deployment = environment.orcadDeployment + if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { + throw new Error('This server is not managed through an orcad SSH deployment.') + } + return { environment, deployment } +} + +export async function resolveLinkedOrcadContext( + environment: KnownRuntimeEnvironment, + deployment: OrcadDeploymentLink, + signal?: AbortSignal +): Promise { + const { connectionManager, targetStore } = requireManagedOrcadInfrastructure() + const target = targetStore.getTarget(deployment.sshTargetId) + if ( + !target || + target.generation !== deployment.sshTargetGeneration || + getManagedOrcadFenceEnvironmentId(target) !== environment.id + ) { + throw new Error('The managed Orca server SSH registration is no longer valid.') + } + const connection = await connectionManager.connect(target) + return resolveOrcadRemoteContext(target, connection, signal) +} + +/** The slot options every remote lifecycle step takes for this managed server. */ +export function managedOrcadSlot(context: OrcadRemoteContext, port: number, signal?: AbortSignal) { + return { + conn: context.connection, + host: context.host, + remoteHome: context.remoteHome, + nodePath: MANAGED_ORCAD_LEGACY_NODE_PATH, + userDataDir: context.userDataDir, + bindHost: ORCAD_BIND_HOST, + port, + signal + } +} + +export function managedOrcadInstallDir(context: OrcadRemoteContext, version: string): string { + return computeRemoteInstallDir( + ORCAD_INSTALL_MODEL, + context.remoteHome, + version, + context.host.pathFlavor + ) +} + +/** The active slot's readiness, accepted only if it is the build this client holds. */ +export function probeManagedOrcadReadiness( + context: OrcadRemoteContext, + localOrcadDir: string, + fullVersion: string, + signal?: AbortSignal +): Promise { + return probeActiveOrcadReadiness( + { + conn: context.connection, + host: context.host, + remoteInstallDir: managedOrcadInstallDir(context, fullVersion), + signal + }, + { buildHash: computeLocalOrcadBuildHash(localOrcadDir), fullVersion } + ) +} + +// Why only this code: deploy never has a session count to force past, so forcing an unknown +// census lands on the daemon-protocol deferral, which force cannot clear. +export function isForceableOrcadDeferral(code: string): boolean { + return code === 'orcad_update_terminals_running' +} diff --git a/src/main/ssh/orcad-managed-serving-verify.ts b/src/main/ssh/orcad-managed-serving-verify.ts new file mode 100644 index 00000000000..65a4b3dedaf --- /dev/null +++ b/src/main/ssh/orcad-managed-serving-verify.ts @@ -0,0 +1,13 @@ +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { verifyManagedTunnelServing } from './orcad-managed-tunnel' + +/** After the tunnel: the server answers, or was proven stopped and started; never throws. */ +export function verifyOrcadManagedServing( + userDataPath: string, + selector: string +): Promise { + return Promise.resolve() + .then(() => verifyManagedTunnelServing(resolveEnvironment(userDataPath, selector))) + .catch((error: unknown) => ({ state: 'unverifiable', detail: String(error) })) +} diff --git a/src/main/ssh/orcad-managed-serving.test.ts b/src/main/ssh/orcad-managed-serving.test.ts new file mode 100644 index 00000000000..3846cef9654 --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.test.ts @@ -0,0 +1,155 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./orcad-remote-context', () => ({ + resolveOrcadRemoteContext: vi.fn(async () => ({ + connection: {}, + host: {}, + remoteHome: '/home/u', + userDataDir: '/home/u/.orca' + })) +})) +vi.mock('./orcad-managed-wake', () => ({ wakeStoppedManagedOrcad: vi.fn() })) + +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { + ensureManagedOrcadServing, + resetManagedOrcadServingForTests, + setManagedOrcadStartListener, + type OrcadManagedServingInput +} from './orcad-managed-serving' + +const listener = { starting: vi.fn(), settled: vi.fn() } +let generation = 1 +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the transport generation is read; the remote context is mocked. +const connection = { getConnectGeneration: () => generation } as never + +function input(probe: () => Promise): OrcadManagedServingInput { + return { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the id is read. + environment: { id: 'env-1' } as never, + target: { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' }, + connection, + remotePort: 6768, + probe: vi.fn(probe) + } +} + +beforeEach(() => { + vi.clearAllMocks() + resetManagedOrcadServingForTests() + setManagedOrcadStartListener(listener) + vi.spyOn(console, 'info').mockImplementation(() => {}) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) + +describe('ensureManagedOrcadServing', () => { + it('costs one round trip when the server answers', async () => { + expect(await ensureManagedOrcadServing(input(async () => true))).toEqual({ state: 'serving' }) + expect(wakeStoppedManagedOrcad).not.toHaveBeenCalled() + expect(listener.starting).not.toHaveBeenCalled() + }) + + it('starts a stopped server from its slot and shows the start on the status line', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only health is read. + return { outcome: 'started', readiness: { health: { previousIdleStop: null } } as never } + }) + + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ + state: 'started', + boundPort: null + }) + expect(vi.mocked(wakeStoppedManagedOrcad).mock.calls[0]?.[0]).toMatchObject({ port: 6768 }) + expect(listener.starting).toHaveBeenCalledOnce() + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', { + state: 'started', + boundPort: null + }) + }) + + it('stays unverifiable with the reason when the start fails, never a terminal verdict', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockImplementation(async (_slot, onStarting) => { + onStarting?.() + throw new Error('orcad did not become ready.\nLast lines of orcad.log:\nboom') + }) + + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving).toEqual({ + state: 'unverifiable', + detail: 'orcad did not become ready.\nLast lines of orcad.log:\nboom' + }) + expect(listener.settled).toHaveBeenCalledWith(expect.anything(), 'env-1', serving) + }) + + it('leaves a live process that did not answer alone', async () => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome: 'serving' }) + expect(await ensureManagedOrcadServing(input(async () => false))).toEqual({ state: 'serving' }) + expect(listener.starting).not.toHaveBeenCalled() + }) + + it.each(['fenced', 'unverifiable', 'not-activated'] as const)( + 'does not start a server whose host says %s', + async (outcome) => { + vi.mocked(wakeStoppedManagedOrcad).mockResolvedValue({ outcome }) + const serving = await ensureManagedOrcadServing(input(async () => false)) + expect(serving.state).toBe('unverifiable') + expect(listener.starting).not.toHaveBeenCalled() + } + ) + + it('shares one check between a fresh tunnel and the connect right after it', async () => { + let now = 0 + const probe = vi.fn(async () => true) + const first = input(probe) + await Promise.all([ + ensureManagedOrcadServing(first, () => now), + ensureManagedOrcadServing(first, () => now) + ]) + now = 4_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledOnce() + now = 6_000 + await ensureManagedOrcadServing(first, () => now) + expect(probe).toHaveBeenCalledTimes(2) + }) + + it('runs its own start on a reconnected transport instead of inheriting a dropped one', async () => { + let dropFirst!: (error: Error) => void + vi.mocked(wakeStoppedManagedOrcad) + .mockImplementationOnce( + (_slot, onStarting) => + new Promise((_resolve, reject) => { + onStarting?.() + dropFirst = reject + }) + ) + .mockImplementationOnce(async () => ({ + outcome: 'started', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only health and boundEndpoint are read. + readiness: { health: { previousIdleStop: null } } as never + })) + const onDropped = ensureManagedOrcadServing(input(async () => false)) + await vi.waitFor(() => expect(wakeStoppedManagedOrcad).toHaveBeenCalledOnce()) + + // The SSH session drops mid-start and the client reconnects, as at app launch. + generation += 1 + const onReconnected = ensureManagedOrcadServing(input(async () => false)) + dropFirst(new Error('SSH connection lost')) + + expect(await onReconnected).toEqual({ state: 'started', boundPort: null }) + expect(await onDropped).toMatchObject({ state: 'unverifiable' }) + expect(wakeStoppedManagedOrcad).toHaveBeenCalledTimes(2) + }) + + it('never answers a new SSH transport from an earlier verdict, as after a reboot', async () => { + const probe = vi.fn(async () => true) + await ensureManagedOrcadServing(input(probe), () => 0) + generation += 1 + await ensureManagedOrcadServing(input(probe), () => 1) + expect(probe).toHaveBeenCalledTimes(2) + // A rebind to the port a restarted server bound is a different server address. + await ensureManagedOrcadServing({ ...input(probe), remotePort: 40_001 }, () => 2) + expect(probe).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/main/ssh/orcad-managed-serving.ts b/src/main/ssh/orcad-managed-serving.ts new file mode 100644 index 00000000000..02891479d4e --- /dev/null +++ b/src/main/ssh/orcad-managed-serving.ts @@ -0,0 +1,160 @@ +/** + * Making sure a managed orcad is serving before a client relies on it: a server that answers + * costs one round trip; one that does not is checked on the host and, only if proven stopped + * (an idle stop, a kill, a host reboot), started from its activated slot. A daemon that survived + * is adopted by the new orcad with its terminals; after a reboot both start fresh. + * + * Never throws. A server that cannot be started stays `unverifiable` with the reason, and is + * never read as evidence that its terminals exited. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { orcadBoundPort } from './orcad-managed-bound-port' +import { managedOrcadSlot } from './orcad-managed-runtime-context' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import type { SshConnection } from './ssh-connection' + +export type OrcadManagedServing = + | { state: 'serving' } + /** `boundPort` is the port the restarted server bound, which a forward must follow. */ + | { state: 'started'; boundPort: number | null } + | { state: 'unverifiable'; detail: string } + +export type OrcadManagedServingInput = { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number + probe: (environment: KnownRuntimeEnvironment, timeoutMs: number) => Promise +} + +/** A wake refused because another update, rollback or recovery holds the host; it clears itself. */ +export const MANAGED_ORCAD_FENCED_DETAIL = + 'An update, rollback or recovery holds this host; it was not started.' + +const PROBE_TIMEOUT_MS = 5_000 +// Why: a connect checks right after its fresh tunnel did; one verdict serves both, on that +// transport and port only, so a kill, reboot or rebind is never answered from cache. +const VERDICT_REUSE_MS = 5_000 + +type ServingTransport = { connection: SshConnection; generation: number; remotePort: number } +// Why per transport: a check on a dropped connection fails, and a caller on the reconnected one +// must run its own instead of inheriting that failure as "could not be started". +const inFlight = new Map }>() +const recent = new Map() + +function sameTransport(a: ServingTransport, b: ServingTransport): boolean { + return ( + a.connection === b.connection && a.generation === b.generation && a.remotePort === b.remotePort + ) +} +export type ManagedOrcadStartListener = { + /** Shows "Starting managed server…" for the host. */ + starting: (target: SshTarget) => void + /** The start finished; an `unverifiable` result carries why, with orcad.log's tail. */ + settled: (target: SshTarget, environmentId: string, serving: OrcadManagedServing) => void +} + +let startListener: ManagedOrcadStartListener | null = null + +/** The SSH status wiring registers where a start is shown. */ +export function setManagedOrcadStartListener(listener: ManagedOrcadStartListener | null): void { + startListener = listener +} + +export function ensureManagedOrcadServing( + input: OrcadManagedServingInput, + now: () => number = Date.now +): Promise { + const id = input.environment.id + const transport: ServingTransport = { + connection: input.connection, + generation: input.connection.getConnectGeneration(), + remotePort: input.remotePort + } + const cached = recent.get(id) + if (cached && sameTransport(cached, transport) && now() - cached.at < VERDICT_REUSE_MS) { + return Promise.resolve(cached.serving) + } + const pending = inFlight.get(id) + if (pending && sameTransport(pending, transport)) { + return pending.check + } + const check = checkAndStart(input) + .then((serving) => { + recent.set(id, { ...transport, at: now(), serving }) + return serving + }) + .finally(() => { + if (inFlight.get(id)?.check === check) { + inFlight.delete(id) + } + }) + inFlight.set(id, { ...transport, check }) + return check +} + +/** Test-only: forget cached verdicts. */ +export function resetManagedOrcadServingForTests(): void { + inFlight.clear() + recent.clear() +} + +async function checkAndStart(input: OrcadManagedServingInput): Promise { + if (await input.probe(input.environment, PROBE_TIMEOUT_MS)) { + return { state: 'serving' } + } + let starting = false + const serving = await wakeIfStopped(input, () => { + starting = true + startListener?.starting(input.target) + }) + if (starting) { + startListener?.settled(input.target, input.environment.id, serving) + } + return serving +} + +async function wakeIfStopped( + input: OrcadManagedServingInput, + onStarting: () => void +): Promise { + const label = input.target.label + try { + const context = await resolveOrcadRemoteContext(input.target, input.connection) + const wake = await wakeStoppedManagedOrcad( + managedOrcadSlot(context, input.remotePort), + onStarting + ) + if (wake.outcome === 'started') { + const idle = wake.readiness.health?.previousIdleStop + const cause = idle + ? `it had stopped after idling at ${idle.stoppedAt}` + : 'it had stopped without an idle-stop record (crash, signal or host restart)' + console.info(`[ssh] Started the managed Orca server on ${label}; ${cause}.`) + return { state: 'started', boundPort: orcadBoundPort(wake.readiness) } + } + // A live process that did not answer may still be starting; it is not restarted. + if (wake.outcome === 'serving') { + return { state: 'serving' } + } + const detail = wakeRefusal(wake.outcome) + console.warn(`[ssh] The managed Orca server on ${label} is not answering: ${detail}`) + return { state: 'unverifiable', detail } + } catch (error) { + console.warn(`[ssh] Could not start the managed Orca server on ${label}:`, error) + return { state: 'unverifiable', detail: error instanceof Error ? error.message : String(error) } + } +} + +function wakeRefusal(outcome: 'not-activated' | 'unverifiable' | 'fenced'): string { + switch (outcome) { + case 'fenced': + return MANAGED_ORCAD_FENCED_DETAIL + case 'not-activated': + return 'This host has no activated managed server to start.' + case 'unverifiable': + return 'Whether the server process is still running could not be proven, so it was not started.' + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-active.ts b/src/main/ssh/orcad-managed-tunnel-active.ts new file mode 100644 index 00000000000..6d8af1835bf --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-active.ts @@ -0,0 +1,91 @@ +/** The managed tunnels a client holds, and the one rule for whether one still belongs to its server. */ +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment, + type RuntimeSshTunnelLink +} from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import type { SshConnection } from './ssh-connection' +import { adoptSshConnection } from './ssh-connection-attribution' +import type { PortForwardEntry, SshPortForwardManager } from './ssh-port-forward' + +export type ActiveOrcadTunnel = { + connection: SshConnection + forwardId: string + localPort: number + /** The port orcad bound, which can differ from the persisted (preferred) one. */ + remotePort: number + preferredPort: number + sshTargetGeneration: number + targetId: string + transportGeneration: number +} + +export type ManagedTunnelExpectation = { + environmentId: string + sshTargetId: string + sshTargetGeneration: number + localPort: number + remotePort: number +} + +/** The environment's SSH access while it and its target still name this tunnel; null otherwise. */ +export function managedTunnelAccess( + environment: KnownRuntimeEnvironment | null | undefined, + target: SshTarget | null | undefined, + expected: ManagedTunnelExpectation +): RuntimeSshTunnelLink | null { + const access = environment ? getRuntimeSshAccess(environment) : undefined + return environment?.id === expected.environmentId && + environment.connectionDependency === 'ssh-tunnel' && + access?.sshTargetId === expected.sshTargetId && + access.sshTargetGeneration === expected.sshTargetGeneration && + access.localPort === expected.localPort && + access.remotePort === expected.remotePort && + target?.generation === expected.sshTargetGeneration && + getManagedOrcadFenceEnvironmentId(target) === expected.environmentId + ? access + : null +} + +/** Removes `entry`'s forward, and the record only if nothing replaced it meanwhile. */ +export async function dropActiveOrcadTunnel( + active: Map, + forwards: SshPortForwardManager, + environmentId: string, + entry: ActiveOrcadTunnel +): Promise { + await forwards.removeForwardAndWait(entry.forwardId) + if (active.get(environmentId) === entry) { + active.delete(environmentId) + } +} + +export function recordActiveOrcadTunnel( + active: Map, + environmentId: string, + forward: PortForwardEntry, + tunnel: Omit +): void { + // The tunnel now relies on this transport, so a cancelled connect that opened it must keep it. + adoptSshConnection(tunnel.connection) + active.set(environmentId, { + ...tunnel, + forwardId: forward.id, + localPort: forward.localPort, + remotePort: forward.remotePort + }) +} + +/** A run a close(), ownership change or transport change overtook: it must not read as success. */ +export class OrcadTunnelSupersededError extends Error { + constructor() { + super('Orca SSH tunnel setup was superseded.') + this.name = 'OrcadTunnelSupersededError' + } +} + +export function supersededTunnelError(): Error { + return new OrcadTunnelSupersededError() +} diff --git a/src/main/ssh/orcad-managed-tunnel-identity-restart.test.ts b/src/main/ssh/orcad-managed-tunnel-identity-restart.test.ts new file mode 100644 index 00000000000..fc107a9f321 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-identity-restart.test.ts @@ -0,0 +1,127 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { encodePairingOffer, parsePairingCode, type PairingOffer } from '../../shared/pairing' +import { + addEnvironmentFromPairingCode, + listEnvironments, + markEnvironmentUsed +} from '../../shared/runtime-environment-store' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { OrcaRuntimeService } from '../runtime/orca-runtime' +import { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc' +import { verifyRuntimePairingIdentity } from '../runtime/runtime-environment-identity-verification' +import { verifyManagedOrcadTunnelIdentity } from './orcad-managed-tunnel-identity' + +vi.mock('../git/worktree', () => ({ + listWorktrees: vi.fn().mockResolvedValue([]), + listWorktreesStrict: vi.fn().mockResolvedValue([]) +})) + +const servers: OrcaRuntimeRpcServer[] = [] +afterEach(async () => { + await Promise.all(servers.splice(0).map((server) => server.stop())) +}) + +/** An orcad process on `userDataPath`, the host profile that holds its E2EE key and devices. */ +async function startOrcad(userDataPath: string): Promise { + const server = new OrcaRuntimeRpcServer({ + runtime: new OrcaRuntimeService(), + userDataPath, + enableWebSocket: true, + wsPort: 0 + }) + await server.start() + servers.push(server) + return server +} + +function pairingOffer(server: OrcaRuntimeRpcServer): PairingOffer { + const offer = server.createPairingOffer({ address: '127.0.0.1', name: 'Desktop A' }) + if (!offer.available) { + throw new Error('pairing unavailable') + } + return parsePairingCode(offer.pairingUrl)! +} + +function managedEnvironment(pairing: PairingOffer, runtimeId: string): KnownRuntimeEnvironment { + return { + id: 'env-1', + name: 'Box server', + createdAt: 1, + updatedAt: 1, + lastUsedAt: null, + runtimeId, + preferredEndpointId: 'ws', + endpoints: [ + { + id: 'ws', + kind: 'websocket', + label: 'Box', + endpoint: pairing.endpoint, + deviceToken: pairing.deviceToken, + publicKeyB64: pairing.publicKeyB64 + } + ], + connectionDependency: 'ssh-tunnel', + orcadDeployment: { + sshTargetId: 'box', + sshTargetGeneration: 1, + localPort: 46_768, + remotePort: 6_768 + } + } +} + +/** The restarted process listens on a new port; the tunnel's local endpoint reaches it. */ +function reachedThrough(pairing: PairingOffer, server: OrcaRuntimeRpcServer): PairingOffer { + const port = new URL(pairingOffer(server).endpoint).port + const endpoint = new URL(pairing.endpoint) + endpoint.port = port + return { ...pairing, endpoint: endpoint.toString() } +} + +describe('reopening a managed tunnel after orcad restarted elsewhere', () => { + it('accepts the same host under a new runtime id, and its status recovers', async () => { + const hostProfile = mkdtempSync(join(tmpdir(), 'orcad-host-')) + const first = await startOrcad(hostProfile) + const paired = pairingOffer(first) + const recorded = (await verifyRuntimePairingIdentity(paired, { runtimeId: null })) + .verifiedRuntimeId + await first.stop() + + // Another desktop updated or woke the host while this one was away. + const restarted = await startOrcad(hostProfile) + const pairing = reachedThrough(paired, restarted) + + await expect( + verifyManagedOrcadTunnelIdentity(managedEnvironment(pairing, recorded)) + ).resolves.toEqual({ verdict: 'verified' }) + // The first authenticated status reply over the reopened tunnel records the new id. + const status = await verifyRuntimePairingIdentity(pairing, { runtimeId: null }) + expect(status.verifiedRuntimeId).not.toBe(recorded) + const desktopProfile = mkdtempSync(join(tmpdir(), 'desktop-a-')) + const saved = addEnvironmentFromPairingCode(desktopProfile, { + name: 'Box server', + pairingCode: encodePairingOffer(pairing) + }) + markEnvironmentUsed(desktopProfile, saved.id, { runtimeId: recorded }) + markEnvironmentUsed(desktopProfile, saved.id, { + runtimeId: status.verifiedRuntimeId, + pairingDeviceToken: pairing.deviceToken + }) + expect(listEnvironments(desktopProfile)[0]?.runtimeId).toBe(status.verifiedRuntimeId) + }) + + it('still refuses a different host that answers on the tunnel', async () => { + const paired = pairingOffer(await startOrcad(mkdtempSync(join(tmpdir(), 'orcad-host-')))) + const other = await startOrcad(mkdtempSync(join(tmpdir(), 'orcad-other-'))) + + await expect( + verifyManagedOrcadTunnelIdentity( + managedEnvironment(reachedThrough(paired, other), 'runtime-recorded') + ) + ).resolves.toMatchObject({ verdict: 'foreign' }) + }) +}) diff --git a/src/main/ssh/orcad-managed-tunnel-identity.test.ts b/src/main/ssh/orcad-managed-tunnel-identity.test.ts new file mode 100644 index 00000000000..af962d87144 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-identity.test.ts @@ -0,0 +1,130 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { encodePairingOffer, PAIRING_OFFER_VERSION, type PairingOffer } from '../../shared/pairing' +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' +import type { ServeReadiness } from '../server/serve-readiness' + +const verifyRuntimePairingIdentity = vi.hoisted(() => + vi.fn<(pairing: PairingOffer, expected: unknown) => Promise>() +) +vi.mock('../runtime/runtime-environment-identity-verification', () => ({ + verifyRuntimePairingIdentity +})) + +const { classifyOrcadTunnelIdentityFailure, deployedOrcadTunnelChecks } = + await import('./orcad-managed-tunnel-identity') + +function readiness(runtimeId: string, port: number): ServeReadiness { + const endpoint = `ws://127.0.0.1:${port}` + return { + runtimeId, + boundEndpoint: endpoint, + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { + available: true, + url: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint, + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + endpoint, + deviceId: 'device-1', + webClientUrl: null, + scope: 'runtime', + qr: null + } + } +} + +describe('classifyOrcadTunnelIdentityFailure', () => { + it('reads a runtime that rejected our keys as another server', () => { + const rejected = new RemoteRuntimeClientError( + 'remote_runtime_unavailable', + 'Remote Orca runtime closed the connection (4001: Unauthorized).', + { closeCode: 4001 } + ) + expect(classifyOrcadTunnelIdentityFailure(rejected).verdict).toBe('foreign') + }) + + it('reads a different runtime id as another server', () => { + const mismatch = new Error('The endpoint does not match this paired runtime identity.') + expect(classifyOrcadTunnelIdentityFailure(mismatch).verdict).toBe('foreign') + }) + + it.each([ + new RemoteRuntimeClientError( + 'remote_runtime_unavailable', + 'Could not connect to the remote Orca runtime: ECONNREFUSED' + ), + new RemoteRuntimeClientError('runtime_timeout', 'Timed out waiting for the remote Orca runtime') + ])('never reads silence as another server: %s', (error) => { + expect(classifyOrcadTunnelIdentityFailure(error).verdict).toBe('unreachable') + }) +}) + +describe('deployedOrcadTunnelChecks', () => { + beforeEach(() => { + verifyRuntimePairingIdentity.mockReset() + }) + + it('targets the bound port and verifies the readiness runtime at the tunnel', async () => { + verifyRuntimePairingIdentity.mockResolvedValue({}) + const checks = deployedOrcadTunnelChecks(readiness('runtime-1', 58_520), vi.fn()) + expect(checks.remotePort).toBe(58_520) + await expect(checks.verify(41_000)).resolves.toEqual({ verdict: 'verified' }) + const [pairing, expected] = verifyRuntimePairingIdentity.mock.calls[0] ?? [] + expect(pairing?.endpoint).toBe('ws://127.0.0.1:41000/') + expect(expected).toEqual({ runtimeId: 'runtime-1' }) + }) + + it('follows a re-read readiness for both the port and the pairing it verifies', async () => { + verifyRuntimePairingIdentity.mockResolvedValue({}) + const reread = vi.fn().mockResolvedValue(readiness('runtime-2', 60_001)) + const checks = deployedOrcadTunnelChecks(readiness('runtime-1', 58_520), reread) + await expect(checks.rereadRemotePort()).resolves.toBe(60_001) + await checks.verify(41_000) + expect(verifyRuntimePairingIdentity.mock.calls[0]?.[1]).toEqual({ runtimeId: 'runtime-2' }) + expect(checks.readiness().runtimeId).toBe('runtime-2') + }) + + it('reports a rejecting runtime at the tunnel as foreign', async () => { + verifyRuntimePairingIdentity.mockRejectedValue( + new RemoteRuntimeClientError('remote_runtime_unavailable', 'closed', { closeCode: 4001 }) + ) + const checks = deployedOrcadTunnelChecks(readiness('runtime-1', 6_768), vi.fn()) + await expect(checks.verify(41_000)).resolves.toMatchObject({ verdict: 'foreign' }) + }) +}) + +describe('verifyManagedOrcadTunnelIdentity', () => { + beforeEach(() => { + verifyRuntimePairingIdentity.mockReset() + }) + + it('proves the server by its pinned key and token, not a per-process runtime id or a stale device id', async () => { + const { verifyManagedOrcadTunnelIdentity } = await import('./orcad-managed-tunnel-identity') + verifyRuntimePairingIdentity.mockResolvedValue({}) + const environment = { + runtimeId: 'runtime-1', + pairedDeviceId: 'device-old', + orcadDeployment: { sshTargetId: 't', sshTargetGeneration: 1, localPort: 1, remotePort: 2 }, + preferredEndpointId: 'ws-1', + endpoints: [ + { + id: 'ws-1', + kind: 'websocket', + label: 'WebSocket', + endpoint: 'ws://127.0.0.1:1', + deviceToken: 'token-new', + publicKeyB64: 'key' + } + ] + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the verifier reads only runtimeId, orcadDeployment and the preferred endpoint stubbed here. + await expect(verifyManagedOrcadTunnelIdentity(environment as never)).resolves.toEqual({ + verdict: 'verified' + }) + expect(verifyRuntimePairingIdentity.mock.calls[0]?.[1]).toEqual({ runtimeId: null }) + }) +}) diff --git a/src/main/ssh/orcad-managed-tunnel-identity.ts b/src/main/ssh/orcad-managed-tunnel-identity.ts new file mode 100644 index 00000000000..05a83685afc --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-identity.ts @@ -0,0 +1,102 @@ +/** + * Proving the runtime behind a managed tunnel is this environment's orcad. A port held by another + * runtime still accepts the connection, so a forward that opened proves nothing; only a status + * call that completes the pairing handshake and reports the paired runtime id does. + */ +import { parsePairingCode, type PairingOffer } from '../../shared/pairing' +import { ORCAD_MANAGED_REMOTE_PORT } from '../../shared/orcad-managed-runtime' +import { + isRecoverableRemoteRuntimeConnectionError, + toRemoteRuntimeClientErrorLike +} from '../../shared/remote-runtime-client-error-classification' +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { verifyRuntimePairingIdentity } from '../runtime/runtime-environment-identity-verification' +import type { ServeReadiness } from '../server/serve-readiness' +import { orcadBoundPort } from './orcad-managed-bound-port' +import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' + +/** `unreachable` means nothing answered; it is never evidence about which server holds the port. */ +export type OrcadTunnelIdentity = + | { verdict: 'verified' } + | { verdict: 'foreign' | 'unreachable'; detail: string } + +export const ORCAD_IDENTITY_MISMATCH_CODE = 'orcad_identity_mismatch' + +export class OrcadManagedIdentityError extends Error { + readonly code = ORCAD_IDENTITY_MISMATCH_CODE + constructor(remotePort: number, detail: string) { + super( + `${ORCAD_IDENTITY_MISMATCH_CODE}: the server on remote port ${remotePort} is not this ` + + `managed Orca server (another Orca may be using that port): ${detail}` + ) + this.name = 'OrcadManagedIdentityError' + } +} + +// Why 4001/4003: a runtime that rejects our keys closes the socket rather than replying. +const FOREIGN_CLOSE_CODES = new Set([4001, 4003]) + +export function classifyOrcadTunnelIdentityFailure(error: unknown): OrcadTunnelIdentity { + const detail = error instanceof Error ? error.message : String(error) + if (error instanceof RemoteRuntimeClientError && FOREIGN_CLOSE_CODES.has(error.closeCode ?? 0)) { + return { verdict: 'foreign', detail } + } + return isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error)) + ? { verdict: 'unreachable', detail } + : { verdict: 'foreign', detail } +} + +export async function verifyOrcadPairingIdentity( + pairing: PairingOffer, + expected: Pick +): Promise { + try { + await verifyRuntimePairingIdentity(pairing, expected) + return { verdict: 'verified' } + } catch (error) { + return classifyOrcadTunnelIdentityFailure(error) + } +} + +/** Independent SSH access proved its identity when it linked; only managed orcad is checked. */ +export function verifyManagedOrcadTunnelIdentity( + environment: KnownRuntimeEnvironment +): Promise { + if (!environment.orcadDeployment) { + return Promise.resolve({ verdict: 'verified' }) + } + // Why no runtime id: it is minted per process, so a restart by another desktop, or while this one + // was away, changes it. The E2EE handshake with our pinned host key and our token being accepted + // prove the server; the first authenticated status reply records the new id. Why no device id: + // one left stale by a racing reply must not block it either. + return verifyOrcadPairingIdentity(getPreferredPairingOffer(environment), { runtimeId: null }) +} + +/** + * A fresh deploy has no saved environment yet: its pairing comes from the readiness, and a + * foreign answer re-reads that readiness, whose bound endpoint the tunnel then follows. + */ +export function deployedOrcadTunnelChecks( + initial: ServeReadiness, + rereadReadiness: () => Promise +) { + let readiness = initial + return { + readiness: () => readiness, + remotePort: orcadBoundPort(initial) ?? ORCAD_MANAGED_REMOTE_PORT, + rereadRemotePort: async () => { + readiness = await rereadReadiness() + return orcadBoundPort(readiness) ?? ORCAD_MANAGED_REMOTE_PORT + }, + verify: async (localPort: number): Promise => { + const pairing = parsePairingCode(tunneledOrcadPairingCode(readiness, localPort)) + return pairing + ? verifyOrcadPairingIdentity(pairing, { runtimeId: readiness.runtimeId || null }) + : { verdict: 'foreign', detail: 'The managed Orca server published an invalid pairing.' } + } + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-manager.ts b/src/main/ssh/orcad-managed-tunnel-manager.ts new file mode 100644 index 00000000000..dcecb58f497 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-manager.ts @@ -0,0 +1,296 @@ +/** + * One SSH forward per managed environment, owned across reconnects and host resume, and the + * serving check that starts a stopped server behind it. + */ +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { SshConnection } from './ssh-connection' +import { isAuthError } from './ssh-connection-utils' +import type { SshConnectionManager } from './ssh-connection-manager' +import { SshPortForwardManager } from './ssh-port-forward' +import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' +import { + OrcadManagedTunnelResumeRecovery, + type OrcadManagedServingCheck, + type OrcadManagedTunnelProbe, + type OrcadManagedTunnelResumeOptions +} from './orcad-managed-tunnel-resume' +import type { OrcadManagedServing } from './orcad-managed-serving' +import { + dropActiveOrcadTunnel, + managedTunnelAccess, + recordActiveOrcadTunnel, + supersededTunnelError, + type ActiveOrcadTunnel +} from './orcad-managed-tunnel-active' +import { checkManagedTunnelServing, type OrcadTunnelServing } from './orcad-managed-tunnel-serving' +import type { getSshTargetRegistryStore } from './ssh-target-registry' +import { + environmentForwardChecks, + forwardToVerifiedOrcad, + PERSISTED_PORT_TARGETING, + type OrcadManagedTunnelTargeting, + type OrcadTunnelStartChecks +} from './orcad-managed-tunnel-target' + +export type OrcadManagedTunnelDependencies = { + getConnectionManager: () => SshConnectionManager | null + getTargetStore: () => ReturnType + forwardManager?: SshPortForwardManager + probeTunnel?: OrcadManagedTunnelProbe + targeting?: OrcadManagedTunnelTargeting + /** Runs after a fresh forward is up; starts a server that stopped (e.g. after idling). */ + ensureServing?: OrcadManagedServingCheck +} + +export class OrcadManagedTunnelManager { + private readonly active = new Map() + private readonly inFlight = new Map>() + private readonly ownershipGenerations = new Map() + private readonly forwards: SshPortForwardManager + private readonly resumeRecovery: OrcadManagedTunnelResumeRecovery + private readonly targeting: OrcadManagedTunnelTargeting + private managerGeneration = 0 + + constructor(private readonly dependencies: OrcadManagedTunnelDependencies) { + this.forwards = + dependencies.forwardManager ?? + new SshPortForwardManager({}, [new OrcadManagedTunnelTransportProvider()]) + this.targeting = dependencies.targeting ?? PERSISTED_PORT_TARGETING + this.resumeRecovery = new OrcadManagedTunnelResumeRecovery({ + active: this.active, + forwards: this.forwards, + getConnectionManager: dependencies.getConnectionManager, + getManagerGeneration: () => this.managerGeneration, + getTargetStore: dependencies.getTargetStore, + inFlight: this.inFlight, + ownershipGenerations: this.ownershipGenerations, + probeTunnel: dependencies.probeTunnel, + targeting: this.targeting, + checkServing: (environment) => this.checkServing(environment) + }) + this.forwards.setCallbacks({ + onForwardClosed: (entry) => { + for (const [environmentId, active] of this.active) { + if (active.forwardId === entry.id) { + this.active.delete(environmentId) + } + } + } + }) + } + + ensure( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null = () => environment, + retryJoined = true + ): Promise { + if (!getRuntimeSshAccess(environment)) { + return Promise.resolve() + } + const pending = this.inFlight.get(environment.id) + if (pending) { + // A joiner did not start that run, so its end (a close(), a disconnect that cancelled its + // connect, a lost exec) is not this caller's answer: build once anew. An auth failure is. + return pending.catch((error: unknown) => + retryJoined && !(error instanceof Error && isAuthError(error)) + ? this.ensure(environment, resolveCurrent, false) + : Promise.reject(error) + ) + } + const operation = this.ensureManagedTunnel(environment, resolveCurrent).finally(() => { + if (this.inFlight.get(environment.id) === operation) { + this.inFlight.delete(environment.id) + } + }) + this.inFlight.set(environment.id, operation) + return operation + } + + async start( + environmentId: string, + target: SshTarget, + connection: SshConnection, + remotePort: number, + checks: OrcadTunnelStartChecks = {} + ): Promise { + if (!target.generation) { + throw new Error('Managed Orca SSH target has no registration generation.') + } + const managerGeneration = this.managerGeneration + const ownershipGeneration = (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + const transportGeneration = connection.getConnectGeneration() + const stillCurrent = (): boolean => + this.managerGeneration === managerGeneration && + this.ownershipGenerations.get(environmentId) === ownershipGeneration && + connection.getConnectGeneration() === transportGeneration + await this.close(environmentId) + if (!stillCurrent()) { + throw supersededTunnelError() + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: 0, + label: 'Managed Orca server', + remotePort, + rereadRemotePort: checks.rereadRemotePort, + verify: checks.verify, + stillCurrent + }) + if (!forward) { + throw supersededTunnelError() + } + recordActiveOrcadTunnel(this.active, environmentId, forward, { + connection, + preferredPort: checks.preferredPort ?? remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + return forward.localPort + } + + async close(environmentId: string): Promise { + this.ownershipGenerations.set( + environmentId, + (this.ownershipGenerations.get(environmentId) ?? 0) + 1 + ) + const active = this.active.get(environmentId) + if (!active) { + return + } + this.active.delete(environmentId) + await this.forwards.removeForwardAndWait(active.forwardId) + } + + dispose(): void { + this.managerGeneration += 1 + this.active.clear() + this.inFlight.clear() + this.ownershipGenerations.clear() + this.resumeRecovery.dispose() + this.forwards.dispose() + } + + /** The server behind the tunnel answers, or is started; a moved port rebuilds the forward. */ + async verifyServing(environment: KnownRuntimeEnvironment): Promise { + if (!this.active.has(environment.id)) { + await this.ensure(environment) + } + const serving = await this.checkServing(environment) + if (serving.rebind) { + await this.ensure(environment) + } + return serving + } + + /** Never rebuilds, so it is safe inside a build: a moved port only drops the forward. */ + checkServing(environment: KnownRuntimeEnvironment): Promise { + return checkManagedTunnelServing({ + environment, + active: this.active, + getTarget: (id) => this.dependencies.getTargetStore()?.getTarget(id), + forwards: this.forwards, + ensureServing: this.dependencies.ensureServing + }) + } + + recoverAfterHostResume(options: OrcadManagedTunnelResumeOptions): Promise { + return this.resumeRecovery.recover(options) + } + + private async ensureManagedTunnel( + environment: KnownRuntimeEnvironment, + resolveCurrent: () => KnownRuntimeEnvironment | null, + rebound = false + ): Promise { + const deployment = getRuntimeSshAccess(environment) + if (!deployment || environment.connectionDependency !== 'ssh-tunnel') { + throw new Error('Managed orcad environment is missing its SSH tunnel dependency.') + } + const targetStore = this.dependencies.getTargetStore() + const connectionManager = this.dependencies.getConnectionManager() + if (!targetStore || !connectionManager) { + throw new Error('SSH is unavailable on this client; the managed Orca server is unverifiable.') + } + const target = targetStore.getTarget(deployment.sshTargetId) + if (!target || target.generation !== deployment.sshTargetGeneration) { + throw new Error( + 'The SSH registration for this managed Orca server was removed or re-created.' + ) + } + if (getManagedOrcadFenceEnvironmentId(target) !== environment.id) { + throw new Error('The SSH target is no longer owned by this managed Orca server.') + } + + const managerGeneration = this.managerGeneration + const ownershipGeneration = this.ownershipGenerations.get(environment.id) ?? 0 + const expected = { environmentId: environment.id, ...deployment } + const stillOwned = (): boolean => { + const currentEnvironment = resolveCurrent() + return ( + this.managerGeneration === managerGeneration && + (this.ownershipGenerations.get(environment.id) ?? 0) === ownershipGeneration && + currentEnvironment?.runtimeId === environment.runtimeId && + (currentEnvironment.pairingRevision ?? currentEnvironment.createdAt) === + (environment.pairingRevision ?? environment.createdAt) && + managedTunnelAccess(currentEnvironment, targetStore.getTarget(target.id), expected) !== null + ) + } + const connection = await connectionManager.connect(target) + if (!stillOwned()) { + throw supersededTunnelError() + } + const transportGeneration = connection.getConnectGeneration() + const active = this.active.get(environment.id) + if ( + active?.connection === connection && + active.transportGeneration === transportGeneration && + active.targetId === target.id && + active.sshTargetGeneration === target.generation && + active.localPort === deployment.localPort && + active.preferredPort === deployment.remotePort + ) { + return + } + const checks = await environmentForwardChecks(this.targeting, { + environment, + target, + connection + }) + if (active && stillOwned()) { + await dropActiveOrcadTunnel(this.active, this.forwards, environment.id, active) + } + if (!stillOwned()) { + throw supersededTunnelError() + } + const forward = await forwardToVerifiedOrcad({ + targetId: target.id, + connection, + forwards: this.forwards, + localPort: deployment.localPort, + label: `Managed Orca server: ${environment.name}`, + ...checks, + stillCurrent: () => stillOwned() && connection.getConnectGeneration() === transportGeneration + }) + if (!forward) { + throw supersededTunnelError() + } + recordActiveOrcadTunnel(this.active, environment.id, forward, { + connection, + preferredPort: deployment.remotePort, + sshTargetGeneration: target.generation, + targetId: target.id, + transportGeneration + }) + if ((await this.checkServing(environment)).rebind && !rebound) { + await this.ensureManagedTunnel(environment, resolveCurrent, true) + } + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-resume.ts b/src/main/ssh/orcad-managed-tunnel-resume.ts new file mode 100644 index 00000000000..32ead10e76d --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-resume.ts @@ -0,0 +1,281 @@ +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment, + type RuntimeSshTunnelLink +} from '../../shared/runtime-environments' +import type { SshConnection } from './ssh-connection' +import type { SshTarget } from '../../shared/ssh-types' +import type { OrcadManagedServing } from './orcad-managed-serving' +import type { SshConnectionManager } from './ssh-connection-manager' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { getSshTargetRegistryStore } from './ssh-target-registry' +import { + dropActiveOrcadTunnel, + managedTunnelAccess, + recordActiveOrcadTunnel, + type ActiveOrcadTunnel +} from './orcad-managed-tunnel-active' +import { + environmentForwardChecks, + forwardToVerifiedOrcad, + type OrcadManagedTunnelTargeting +} from './orcad-managed-tunnel-target' + +export type OrcadManagedTunnelProbe = ( + environment: KnownRuntimeEnvironment, + timeoutMs: number +) => Promise + +export type OrcadManagedServingCheck = (input: { + environment: KnownRuntimeEnvironment + target: SshTarget + connection: SshConnection + remotePort: number +}) => Promise + +export type OrcadManagedTunnelResumeOptions = { + attempts: number + resolveEnvironment: (environmentId: string) => KnownRuntimeEnvironment | null + timeoutMs: number +} + +type ResumeRecoveryDependencies = { + active: Map + forwards: SshPortForwardManager + getConnectionManager: () => SshConnectionManager | null + getManagerGeneration: () => number + getTargetStore: () => ReturnType + inFlight: Map> + ownershipGenerations: Map + probeTunnel?: OrcadManagedTunnelProbe + targeting: OrcadManagedTunnelTargeting + /** Never rebuilds: a server restarted on a new port drops this forward for the next ensure. */ + checkServing?: (environment: KnownRuntimeEnvironment) => Promise +} + +type ResolvedManagedTunnelEnvironment = { + deployment: RuntimeSshTunnelLink + environment: KnownRuntimeEnvironment +} + +export class OrcadManagedTunnelResumeRecovery { + private readonly probeTunnel: OrcadManagedTunnelProbe + private resumeInFlight: Promise | null = null + + constructor(private readonly dependencies: ResumeRecoveryDependencies) { + this.probeTunnel = dependencies.probeTunnel ?? probeManagedOrcadTunnel + } + + dispose(): void { + this.resumeInFlight = null + } + + recover(options: OrcadManagedTunnelResumeOptions): Promise { + if (this.resumeInFlight) { + return this.resumeInFlight + } + const managerGeneration = this.dependencies.getManagerGeneration() + const recoveries = [...this.dependencies.active].map(([environmentId, active]) => + this.recoverActive(environmentId, active, managerGeneration, options) + ) + const operation = Promise.allSettled(recoveries) + .then((results) => { + const failed = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failed) { + throw failed.reason + } + }) + .finally(() => { + if (this.resumeInFlight === operation) { + this.resumeInFlight = null + } + }) + this.resumeInFlight = operation + return operation + } + + private async recoverActive( + environmentId: string, + active: ActiveOrcadTunnel, + managerGeneration: number, + options: OrcadManagedTunnelResumeOptions + ): Promise { + const ownershipGeneration = this.dependencies.ownershipGenerations.get(environmentId) ?? 0 + for (let attempt = 0; attempt < options.attempts; attempt++) { + const resolved = this.resolveEnvironment(environmentId, active, options) + if (!resolved) { + return + } + if (await this.probeTunnel(resolved.environment, options.timeoutMs)) { + return + } + if (!this.stillOwned(environmentId, active, ownershipGeneration, managerGeneration)) { + return + } + } + + const pending = this.dependencies.inFlight.get(environmentId) + if (pending) { + await pending.catch(() => undefined) + } + if (!this.stillOwned(environmentId, active, ownershipGeneration, managerGeneration)) { + return + } + const resolved = this.resolveEnvironment(environmentId, active, options) + const connectionManager = this.dependencies.getConnectionManager() + if ( + !resolved || + !connectionManager || + connectionManager.getConnection(active.targetId) !== active.connection + ) { + return + } + const operation = this.reconnectAndRebuild( + resolved.environment, + active, + connectionManager, + ownershipGeneration, + managerGeneration, + options + ).finally(() => { + if (this.dependencies.inFlight.get(environmentId) === operation) { + this.dependencies.inFlight.delete(environmentId) + } + }) + this.dependencies.inFlight.set(environmentId, operation) + await operation + } + + private async reconnectAndRebuild( + environment: KnownRuntimeEnvironment, + active: ActiveOrcadTunnel, + connectionManager: SshConnectionManager, + ownershipGeneration: number, + managerGeneration: number, + options: OrcadManagedTunnelResumeOptions + ): Promise { + await connectionManager.reconnect(active.targetId) + if ( + !this.stillOwned(environment.id, active, ownershipGeneration, managerGeneration) || + connectionManager !== this.dependencies.getConnectionManager() || + connectionManager.getConnection(active.targetId) !== active.connection || + connectionManager.getState(active.targetId)?.status !== 'connected' || + active.connection.getConnectGeneration() <= active.transportGeneration + ) { + return + } + if (!this.resolveEnvironment(environment.id, active, options)) { + return + } + + const currentActive = this.dependencies.active.get(environment.id) + if (currentActive && currentActive !== active) { + return + } + if (currentActive === active) { + await dropActiveOrcadTunnel( + this.dependencies.active, + this.dependencies.forwards, + environment.id, + active + ) + } + if (!this.stillOwned(environment.id, active, ownershipGeneration, managerGeneration)) { + return + } + + const current = this.resolveEnvironment(environment.id, active, options) + if (!current) { + return + } + const { deployment } = current + const target = this.dependencies.getTargetStore()?.getTarget(active.targetId) + if (!target) { + return + } + const transportGeneration = active.connection.getConnectGeneration() + const checks = await environmentForwardChecks(this.dependencies.targeting, { + environment: current.environment, + target, + connection: active.connection + }) + const forward = await forwardToVerifiedOrcad({ + targetId: active.targetId, + connection: active.connection, + forwards: this.dependencies.forwards, + localPort: deployment.localPort, + label: `Managed Orca server: ${current.environment.name}`, + ...checks, + stillCurrent: () => + active.connection.getConnectGeneration() === transportGeneration && + this.stillOwned(environment.id, active, ownershipGeneration, managerGeneration) && + this.resolveEnvironment(environment.id, active, options) !== null + }) + if (!forward) { + return + } + recordActiveOrcadTunnel(this.dependencies.active, environment.id, forward, { + connection: active.connection, + preferredPort: deployment.remotePort, + sshTargetGeneration: active.sshTargetGeneration, + targetId: active.targetId, + transportGeneration + }) + // A server that idled out while this client slept is started here, not reported as lost. + await this.dependencies.checkServing?.(current.environment) + } + + private resolveEnvironment( + environmentId: string, + active: ActiveOrcadTunnel, + options: OrcadManagedTunnelResumeOptions + ): ResolvedManagedTunnelEnvironment | null { + const environment = options.resolveEnvironment(environmentId) + const deployment = managedTunnelAccess( + environment, + this.dependencies.getTargetStore()?.getTarget(active.targetId), + { + environmentId, + sshTargetId: active.targetId, + sshTargetGeneration: active.sshTargetGeneration, + localPort: active.localPort, + remotePort: active.preferredPort + } + ) + return environment && deployment ? { deployment, environment } : null + } + + private stillOwned( + environmentId: string, + active: ActiveOrcadTunnel, + ownershipGeneration: number, + managerGeneration: number + ): boolean { + const current = this.dependencies.active.get(environmentId) + return ( + this.dependencies.getManagerGeneration() === managerGeneration && + (this.dependencies.ownershipGenerations.get(environmentId) ?? 0) === ownershipGeneration && + (current === active || current === undefined) + ) + } +} + +export async function probeManagedOrcadTunnel( + environment: KnownRuntimeEnvironment, + timeoutMs: number +): Promise { + try { + await sendRemoteRuntimeRequest( + getPreferredPairingOffer(environment), + 'status.get', + undefined, + timeoutMs + ) + return true + } catch { + return false + } +} diff --git a/src/main/ssh/orcad-managed-tunnel-serving.ts b/src/main/ssh/orcad-managed-tunnel-serving.ts new file mode 100644 index 00000000000..a21bfc7392b --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-serving.ts @@ -0,0 +1,46 @@ +/** + * The serving check behind an established managed tunnel. A restarted orcad may bind a port + * other than the one the tunnel forwards to; the forward is then dropped, and the next build + * forwards to the port the new server actually bound. + */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { OrcadManagedServing } from './orcad-managed-serving' +import type { OrcadManagedServingCheck } from './orcad-managed-tunnel-resume' +import type { ActiveOrcadTunnel } from './orcad-managed-tunnel-active' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { SshTarget } from '../../shared/ssh-types' + +export type OrcadTunnelServing = OrcadManagedServing & { rebind?: true } + +export async function checkManagedTunnelServing(input: { + environment: KnownRuntimeEnvironment + active: Map + getTarget: (targetId: string) => SshTarget | null | undefined + forwards: Pick + ensureServing: OrcadManagedServingCheck | undefined +}): Promise { + const { environment, active: tunnels } = input + const active = tunnels.get(environment.id) + const target = active && input.getTarget(active.targetId) + if (!active || !target || !input.ensureServing) { + return { state: 'unverifiable', detail: 'The managed server tunnel is not up.' } + } + const serving = await input.ensureServing({ + environment, + target, + connection: active.connection, + remotePort: active.remotePort + }) + if ( + serving.state !== 'started' || + serving.boundPort === null || + serving.boundPort === active.remotePort + ) { + return serving + } + if (tunnels.get(environment.id) === active) { + tunnels.delete(environment.id) + } + await input.forwards.removeForwardAndWait(active.forwardId) + return { ...serving, rebind: true } +} diff --git a/src/main/ssh/orcad-managed-tunnel-target.test.ts b/src/main/ssh/orcad-managed-tunnel-target.test.ts new file mode 100644 index 00000000000..654a411ca9e --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-target.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshConnection } from './ssh-connection' +import type { SshPortForwardManager } from './ssh-port-forward' +import { + OrcadManagedIdentityError, + type OrcadTunnelIdentity +} from './orcad-managed-tunnel-identity' +import { forwardToVerifiedOrcad } from './orcad-managed-tunnel-target' + +const FOREIGN: OrcadTunnelIdentity = { verdict: 'foreign', detail: '4001: Unauthorized' } + +function setup(verdicts: OrcadTunnelIdentity[], rereadPort: number) { + const addForward = vi.fn( + async (_id: string, _conn: SshConnection, localPort: number, _host: string, port: number) => ({ + id: `forward-${addForward.mock.calls.length}`, + localPort: localPort || 41_000, + remotePort: port + }) + ) + const removeForwardAndWait = vi.fn().mockResolvedValue(null) + const verify = vi.fn(async () => verdicts.shift() ?? { verdict: 'verified' as const }) + const rereadRemotePort = vi.fn().mockResolvedValue(rereadPort) + const args = { + targetId: 'ssh-1', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the forward manager double never reads it. + connection: {} as SshConnection, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the two members forwarding calls. + forwards: { addForward, removeForwardAndWait } as unknown as SshPortForwardManager, + localPort: 46_768, + label: 'Managed Orca server', + remotePort: 6_768, + rereadRemotePort, + verify, + stillCurrent: () => true + } + return { addForward, args, removeForwardAndWait, rereadRemotePort, verify } +} + +describe('forwardToVerifiedOrcad', () => { + it('keeps a forward whose server proves it is this orcad', async () => { + const state = setup([], 6_768) + await expect(forwardToVerifiedOrcad(state.args)).resolves.toMatchObject({ remotePort: 6_768 }) + expect(state.rereadRemotePort).not.toHaveBeenCalled() + }) + + it('re-reads the port after another runtime answers, and follows orcad to it', async () => { + const state = setup([FOREIGN], 58_520) + await expect(forwardToVerifiedOrcad(state.args)).resolves.toMatchObject({ remotePort: 58_520 }) + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward.mock.calls.map((call) => call[4])).toEqual([6_768, 58_520]) + }) + + it('fails with the mismatch, never a silent success, when the port did not move', async () => { + const state = setup([FOREIGN], 6_768) + const result = forwardToVerifiedOrcad(state.args) + await expect(result).rejects.toBeInstanceOf(OrcadManagedIdentityError) + await expect(result).rejects.toThrow(/orcad_identity_mismatch.*remote port 6768/) + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + }) + + it('fails when the re-read port is foreign too, without re-reading forever', async () => { + const state = setup([FOREIGN, FOREIGN], 58_520) + await expect(forwardToVerifiedOrcad(state.args)).rejects.toThrow(/remote port 58520/) + expect(state.rereadRemotePort).toHaveBeenCalledTimes(1) + expect(state.removeForwardAndWait).toHaveBeenCalledTimes(2) + }) + + it('keeps the forward when nothing answers; a stopped server is not a wrong one', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const state = setup([{ verdict: 'unreachable', detail: 'ECONNREFUSED' }], 6_768) + await expect(forwardToVerifiedOrcad(state.args)).resolves.toMatchObject({ remotePort: 6_768 }) + expect(state.removeForwardAndWait).not.toHaveBeenCalled() + warn.mockRestore() + }) + + it('drops the forward and reports nothing when setup was superseded', async () => { + const state = setup([], 6_768) + await expect( + forwardToVerifiedOrcad({ ...state.args, stillCurrent: () => false }) + ).resolves.toBeNull() + expect(state.verify).not.toHaveBeenCalled() + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + }) +}) diff --git a/src/main/ssh/orcad-managed-tunnel-target.ts b/src/main/ssh/orcad-managed-tunnel-target.ts new file mode 100644 index 00000000000..ec1ae964354 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-target.ts @@ -0,0 +1,127 @@ +/** Opens a managed tunnel at the port orcad bound and keeps it only if orcad is what answers. */ +import { + getRuntimeSshAccess, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { + resolveManagedOrcadTunnelPort, + type OrcadTunnelPortInput +} from './orcad-managed-bound-port' +import { + OrcadManagedIdentityError, + verifyManagedOrcadTunnelIdentity, + type OrcadTunnelIdentity +} from './orcad-managed-tunnel-identity' +import type { SshConnection } from './ssh-connection' +import type { PortForwardEntry, SshPortForwardManager } from './ssh-port-forward' + +export type OrcadManagedTunnelTargeting = { + resolveRemotePort: (input: OrcadTunnelPortInput) => Promise + /** Checks the runtime the environment's pairing endpoint (the tunnel's local port) reaches. */ + verifyIdentity: (environment: KnownRuntimeEnvironment) => Promise +} + +export const MANAGED_ORCAD_TUNNEL_TARGETING: OrcadManagedTunnelTargeting = { + resolveRemotePort: resolveManagedOrcadTunnelPort, + verifyIdentity: verifyManagedOrcadTunnelIdentity +} + +/** Test default: the persisted port, and no identity round trip. */ +export const PERSISTED_PORT_TARGETING: OrcadManagedTunnelTargeting = { + resolveRemotePort: async ({ environment }) => getRuntimeSshAccess(environment)?.remotePort ?? 0, + verifyIdentity: async () => ({ verdict: 'verified' }) +} + +export type VerifiedOrcadForwardArgs = { + targetId: string + connection: SshConnection + forwards: SshPortForwardManager + /** 0 picks a free local port. */ + localPort: number + label: string + remotePort: number + /** Defaults to the same port, so a foreign answer fails at once. */ + rereadRemotePort?: () => Promise + /** Defaults to trusting the forward, for callers that verify on their own. */ + verify?: (localPort: number) => Promise + stillCurrent: () => boolean +} + +export type OrcadTunnelStartChecks = Pick< + VerifiedOrcadForwardArgs, + 'rereadRemotePort' | 'verify' +> & { + /** The persisted launch port, when the forward targets a port orcad fell back to. */ + preferredPort?: number +} + +/** The manager's targeting, bound to one environment's tunnel, with the port it reads now. */ +export async function environmentForwardChecks( + targeting: OrcadManagedTunnelTargeting, + input: OrcadTunnelPortInput +): Promise>> { + return { + remotePort: await targeting.resolveRemotePort(input), + rereadRemotePort: () => targeting.resolveRemotePort(input), + verify: () => targeting.verifyIdentity(input.environment) + } +} + +/** + * Null when `stillCurrent` turned false mid-setup; the forward is already removed then. + * An answer from another runtime re-reads the port once, since orcad may have restarted onto a + * new one, and throws if the port did not move or the new one is foreign too. + */ +export async function forwardToVerifiedOrcad( + args: VerifiedOrcadForwardArgs +): Promise { + let remotePort = args.remotePort + for (let attempt = 0; ; attempt++) { + const forward = await addCurrentForward(args, remotePort) + if (!forward) { + return null + } + const identity = args.verify + ? await args.verify(forward.localPort) + : ({ verdict: 'verified' } as const) + if (identity.verdict !== 'foreign') { + if (identity.verdict === 'unreachable') { + // Why not fail: a server that is down or still starting is not a wrong server. + console.warn( + `[ssh] Managed Orca server did not answer through its tunnel: ${identity.detail}` + ) + } + return forward + } + await args.forwards.removeForwardAndWait(forward.id) + const reread = + attempt === 0 && args.rereadRemotePort ? await args.rereadRemotePort() : remotePort + if (reread === remotePort || !args.stillCurrent()) { + throw new OrcadManagedIdentityError(remotePort, identity.detail) + } + remotePort = reread + } +} + +async function addCurrentForward( + args: VerifiedOrcadForwardArgs, + remotePort: number +): Promise { + const forward = await args.forwards.addForward( + args.targetId, + args.connection, + args.localPort, + '127.0.0.1', + remotePort, + args.label + ) + if (args.localPort !== 0 && forward.localPort !== args.localPort) { + await args.forwards.removeForwardAndWait(forward.id) + throw new Error('Managed Orca tunnel bound an unexpected local port.') + } + if (!args.stillCurrent()) { + await args.forwards.removeForwardAndWait(forward.id) + return null + } + return forward +} diff --git a/src/main/ssh/orcad-managed-tunnel-transport.test.ts b/src/main/ssh/orcad-managed-tunnel-transport.test.ts new file mode 100644 index 00000000000..64650eb4660 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-transport.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from 'vitest' +import { OrcadManagedTunnelTransportProvider } from './orcad-managed-tunnel-transport' +import { knownOrcadTunnelTransport } from './orcad-tunnel-transport-memo' +import type { SshConnection } from './ssh-connection' +import type { + PortForwardStartOptions, + SshPortForwardProvider, + StartedPortForward +} from './ssh-port-forward-provider' +import type { TcpForwardingVerdict } from './ssh-tcp-forwarding-probe' + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the selector passes the connection through untouched. +const conn = {} as SshConnection +const options: PortForwardStartOptions = { + id: 'pf-1', + connectionId: 'ssh-1', + localHost: '127.0.0.1', + localPort: 46_001, + remoteHost: '127.0.0.1', + remotePort: 6768 +} + +function provider(name: string, canHandle = true): SshPortForwardProvider & { name: string } { + return { + name, + canHandle: () => canHandle, + start: vi.fn(async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the test reads only which provider answered. + return { entry: { id: name } } as unknown as StartedPortForward + }) + } +} + +function select(verdict: TcpForwardingVerdict) { + const forward = provider('forward') + const stdio = provider('stdio') + const transport = new OrcadManagedTunnelTransportProvider({ + forwards: [provider('unusable', false), forward], + stdio, + probe: vi.fn(async () => verdict) + }) + return { transport, forward, stdio } +} + +describe('the managed tunnel’s transport', () => { + it('takes the stdio bridge only where sshd refuses forwarding', async () => { + const refused = select('refused') + await expect(refused.transport.start(conn, options)).resolves.toMatchObject({ + entry: { id: 'stdio' } + }) + expect(refused.stdio.start).toHaveBeenCalledWith(conn, options) + expect(refused.forward.start).not.toHaveBeenCalled() + expect(knownOrcadTunnelTransport('ssh-1')).toBe('stdio_bridge') + }) + + it('keeps the forward when forwarding is allowed or the answer is unverifiable', async () => { + for (const verdict of ['allowed', 'unverifiable'] as const) { + const chosen = select(verdict) + await expect(chosen.transport.start(conn, options)).resolves.toMatchObject({ + entry: { id: 'forward' } + }) + expect(chosen.stdio.start).not.toHaveBeenCalled() + expect(knownOrcadTunnelTransport('ssh-1')).toBe('tcp_forward') + } + }) +}) diff --git a/src/main/ssh/orcad-managed-tunnel-transport.ts b/src/main/ssh/orcad-managed-tunnel-transport.ts new file mode 100644 index 00000000000..b8f02b495bc --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel-transport.ts @@ -0,0 +1,52 @@ +/** + * The managed tunnel's transport choice, made each time a tunnel starts: the SSH local forward + * wherever the host allows it, and the stdio bridge where its sshd refuses forwarding. + */ +import { OrcadStdioBridgePortForwardProvider } from './orcad-stdio-bridge-provider' +import { rememberOrcadTunnelTransport } from './orcad-tunnel-transport-memo' +import type { SshConnection } from './ssh-connection' +import type { + PortForwardStartOptions, + SshPortForwardProvider, + StartedPortForward +} from './ssh-port-forward-provider' +import { probeTcpForwarding, type TcpForwardingVerdict } from './ssh-tcp-forwarding-probe' +import { Ssh2PortForwardProvider } from './ssh2-port-forward-provider' +import { SystemSshPortForwardProvider } from './system-ssh-port-forward-provider' + +type OrcadManagedTunnelTransportDependencies = { + forwards: SshPortForwardProvider[] + stdio: SshPortForwardProvider + probe: (conn: SshConnection, port: number) => Promise +} + +export class OrcadManagedTunnelTransportProvider implements SshPortForwardProvider { + constructor( + private readonly dependencies: OrcadManagedTunnelTransportDependencies = { + forwards: [new Ssh2PortForwardProvider(), new SystemSshPortForwardProvider()], + stdio: new OrcadStdioBridgePortForwardProvider(), + probe: probeTcpForwarding + } + ) {} + + canHandle(conn: SshConnection): boolean { + return this.dependencies.forwards.some((provider) => provider.canHandle(conn)) + } + + async start(conn: SshConnection, options: PortForwardStartOptions): Promise { + const { forwards, stdio, probe } = this.dependencies + // Why only on refusal: an unverifiable answer keeps the forward, as before this transport. + if ((await probe(conn, options.remotePort)) === 'refused' && stdio.canHandle(conn)) { + const started = await stdio.start(conn, options) + rememberOrcadTunnelTransport(options.connectionId, 'stdio_bridge') + return started + } + const forward = forwards.find((provider) => provider.canHandle(conn)) + if (!forward) { + throw new Error('SSH connection is not established') + } + const started = await forward.start(conn, options) + rememberOrcadTunnelTransport(options.connectionId, 'tcp_forward') + return started + } +} diff --git a/src/main/ssh/orcad-managed-tunnel.test.ts b/src/main/ssh/orcad-managed-tunnel.test.ts new file mode 100644 index 00000000000..57d5e6f7c97 --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel.test.ts @@ -0,0 +1,735 @@ +import { describe, expect, it, vi } from 'vitest' +import { + createEnvironmentFromPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { PAIRING_OFFER_VERSION } from '../../shared/pairing' +import type { SshTarget } from '../../shared/ssh-types' +import type { SshConnection } from './ssh-connection' +import type { SshConnectionManager } from './ssh-connection-manager' +import type { SshConnectionStore } from './ssh-connection-store' +import type { SshPortForwardManager } from './ssh-port-forward' +import { OrcadManagedTunnelManager } from './orcad-managed-tunnel' +import type { OrcadManagedTunnelTargeting } from './orcad-managed-tunnel-target' +import { createCancelledConnectAttemptError } from './ssh-connect-attempt-cancellation' + +function createEnvironment(linkKind: 'orcadDeployment' | 'sshAccess'): KnownRuntimeEnvironment { + const paired = createEnvironmentFromPairingOffer({ + id: 'environment-1', + name: 'Managed server', + now: 1, + offer: { + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768', + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }, + connectionDependency: 'ssh-tunnel' + }) + const access = { + sshTargetId: 'ssh-1', + sshTargetGeneration: 7, + localPort: 46_768, + remotePort: 6_768 + } + return linkKind === 'orcadDeployment' + ? { ...paired, orcadDeployment: access } + : { + ...paired, + sshAccess: { + ...access, + endpointId: paired.preferredEndpointId, + previousPreferredEndpointId: paired.preferredEndpointId + } + } +} + +function setup(overrides: Partial = {}, targeting?: OrcadManagedTunnelTargeting) { + const target: SshTarget = { + id: 'ssh-1', + label: 'Managed server', + host: 'example.com', + port: 22, + username: 'deploy', + generation: 7, + orcadFence: { environmentId: 'environment-1' }, + ...overrides + } + let transportGeneration = 3 + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls. + const connection = { + getConnectGeneration: vi.fn(() => transportGeneration) + } as unknown as SshConnection + const connect = vi.fn().mockResolvedValue(connection) + const reconnect = vi.fn().mockImplementation(async () => { + transportGeneration += 1 + }) + const getConnection = vi.fn(() => connection) + const getState = vi.fn(() => ({ status: 'connected' })) + const probeTunnel = vi.fn().mockResolvedValue(true) + const addForward = vi + .fn() + .mockImplementation( + async ( + connectionId: string, + _connection: SshConnection, + localPort: number, + _remoteHost: string, + remotePort: number + ) => ({ + id: `forward-${addForward.mock.calls.length}`, + connectionId, + localPort, + remoteHost: '127.0.0.1', + remotePort + }) + ) + const removeForwardAndWait = vi.fn().mockResolvedValue(null) + const ensureServing = vi.fn().mockResolvedValue({ state: 'serving' }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls. + const forwardManager = { + setCallbacks: vi.fn(), + addForward, + removeForwardAndWait, + dispose: vi.fn() + } as unknown as SshPortForwardManager + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a test double for the members the tunnel manager calls. + const connectionManager = { + connect, + getConnection, + getState, + reconnect + } as unknown as SshConnectionManager + const manager = new OrcadManagedTunnelManager({ + getConnectionManager: () => connectionManager, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the tunnel manager reads only getTarget. + getTargetStore: () => ({ getTarget: vi.fn(() => target) }) as unknown as SshConnectionStore, + forwardManager, + probeTunnel, + targeting, + ensureServing + }) + return { + addForward, + connect, + connection, + ensureServing, + getConnection, + getState, + manager, + probeTunnel, + reconnect, + removeForwardAndWait, + target, + setTransportGeneration: (generation: number) => { + transportGeneration = generation + } + } +} + +describe('OrcadManagedTunnelManager initial binding', () => { + it.each(['close', 'dispose', 'reconnect'] as const)( + 'removes a late initial forward after %s supersedes setup', + async (action) => { + const state = setup() + state.addForward.mockImplementationOnce(async () => { + if (action === 'close') { + await state.manager.close('environment-1') + } + if (action === 'dispose') { + state.manager.dispose() + } + if (action === 'reconnect') { + state.setTransportGeneration(4) + } + return { id: 'late-initial-forward', localPort: 46_768, remotePort: 6_768 } + }) + + await expect( + state.manager.start('environment-1', state.target, state.connection, 6_768) + ).rejects.toThrow('superseded') + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('late-initial-forward') + await state.manager.close('environment-1') + expect(state.removeForwardAndWait).toHaveBeenCalledTimes(1) + } + ) + + it('does not open a forward when canceled while the prior forward is closing', async () => { + const state = setup() + await state.manager.ensure(createEnvironment('sshAccess')) + state.removeForwardAndWait.mockImplementationOnce(async () => { + await state.manager.close('environment-1') + }) + + await expect( + state.manager.start('environment-1', state.target, state.connection, 6_768) + ).rejects.toThrow('superseded') + + expect(state.addForward).toHaveBeenCalledTimes(1) + }) + + it('discards an older binding without closing the newer tunnel', async () => { + const state = setup() + let finishFirst!: () => void + state.addForward.mockImplementationOnce( + () => + new Promise((resolve) => { + finishFirst = () => resolve({ id: 'old-forward', localPort: 46_768, remotePort: 6_768 }) + }) + ) + const first = state.manager.start('environment-1', state.target, state.connection, 6_768) + const rejected = expect(first).rejects.toThrow('superseded') + await vi.waitFor(() => expect(state.addForward).toHaveBeenCalledOnce()) + + await state.manager.start('environment-1', state.target, state.connection, 6_768) + finishFirst() + await rejected + + expect(state.removeForwardAndWait.mock.calls).toEqual([['old-forward']]) + await state.manager.close('environment-1') + expect(state.removeForwardAndWait.mock.calls).toEqual([['old-forward'], ['forward-2']]) + }) +}) + +describe.each(['orcadDeployment', 'sshAccess'] as const)( + 'OrcadManagedTunnelManager (%s)', + (linkKind) => { + const environment = () => createEnvironment(linkKind) + const resumeOptions = () => ({ + attempts: 2, + resolveEnvironment: () => environment(), + timeoutMs: 5_000 + }) + it('connects through the raw SSH manager and creates the exact loopback forward', async () => { + const state = setup() + + await state.manager.ensure(environment()) + + expect(state.connect).toHaveBeenCalledOnce() + if (linkKind === 'sshAccess') { + expect(environment().orcadDeployment).toBeUndefined() + } + expect(state.addForward).toHaveBeenCalledWith( + 'ssh-1', + expect.anything(), + 46_768, + '127.0.0.1', + 6_768, + 'Managed Orca server: Managed server' + ) + }) + + it('reuses a tunnel only for the same SSH transport generation', async () => { + const state = setup() + + await state.manager.ensure(environment()) + await state.manager.ensure(environment()) + expect(state.addForward).toHaveBeenCalledOnce() + + state.setTransportGeneration(4) + await state.manager.ensure(environment()) + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward).toHaveBeenCalledTimes(2) + }) + + it.each(['ensure', 'resume'] as const)( + 'discards a forward when SSH reconnects during %s binding', + async (operation) => { + const state = setup() + if (operation === 'resume') { + await state.manager.ensure(environment()) + state.probeTunnel.mockResolvedValue(false) + } + state.addForward.mockImplementationOnce(async () => { + state.setTransportGeneration(9) + return { id: 'stale-forward', localPort: 46_768, remotePort: 6_768 } + }) + await (operation === 'resume' + ? state.manager.recoverAfterHostResume(resumeOptions()) + : expect(state.manager.ensure(environment())).rejects.toThrow('superseded')) + expect(state.removeForwardAndWait).toHaveBeenCalledWith('stale-forward') + state.probeTunnel.mockClear() + await state.manager.recoverAfterHostResume(resumeOptions()) + expect(state.probeTunnel).not.toHaveBeenCalled() + await state.manager.ensure(environment()) + expect(state.addForward).toHaveBeenCalledTimes(operation === 'resume' ? 3 : 2) + } + ) + + it('fails closed when the SSH registration generation changed', async () => { + const state = setup({ generation: 8 }) + + await expect(state.manager.ensure(environment())).rejects.toThrow('removed or re-created') + expect(state.connect).not.toHaveBeenCalled() + }) + + it('fails closed when another environment owns the target', async () => { + const state = setup({ orcadFence: { environmentId: 'environment-2' } }) + + await expect(state.manager.ensure(environment())).rejects.toThrow('no longer owned') + expect(state.connect).not.toHaveBeenCalled() + }) + + it('coalesces concurrent tunnel preflights', async () => { + const state = setup() + let finishConnect!: () => void + state.connect.mockImplementationOnce( + () => + new Promise((resolve) => { + finishConnect = () => resolve(state.connection) + }) + ) + + const first = state.manager.ensure(environment()) + const second = state.manager.ensure(environment()) + finishConnect() + await Promise.all([first, second]) + + expect(state.connect).toHaveBeenCalledOnce() + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('does not create a forward after close cancels an in-flight connection', async () => { + const state = setup() + state.connect.mockImplementationOnce(async () => { + await state.manager.close('environment-1') + return state.connection + }) + + await expect(state.manager.ensure(environment())).rejects.toThrow('superseded') + + expect(state.addForward).not.toHaveBeenCalled() + }) + + it('rechecks SSH ownership after connection resolves', async () => { + const state = setup() + state.connect.mockImplementationOnce(async () => { + state.target.orcadFence = { environmentId: 'environment-2' } + return state.connection + }) + + await expect(state.manager.ensure(environment())).rejects.toThrow('superseded') + + expect(state.addForward).not.toHaveBeenCalled() + }) + + it('builds a fresh tunnel for a caller that joins a run a close() superseded', async () => { + const state = setup() + let finishConnect!: () => void + state.connect.mockImplementationOnce( + () => + new Promise((resolve) => { + finishConnect = () => resolve(state.connection) + }) + ) + const first = state.manager.ensure(environment()) + await state.manager.close('environment-1') + const joined = state.manager.ensure(environment()) + finishConnect() + await expect(first).rejects.toThrow('superseded') + await expect(joined).resolves.toBeUndefined() + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('lets a caller that joined a run a transport change overtook build its own', async () => { + const state = setup() + state.addForward.mockImplementationOnce(async () => { + state.setTransportGeneration(9) + return { id: 'stale-forward', localPort: 46_768, remotePort: 6_768 } + }) + const first = state.manager.ensure(environment()) + const joined = state.manager.ensure(environment()) + await expect(first).rejects.toThrow('superseded') + await expect(joined).resolves.toBeUndefined() + expect(state.addForward).toHaveBeenCalledTimes(2) + }) + + it('lets a caller that joined a run a disconnect cancelled build its own, once', async () => { + const state = setup() + let cancel!: () => void + state.connect.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + cancel = () => reject(createCancelledConnectAttemptError()) + }) + ) + const restore = state.manager.ensure(environment()) + const connect = state.manager.ensure(environment()) + cancel() + await expect(restore).rejects.toThrow('SSH connection attempt was cancelled') + await expect(connect).resolves.toBeUndefined() + expect(state.connect).toHaveBeenCalledTimes(2) + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('hands a joiner the joined run’s auth failure instead of prompting again', async () => { + const state = setup() + const auth = new Error('All configured authentication methods failed') + state.connect.mockRejectedValueOnce(auth) + const first = state.manager.ensure(environment()) + const joined = state.manager.ensure(environment()) + await expect(first).rejects.toBe(auth) + await expect(joined).rejects.toBe(auth) + expect(state.connect).toHaveBeenCalledOnce() + }) + + it('re-reads the saved environment after connection rather than trusting its initial snapshot', async () => { + const state = setup() + const original = environment() + let current = original + state.connect.mockImplementationOnce(async () => { + current = { + ...original, + pairingRevision: (original.pairingRevision ?? original.createdAt) + 1 + } + return state.connection + }) + + await expect(state.manager.ensure(original, () => current)).rejects.toThrow('superseded') + + expect(state.addForward).not.toHaveBeenCalled() + }) + + it('removes a newly bound forward if the saved environment disappears during binding', async () => { + const state = setup() + const original = environment() + let current: KnownRuntimeEnvironment | null = original + state.addForward.mockImplementationOnce(async () => { + current = null + return { id: 'late-forward', localPort: 46_768, remotePort: 6_768 } + }) + + await expect(state.manager.ensure(original, () => current)).rejects.toThrow('superseded') + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('late-forward') + }) + + it('removes a newly bound forward if the environment closes during binding', async () => { + const state = setup() + state.addForward.mockImplementationOnce(async () => { + await state.manager.close('environment-1') + return { id: 'late-forward', localPort: 46_768, remotePort: 6_768 } + }) + + await expect(state.manager.ensure(environment())).rejects.toThrow('superseded') + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('late-forward') + await state.manager.recoverAfterHostResume(resumeOptions()) + expect(state.probeTunnel).not.toHaveBeenCalled() + }) + + it('checks that the server is running only when it sets up a fresh forward', async () => { + const state = setup() + + await state.manager.ensure(environment()) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledWith({ + environment: expect.objectContaining({ id: 'environment-1' }), + target: state.target, + connection: state.connection, + remotePort: 6_768 + }) + + state.setTransportGeneration(4) + await state.manager.ensure(environment()) + expect(state.ensureServing).toHaveBeenCalledTimes(2) + }) + + it('starts a server that stopped while the client slept once the tunnel is rebuilt', async () => { + const state = setup() + await state.manager.ensure(environment()) + state.probeTunnel.mockResolvedValue(false) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.reconnect).toHaveBeenCalledOnce() + expect(state.ensureServing).toHaveBeenCalledTimes(2) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ target: state.target, remotePort: 6_768 }) + ) + }) + + it('keeps a healthy managed tunnel intact after host resume', async () => { + const state = setup() + await state.manager.ensure(environment()) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.probeTunnel).toHaveBeenCalledOnce() + expect(state.probeTunnel).toHaveBeenCalledWith( + expect.objectContaining({ id: 'environment-1' }), + 5_000 + ) + expect(state.reconnect).not.toHaveBeenCalled() + expect(state.removeForwardAndWait).not.toHaveBeenCalled() + }) + + it('retries a failed wake probe before reconnecting', async () => { + const state = setup() + state.probeTunnel.mockResolvedValueOnce(false).mockResolvedValueOnce(true) + await state.manager.ensure(environment()) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.probeTunnel).toHaveBeenCalledTimes(2) + expect(state.reconnect).not.toHaveBeenCalled() + }) + + it('reconnects and rebuilds the exact persisted port after failed wake probes', async () => { + const state = setup() + state.probeTunnel.mockResolvedValue(false) + await state.manager.ensure(environment()) + + await state.manager.recoverAfterHostResume(resumeOptions()) + + expect(state.reconnect).toHaveBeenCalledOnce() + expect(state.reconnect).toHaveBeenCalledWith('ssh-1') + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 6_768, + 'Managed Orca server: Managed server' + ) + + await state.manager.ensure(environment()) + expect(state.addForward).toHaveBeenCalledTimes(2) + }) + + it('coalesces concurrent host-resume recoveries', async () => { + const state = setup() + let finishProbe!: () => void + state.probeTunnel.mockImplementationOnce( + () => + new Promise((resolve) => { + finishProbe = () => resolve(true) + }) + ) + await state.manager.ensure(environment()) + + const first = state.manager.recoverAfterHostResume(resumeOptions()) + const second = state.manager.recoverAfterHostResume(resumeOptions()) + expect(second).toBe(first) + finishProbe() + await Promise.all([first, second]) + + expect(state.probeTunnel).toHaveBeenCalledOnce() + }) + + it('does not reconnect an environment closed during its wake probe', async () => { + const state = setup() + let finishProbe!: () => void + state.probeTunnel.mockImplementationOnce( + () => + new Promise((resolve) => { + finishProbe = () => resolve(false) + }) + ) + await state.manager.ensure(environment()) + + const recovery = state.manager.recoverAfterHostResume({ + ...resumeOptions(), + attempts: 1 + }) + await vi.waitFor(() => expect(state.probeTunnel).toHaveBeenCalledOnce()) + await state.manager.close('environment-1') + finishProbe() + await recovery + + expect(state.reconnect).not.toHaveBeenCalled() + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('does not rebuild when reconnect did not establish a newer transport', async () => { + const state = setup() + state.probeTunnel.mockResolvedValue(false) + state.reconnect.mockImplementationOnce(async () => undefined) + await state.manager.ensure(environment()) + + await state.manager.recoverAfterHostResume({ + ...resumeOptions(), + attempts: 1 + }) + + expect(state.reconnect).toHaveBeenCalledOnce() + expect(state.removeForwardAndWait).not.toHaveBeenCalled() + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('does not rebuild after re-pairing removes SSH access during reconnect', async () => { + const state = setup() + let current = environment() + state.probeTunnel.mockResolvedValue(false) + await state.manager.ensure(current) + state.reconnect.mockImplementationOnce(async () => { + state.setTransportGeneration(4) + current = { ...current, orcadDeployment: undefined, sshAccess: undefined } + }) + + await state.manager.recoverAfterHostResume({ + ...resumeOptions(), + resolveEnvironment: () => current + }) + + expect(state.addForward).toHaveBeenCalledOnce() + expect(state.removeForwardAndWait).not.toHaveBeenCalled() + }) + + it('removes the replacement forward when re-pairing happens during resume binding', async () => { + const state = setup() + let current = environment() + state.probeTunnel.mockResolvedValue(false) + await state.manager.ensure(current) + state.addForward.mockImplementationOnce(async () => { + current = { ...current, orcadDeployment: undefined, sshAccess: undefined } + return { id: 'late-forward', localPort: 46_768, remotePort: 6_768 } + }) + + await state.manager.recoverAfterHostResume({ + ...resumeOptions(), + resolveEnvironment: () => current + }) + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('late-forward') + }) + } +) + +describe('OrcadManagedTunnelManager bound port', () => { + function boundPortSetup(ports: number[], verdicts: ('verified' | 'foreign')[] = []) { + const resolveRemotePort = vi.fn(async () => ports.shift() ?? 6_768) + const verifyIdentity = vi.fn(async () => + verdicts.shift() === 'foreign' + ? { verdict: 'foreign' as const, detail: '4001: Unauthorized' } + : { verdict: 'verified' as const } + ) + return { + ...setup({}, { resolveRemotePort, verifyIdentity }), + resolveRemotePort, + verifyIdentity + } + } + + it('forwards to the port orcad bound when another runtime holds the preferred one', async () => { + const state = boundPortSetup([58_520]) + await state.manager.ensure(createEnvironment('orcadDeployment')) + + expect(state.addForward).toHaveBeenCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + expect(state.verifyIdentity).toHaveBeenCalledOnce() + }) + + it('follows a restarted server to the port it bound, within the same ensure', async () => { + const state = boundPortSetup([6_768, 58_520]) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + await state.manager.ensure(createEnvironment('orcadDeployment')) + + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + expect(state.ensureServing).toHaveBeenLastCalledWith( + expect.objectContaining({ remotePort: 58_520 }) + ) + }) + + it('rebuilds after an explicit check finds the server restarted on another port', async () => { + const state = boundPortSetup([6_768, 58_520]) + const environment = createEnvironment('orcadDeployment') + await state.manager.ensure(environment) + state.ensureServing + .mockResolvedValueOnce({ state: 'started', boundPort: 58_520 }) + .mockResolvedValue({ state: 'serving' }) + + await expect(state.manager.verifyServing(environment)).resolves.toMatchObject({ + state: 'started', + rebind: true + }) + expect(state.addForward).toHaveBeenCalledTimes(2) + expect(state.addForward).toHaveBeenLastCalledWith( + 'ssh-1', + state.connection, + 46_768, + '127.0.0.1', + 58_520, + 'Managed Orca server: Managed server' + ) + }) + + it('reuses a verified tunnel without reading the port again', async () => { + const state = boundPortSetup([58_520]) + const environment = createEnvironment('orcadDeployment') + await state.manager.ensure(environment) + await state.manager.ensure(environment) + + expect(state.resolveRemotePort).toHaveBeenCalledOnce() + expect(state.addForward).toHaveBeenCalledOnce() + }) + + it('fails the connect, leaving no forward, when the bound port serves another runtime', async () => { + const state = boundPortSetup([6_768, 6_768], ['foreign']) + await expect(state.manager.ensure(createEnvironment('orcadDeployment'))).rejects.toThrow( + 'orcad_identity_mismatch' + ) + expect(state.removeForwardAndWait).toHaveBeenCalledWith('forward-1') + await state.manager.close('environment-1') + expect(state.removeForwardAndWait).toHaveBeenCalledOnce() + }) + + it('rebuilds at the re-read port after a host resume', async () => { + const state = boundPortSetup([58_520, 60_001]) + const environment = createEnvironment('orcadDeployment') + await state.manager.ensure(environment) + state.probeTunnel.mockResolvedValue(false) + + await state.manager.recoverAfterHostResume({ + attempts: 1, + resolveEnvironment: () => environment, + timeoutMs: 5_000 + }) + + expect(state.addForward.mock.calls.map((call) => call[4])).toEqual([58_520, 60_001]) + }) + + it('starts a deploy tunnel at the bound port but keeps the preferred port for reuse', async () => { + const state = boundPortSetup([]) + const localPort = await state.manager.start( + 'environment-1', + state.target, + state.connection, + 58_520, + { preferredPort: 6_768 } + ) + expect(state.addForward.mock.calls[0]?.[4]).toBe(58_520) + state.addForward.mockClear() + // The persisted link names the preferred port, so the next ensure reuses this forward. + await state.manager.ensure({ + ...createEnvironment('orcadDeployment'), + orcadDeployment: { + sshTargetId: 'ssh-1', + sshTargetGeneration: 7, + localPort, + remotePort: 6_768 + } + }) + expect(state.addForward).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-managed-tunnel.ts b/src/main/ssh/orcad-managed-tunnel.ts new file mode 100644 index 00000000000..f0b6509e38c --- /dev/null +++ b/src/main/ssh/orcad-managed-tunnel.ts @@ -0,0 +1,71 @@ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { resolveEnvironment } from '../../shared/runtime-environment-store' +import type { SshTarget } from '../../shared/ssh-types' +import type { SshConnection } from './ssh-connection' +import { probeManagedOrcadTunnel } from './orcad-managed-tunnel-resume' +import { ensureManagedOrcadServing } from './orcad-managed-serving' +import { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' +import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' +import { + MANAGED_ORCAD_TUNNEL_TARGETING, + type OrcadTunnelStartChecks +} from './orcad-managed-tunnel-target' + +export { OrcadManagedTunnelManager } from './orcad-managed-tunnel-manager' + +const managedTunnels = new OrcadManagedTunnelManager({ + getConnectionManager: getSshConnectionManager, + getTargetStore: getSshTargetRegistryStore, + targeting: MANAGED_ORCAD_TUNNEL_TARGETING, + ensureServing: (input) => ensureManagedOrcadServing({ ...input, probe: probeManagedOrcadTunnel }) +}) + +export async function ensureOrcadManagedTunnel( + userDataPath: string, + selector: string +): Promise { + const environment = resolveEnvironment(userDataPath, selector) + await managedTunnels.ensure(environment, () => + resolveEnvironmentOrNull(userDataPath, environment.id) + ) +} + +function resolveEnvironmentOrNull(userDataPath: string, id: string) { + try { + return resolveEnvironment(userDataPath, id) + } catch { + return null + } +} + +export function verifyManagedTunnelServing(environment: KnownRuntimeEnvironment) { + return managedTunnels.verifyServing(environment) +} + +export function disposeOrcadManagedTunnels(): void { + managedTunnels.dispose() +} + +export function recoverOrcadManagedTunnelsAfterHostResume( + userDataPath: string, + options: { attempts: number; timeoutMs: number } +): Promise { + return managedTunnels.recoverAfterHostResume({ + ...options, + resolveEnvironment: (environmentId) => resolveEnvironmentOrNull(userDataPath, environmentId) + }) +} + +export function startOrcadManagedTunnel( + environmentId: string, + target: SshTarget, + connection: SshConnection, + remotePort: number, + checks?: OrcadTunnelStartChecks +): Promise { + return managedTunnels.start(environmentId, target, connection, remotePort, checks) +} + +export function closeOrcadManagedTunnel(environmentId: string): Promise { + return managedTunnels.close(environmentId) +} diff --git a/src/main/ssh/orcad-managed-update-deferrals.test.ts b/src/main/ssh/orcad-managed-update-deferrals.test.ts new file mode 100644 index 00000000000..f3d95947c7b --- /dev/null +++ b/src/main/ssh/orcad-managed-update-deferrals.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + clearManagedOrcadUpdateDeferral, + currentManagedOrcadUpdateDeferral, + recordManagedOrcadUpdateDeferral +} from './orcad-managed-update-deferrals' + +function defer(candidateVersion: string): void { + recordManagedOrcadUpdateDeferral('env-1', { + outcome: 'deferred', + candidateVersion, + code: 'orcad_activation_fence_busy', + reason: 'Another update holds this host.', + forceable: false + }) +} + +afterEach(() => clearManagedOrcadUpdateDeferral('env-1')) + +describe('a recorded update deferral', () => { + it('is dropped once another desktop activated its candidate', () => { + defer('0.1.0+9150d77ce015') + expect(currentManagedOrcadUpdateDeferral('env-1', '0.1.0+9150d77ce015')).toBeNull() + expect(currentManagedOrcadUpdateDeferral('env-1', '0.1.0+fb47a73e01d4')).toBeNull() + }) + + it('is dropped once the host runs a newer release', () => { + defer('0.1.0+9150d77ce015') + expect(currentManagedOrcadUpdateDeferral('env-1', '0.2.0+aaaaaaaaaaaa')).toBeNull() + }) + + it('stays while the host still runs an older or different build', () => { + defer('0.1.0+9150d77ce015') + expect(currentManagedOrcadUpdateDeferral('env-1', '0.1.0+fb47a73e01d4')).toMatchObject({ + candidateVersion: '0.1.0+9150d77ce015' + }) + expect(currentManagedOrcadUpdateDeferral('env-1', null)).not.toBeNull() + }) +}) diff --git a/src/main/ssh/orcad-managed-update-deferrals.ts b/src/main/ssh/orcad-managed-update-deferrals.ts new file mode 100644 index 00000000000..e05dba6e3bc --- /dev/null +++ b/src/main/ssh/orcad-managed-update-deferrals.ts @@ -0,0 +1,44 @@ +/** The last update each managed server deferred in this session, so status can report it. */ +import type { OrcadManagedDeferral } from '../../shared/orcad-managed-runtime' +import { compareAppVersions } from '../../shared/app-version' + +type RecordedDeferral = OrcadManagedDeferral & { deferredAt: string } + +const deferrals = new Map() + +export function recordManagedOrcadUpdateDeferral( + environmentId: string, + deferral: OrcadManagedDeferral, + now = new Date() +): void { + deferrals.set(environmentId, { ...deferral, deferredAt: now.toISOString() }) +} + +export function clearManagedOrcadUpdateDeferral(environmentId: string): void { + deferrals.delete(environmentId) +} + +export function readManagedOrcadUpdateDeferral(environmentId: string): RecordedDeferral | null { + return deferrals.get(environmentId) ?? null +} + +/** + * The deferral, unless the host already runs its candidate or a newer release: another desktop's + * update can land it after this one deferred, and nothing here would otherwise forget it. + */ +export function currentManagedOrcadUpdateDeferral( + environmentId: string, + activeVersion: string | null +): RecordedDeferral | null { + const deferral = readManagedOrcadUpdateDeferral(environmentId) + if ( + deferral && + activeVersion && + (activeVersion === deferral.candidateVersion || + compareAppVersions(activeVersion, deferral.candidateVersion) > 0) + ) { + clearManagedOrcadUpdateDeferral(environmentId) + return null + } + return deferral +} diff --git a/src/main/ssh/orcad-managed-update-on-restore.test.ts b/src/main/ssh/orcad-managed-update-on-restore.test.ts new file mode 100644 index 00000000000..38060362e5d --- /dev/null +++ b/src/main/ssh/orcad-managed-update-on-restore.test.ts @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' +import type { ManagedOrcadAutoUpdateOutcome } from './orcad-managed-auto-update' +import { + resetManagedOrcadRestoreUpdatesForTests, + updateManagedOrcadOnRestore, + type ManagedOrcadRestoreUpdateDeps +} from './orcad-managed-update-on-restore' + +const target: SshTarget = { id: 'ssh-1', label: 'Box', host: 'box', port: 22, username: 'me' } + +function deps( + outcome: ManagedOrcadAutoUpdateOutcome, + overrides: Partial = {} +): ManagedOrcadRestoreUpdateDeps { + return { + target: () => target, + publish: vi.fn(), + autoUpdate: vi.fn(async (_id, options) => { + if (outcome.outcome === 'updated' || outcome.outcome === 'failed') { + options.onUpdating() + } + return outcome + }), + recordedUpdateFailure: () => null, + recordUpdateFailure: vi.fn(), + clearUpdateFailure: vi.fn(), + ...overrides + } +} + +describe('updating a managed server when the launch restores its tunnel', () => { + beforeEach(() => resetManagedOrcadRestoreUpdatesForTests()) + + it('runs the same update check a connect runs, and publishes it', async () => { + const d = deps({ outcome: 'updated', activeVersion: '0.1.0+b' }) + await updateManagedOrcadOnRestore('env-1', () => d) + expect(d.autoUpdate).toHaveBeenCalledWith( + 'env-1', + expect.objectContaining({ failedBefore: false }) + ) + expect(vi.mocked(d.publish).mock.calls).toEqual([ + [target, 'env-1', 'updating'], + [target, 'env-1', 'settled', undefined] + ]) + }) + + it('checks each server once per session, however often its tunnel is ensured', async () => { + const d = deps({ outcome: 'skipped', reason: 'current' }) + await updateManagedOrcadOnRestore('env-1', () => d) + expect(updateManagedOrcadOnRestore('env-1', () => d)).toBeNull() + expect(d.autoUpdate).toHaveBeenCalledTimes(1) + }) + + it('keeps the same safety: waits on terminals, never downgrades, records a failure', async () => { + const waiting = deps({ + outcome: 'deferred', + code: 'orcad_update_terminals_running', + reason: 'r' + }) + await updateManagedOrcadOnRestore('env-wait', () => waiting) + expect(waiting.publish).toHaveBeenLastCalledWith(target, 'env-wait', 'settled', { + state: 'deferred', + detail: 'r' + }) + + const newer = deps({ outcome: 'skipped', reason: 'host-newer' }) + await updateManagedOrcadOnRestore('env-newer', () => newer) + expect(newer.publish).toHaveBeenLastCalledWith(target, 'env-newer', 'settled', { + state: 'host-newer' + }) + + const failed = deps({ outcome: 'failed', reason: 'rolled back' }) + await updateManagedOrcadOnRestore('env-failed', () => failed) + expect(failed.recordUpdateFailure).toHaveBeenCalledWith(target, 'rolled back') + + const held = deps( + { outcome: 'skipped', reason: 'failed-before' }, + { recordedUpdateFailure: () => 'rolled back' } + ) + await updateManagedOrcadOnRestore('env-held', () => held) + expect(held.autoUpdate).toHaveBeenCalledWith( + 'env-held', + expect.objectContaining({ failedBefore: true }) + ) + }) + + it('never fails the restore that triggered it', async () => { + const thrown = deps( + { outcome: 'skipped', reason: 'current' }, + { + autoUpdate: async () => { + throw new Error('ssh dropped') + } + } + ) + await expect(updateManagedOrcadOnRestore('env-1', () => thrown)).resolves.toBeUndefined() + expect( + updateManagedOrcadOnRestore('env-2', () => { + throw new Error('no SSH store') + }) + ).toBeNull() + const unlinked = deps({ outcome: 'skipped', reason: 'current' }, { target: () => null }) + expect(updateManagedOrcadOnRestore('env-3', () => unlinked)).toBeNull() + expect(unlinked.autoUpdate).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-managed-update-on-restore.ts b/src/main/ssh/orcad-managed-update-on-restore.ts new file mode 100644 index 00000000000..4aceebe6b28 --- /dev/null +++ b/src/main/ssh/orcad-managed-update-on-restore.ts @@ -0,0 +1,58 @@ +/** + * The launch-time tunnel restore reaches a managed server without an SSH connect, so it runs the + * same update check a connect would, once per server per session and off the caller's path. + */ +import type { SshManagedServerUpdateNote, SshTarget } from '../../shared/ssh-types' +import { + checkManagedServerUpdate, + type ManagedServerUpdateDeps +} from './managed-server-update-check' + +export type ManagedOrcadRestoreUpdateDeps = ManagedServerUpdateDeps & { + /** The SSH target that serves this managed server, or null when it is gone or unlinked. */ + target: (environmentId: string) => SshTarget | null + /** Records the outcome so the host's status line can show it. */ + publish: ( + target: SshTarget, + environmentId: string, + phase: 'updating' | 'settled', + note?: SshManagedServerUpdateNote + ) => void +} + +const checked = new Set() + +/** Resolves when the check settles; null when this session already checked the server. */ +export function updateManagedOrcadOnRestore( + environmentId: string, + createDeps: () => ManagedOrcadRestoreUpdateDeps +): Promise | null { + if (checked.has(environmentId)) { + return null + } + checked.add(environmentId) + let deps: ManagedOrcadRestoreUpdateDeps + let target: SshTarget | null + try { + deps = createDeps() + target = deps.target(environmentId) + } catch (error) { + console.warn('[ssh] Update check on tunnel restore skipped:', error) + return null + } + if (!target) { + return null + } + return checkManagedServerUpdate(target, environmentId, deps, () => + deps.publish(target, environmentId, 'updating') + ).then( + ({ note }) => deps.publish(target, environmentId, 'settled', note), + (error: unknown) => { + console.warn('[ssh] Update check on tunnel restore failed:', error) + } + ) +} + +export function resetManagedOrcadRestoreUpdatesForTests(): void { + checked.clear() +} diff --git a/src/main/ssh/orcad-managed-version-gc.test.ts b/src/main/ssh/orcad-managed-version-gc.test.ts new file mode 100644 index 00000000000..1f93e2dbe39 --- /dev/null +++ b/src/main/ssh/orcad-managed-version-gc.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ServeReadiness } from '../server/serve-readiness' +import { NODE_RUNTIME_ASSETS, NODE_RUNTIME_COMPAT_ASSETS } from '../../shared/node-runtime-pin' + +const { gcMock, readRecordMock } = vi.hoisted(() => ({ + gcMock: vi.fn(), + readRecordMock: vi.fn() +})) + +vi.mock('./orcad-remote-gc', () => ({ gcOldOrcadVersions: gcMock })) +vi.mock('./orcad-activation-record-store', () => ({ readOrcadActivationRecord: readRecordMock })) + +import { provenLiveDaemonVersion, pruneManagedOrcadVersions } from './orcad-managed-version-gc' + +function readiness(terminalDaemon: unknown): ServeReadiness { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only health.terminalDaemon is read. + return { health: terminalDaemon === undefined ? undefined : { terminalDaemon } } as ServeReadiness +} + +const slot = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: GC is mocked and never touches the connection. + conn: {} as never, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: GC is mocked; the slot's host is only passed through. + host: { os: 'linux', pathFlavor: 'posix', commandDialect: 'posix' } as never, + remoteHome: '/home/u', + nodePath: 'node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 0 +} + +afterEach(() => { + vi.resetAllMocks() +}) + +describe('managed orcad version GC after a deploy', () => { + it('pins only a daemon version the readiness proves', () => { + expect(provenLiveDaemonVersion(readiness({ state: 'live', buildVersion: '1.0.0+a' }))).toBe( + '1.0.0+a' + ) + expect(provenLiveDaemonVersion(readiness({ state: 'absent', buildVersion: null }))).toBeNull() + expect( + provenLiveDaemonVersion(readiness({ state: 'degraded', buildVersion: '1' })) + ).toBeUndefined() + expect( + provenLiveDaemonVersion(readiness({ state: 'live', buildVersion: null })) + ).toBeUndefined() + expect(provenLiveDaemonVersion(readiness(undefined))).toBeUndefined() + }) + + it('runs GC with the live daemon pinned, and keeps everything when it cannot say', async () => { + readRecordMock.mockResolvedValue({ active: '2.0.0+b' }) + await pruneManagedOrcadVersions({ + slot, + serverTarget: 'linux-x64-glibc', + activeVersion: '2.0.0+b', + readiness: readiness({ state: 'live', buildVersion: '1.0.0+a' }) + }) + expect(gcMock).toHaveBeenCalledWith( + expect.objectContaining({ + liveDaemonVersion: '1.0.0+a', + record: { active: '2.0.0+b' }, + // The compat runtime stays pinned: a compat orcad and a rung A relay share the store. + nodeRuntimePins: [ + NODE_RUNTIME_ASSETS['linux-x64-glibc'].executableSha256, + NODE_RUNTIME_COMPAT_ASSETS['linux-x64-glibc217'].executableSha256 + ] + }) + ) + + gcMock.mockClear() + await pruneManagedOrcadVersions({ + slot, + serverTarget: 'linux-x64-glibc', + activeVersion: '2.0.0+b', + readiness: readiness({ state: 'degraded', buildVersion: null }) + }) + expect(gcMock).not.toHaveBeenCalled() + }) + + it('never fails the deploy it follows', async () => { + readRecordMock.mockResolvedValue({ active: '2.0.0+b' }) + gcMock.mockRejectedValue(new Error('ssh dropped')) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await expect( + pruneManagedOrcadVersions({ + slot, + serverTarget: 'linux-x64-glibc', + activeVersion: '2.0.0+b', + readiness: readiness({ state: 'absent', buildVersion: null }) + }) + ).resolves.toBeUndefined() + }) +}) diff --git a/src/main/ssh/orcad-managed-version-gc.ts b/src/main/ssh/orcad-managed-version-gc.ts new file mode 100644 index 00000000000..b4c3a9a1341 --- /dev/null +++ b/src/main/ssh/orcad-managed-version-gc.ts @@ -0,0 +1,51 @@ +/** + * After a managed deploy or update, removes this host's older orcad version dirs that are + * proven stopped (design D10). Pins come from gcOldOrcadVersions: the active version, the + * rollback target, journaled versions, and the version the live terminal daemon was forked + * from. Anything the host cannot answer about is kept. + */ +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import type { ServeReadiness } from '../server/serve-readiness' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { orcadSlotDir, type OrcadSlotOptions } from './orcad-recovery-slot' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { relayRuntimeStorePins } from './ssh-relay-runtime-ladder' + +/** The daemon's build when the readiness proves it; undefined when it cannot say. */ +export function provenLiveDaemonVersion(readiness: ServeReadiness): string | null | undefined { + const daemon = readiness.health?.terminalDaemon + if (daemon?.state === 'live' && daemon.buildVersion) { + return daemon.buildVersion + } + // A daemon that answered as absent forked from nothing; degraded or unreported proves nothing. + return daemon?.state === 'absent' ? null : undefined +} + +export async function pruneManagedOrcadVersions(args: { + slot: OrcadSlotOptions + serverTarget: NodeRuntimeTarget + activeVersion: string + readiness: ServeReadiness +}): Promise { + const liveDaemonVersion = provenLiveDaemonVersion(args.readiness) + // Why skip: without the daemon's version, GC could remove the tree a live daemon runs from. + if (liveDaemonVersion === undefined) { + return + } + try { + await gcOldOrcadVersions({ + conn: args.slot.conn, + host: args.slot.host, + remoteHome: args.slot.remoteHome, + currentDirAbsPath: orcadSlotDir(args.slot, args.activeVersion), + record: await readOrcadActivationRecord(args.slot), + liveDaemonVersion, + // Why the relay's pins: a compat orcad and a rung A relay share one runtime store. + nodeRuntimePins: relayRuntimeStorePins(args.serverTarget), + signal: args.slot.signal + }) + } catch (error) { + // Best effort: the deploy already succeeded, and the next connect tries again. + console.warn('[orcad-gc] Pruning old orcad versions failed:', error) + } +} diff --git a/src/main/ssh/orcad-managed-wake.test.ts b/src/main/ssh/orcad-managed-wake.test.ts new file mode 100644 index 00000000000..7b83e138604 --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.test.ts @@ -0,0 +1,272 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { wakeStoppedManagedOrcad } from './orcad-managed-wake' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { FakeOrcadHost, OLD } from './orcad-activation-host-test-harness' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV +} from '../../shared/orcad-idle-exit' + +let host = new FakeOrcadHost() + +const slot = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked; the wake reads only the target id. + conn: { getTarget: () => ({ id: 'ssh-1' }) } as unknown as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {} +} + +function launches(): string[] { + return host.commands.filter((command) => command.includes('nohup')) +} + +/** The slot's process exited on its own, as an idle stop leaves it. */ +function stoppedHost(): FakeOrcadHost { + const stopped = FakeOrcadHost.deployedOld() + stopped.alive.clear() + return stopped +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async (_conn, _root, _host, options) => { + host.acquireFence() + // The real lock writes the owner token in the same command that creates it. + host.wakeOwner = options?.owner?.token ?? null + }) +}) + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('wakeStoppedManagedOrcad', () => { + it('starts a stopped active slot as a managed launch and releases the fence', async () => { + host = stoppedHost() + + const wake = await wakeStoppedManagedOrcad(slot) + + expect(wake.outcome).toBe('started') + expect([...host.alive]).toEqual([OLD]) + expect(host.fence).toBe(false) + expect(launches()).toHaveLength(1) + expect(launches()[0]).toContain( + `${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='/home/u/.orca-remote/.orcad-activation-transaction'` + ) + expect(launches()[0]).not.toContain(ORCAD_E2E_IDLE_TIMEOUT_ENV) + }) + + it('forwards the test-only idle timeout to the server it starts', async () => { + vi.stubEnv(ORCAD_E2E_IDLE_TIMEOUT_ENV, '3000') + host = stoppedHost() + + await wakeStoppedManagedOrcad(slot) + + expect(launches()[0]).toContain(`${ORCAD_E2E_IDLE_TIMEOUT_ENV}='3000'`) + }) + + it('leaves a running server alone', async () => { + host = FakeOrcadHost.deployedOld() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'serving' }) + expect(launches()).toEqual([]) + expect(acquireInstallLock).not.toHaveBeenCalled() + }) + + it('never starts a second process when the slot state is unprovable', async () => { + host = stoppedHost() + host.pidFiles.clear() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'unverifiable' }) + expect(launches()).toEqual([]) + }) + + it('defers to an update or recovery that holds the activation fence', async () => { + host = stoppedHost() + host.fence = true + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'fenced' }) + expect(launches()).toEqual([]) + expect(host.fence).toBe(true) + }) + + it('releases the fence its own wake left when the connection dropped, and starts the slot', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + vi.mocked(execCommand).mockImplementation(async (_conn, command) => { + if (command.includes('nohup')) { + throw lost + } + return host.exec(command) + }) + await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost) + expect(host.fence).toBe(true) + + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + expect(await wakeStoppedManagedOrcad(slot)).toMatchObject({ outcome: 'started' }) + expect(launches()).toHaveLength(1) + expect(host.fence).toBe(false) + }) + + it('releases its own fence on reconnect when the drop surfaced as a plain failure', async () => { + host = stoppedHost() + // A disconnect fails every later step, the fence release included, without the unconfirmed flag. + const gone = new Error('Not connected') + vi.mocked(execCommand).mockImplementation(async (_conn, command) => { + if (command.includes('nohup') || command.includes('echo RELEASED')) { + throw gone + } + return host.exec(command) + }) + await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(gone) + expect(host.fence).toBe(true) + + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + expect(await wakeStoppedManagedOrcad(slot)).toMatchObject({ outcome: 'started' }) + expect(launches()).toHaveLength(1) + expect(host.fence).toBe(false) + }) + + it('never claims a fence with no owner token, even while its own interrupted token is held', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + vi.mocked(execCommand).mockImplementation(async (_conn, command) => { + if (command.includes('nohup')) { + throw lost + } + return host.exec(command) + }) + await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost) + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + // That fence was cleared unseen, and another desktop took a fresh one: no owner file yet. + host.wakeOwner = null + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'fenced' }) + expect(host.fence).toBe(true) + expect(launches()).toEqual([]) + }) + + it('lets a wake on a dropped connection settle before a reconnected wake reads the fence', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + let drop: (() => void) | undefined + vi.mocked(execCommand).mockImplementation(async (_conn, command) => { + // Drops the first wake under its fence, after its owner token landed but before its launch. + if (host.fence && host.wakeOwner && command.includes('orcad-active.json') && !drop) { + await new Promise((resolve) => (drop = resolve)) + throw lost + } + return host.exec(command) + }) + const dropped = wakeStoppedManagedOrcad(slot) + await vi.waitFor(() => expect(drop).toBeDefined()) + + // The reconnected wake starts while the first is still holding the fence it just took. + const reconnected = wakeStoppedManagedOrcad(slot) + drop?.() + await expect(dropped).rejects.toBe(lost) + expect(await reconnected).toMatchObject({ outcome: 'started' }) + expect(launches()).toHaveLength(1) + expect(host.fence).toBe(false) + }) + + it('proves a fence its own wake took even when the drop hid whether the lock was created', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + // The host created the lock (with its owner token), but the client never saw OK. + vi.mocked(acquireInstallLock).mockImplementationOnce(async (_conn, _root, _host, options) => { + host.acquireFence() + host.wakeOwner = options?.owner?.token ?? null + throw lost + }) + await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost) + expect(host.fence).toBe(true) + + expect(await wakeStoppedManagedOrcad(slot)).toMatchObject({ outcome: 'started' }) + expect(launches()).toHaveLength(1) + expect(host.fence).toBe(false) + }) + + it('waits for a wake still before its fence instead of racing it to the lock', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + let drop: (() => void) | undefined + vi.mocked(acquireInstallLock).mockImplementationOnce(async (_conn, _root, _host, options) => { + host.acquireFence() + host.wakeOwner = options?.owner?.token ?? null + await new Promise((resolve) => (drop = resolve)) + throw lost + }) + const dropped = wakeStoppedManagedOrcad(slot) + // The reconnected wake starts while the first is still taking the fence. + const reconnected = wakeStoppedManagedOrcad(slot) + await vi.waitFor(() => expect(drop).toBeDefined()) + drop?.() + await expect(dropped).rejects.toBe(lost) + expect(await reconnected).toMatchObject({ outcome: 'started' }) + expect(launches()).toHaveLength(1) + expect(host.fence).toBe(false) + }) + + it('never releases a fence another run took after its own interrupted fence was cleared', async () => { + host = stoppedHost() + const lost = Object.assign(new Error('connection lost'), { sshChannelCloseConfirmed: false }) + vi.mocked(execCommand).mockImplementation(async (_conn, command) => { + if (command.includes('nohup')) { + throw lost + } + return host.exec(command) + }) + await expect(wakeStoppedManagedOrcad(slot)).rejects.toBe(lost) + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + // A recovery cleared that fence, then another run took the host before journaling. + host.wakeOwner = 'another-run' + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'fenced' }) + expect(host.fence).toBe(true) + expect(launches()).toEqual([]) + }) + + it('never releases a fence another client may hold, even after its own wake dropped', async () => { + host = stoppedHost() + host.fence = true + expect( + await wakeStoppedManagedOrcad({ + ...slot, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: as above, a different target. + conn: { getTarget: () => ({ id: 'ssh-2' }) } as unknown as SshConnection + }) + ).toEqual({ outcome: 'fenced' }) + expect(host.fence).toBe(true) + }) + + it('has nothing to start on a host with no activated server', async () => { + host = new FakeOrcadHost() + + expect(await wakeStoppedManagedOrcad(slot)).toEqual({ outcome: 'not-activated' }) + expect(launches()).toEqual([]) + }) +}) diff --git a/src/main/ssh/orcad-managed-wake.ts b/src/main/ssh/orcad-managed-wake.ts new file mode 100644 index 00000000000..486553830b5 --- /dev/null +++ b/src/main/ssh/orcad-managed-wake.ts @@ -0,0 +1,144 @@ +/** + * Starting a managed orcad that is installed and activated but not running, most often one + * that stopped itself after idling. A stopped server is just "not running": it is neither a + * failure nor evidence about terminals, which the daemon owns and which outlive orcad. + */ +import { ORCAD_FENCE_OWNER_FILENAME } from './orcad-activation-fence-scope' +import type { ServeReadiness } from '../server/serve-readiness' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { randomUUID } from 'node:crypto' +import { + orcadActivationFenceExists, + orcadActivationTransactionRoot, + releaseOrcadActivationFence, + withOrcadActivationLock +} from './orcad-activation-lock' +import { readBoundedOrcadRemoteRecord } from './orcad-remote-record-file' +import { RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock' +import { joinRemotePath } from './ssh-remote-platform' + +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { + ensureOrcadSlotServing, + orcadSlotDir, + resolveOrcadSlotIdentity, + slotLiveness, + type OrcadSlotOptions +} from './orcad-recovery-slot' + +export type OrcadManagedWake = + | { outcome: 'serving' | 'not-activated' | 'unverifiable' } + /** An update, rollback or recovery holds the host; it owns which slot serves. */ + | { outcome: 'fenced' } + | { outcome: 'started'; readiness: ServeReadiness } + +/** Launches the active slot only on proven exit; a live or unprovable process is left alone. */ +export function wakeStoppedManagedOrcad( + options: OrcadSlotOptions, + onStarting: () => void = () => {} +): Promise { + const host = wakeHostKey(options) + // Registered before any remote step: a wake on a dropped connection settles first, so a + // reconnected wake never races it to the fence and finds its fence unexplained. + const prior = runningWakes.get(host) + const wake = (async (): Promise => { + await prior?.catch(() => {}) + return wakeAfterPrior(options, host, onStarting) + })() + runningWakes.set(host, wake) + void wake + .catch(() => {}) + .finally(() => { + if (runningWakes.get(host) === wake) { + runningWakes.delete(host) + } + }) + return wake +} + +async function wakeAfterPrior( + options: OrcadSlotOptions, + host: string, + onStarting: () => void +): Promise { + const before = await readOrcadActivationRecord(options) + if (!before.active) { + return { outcome: 'not-activated' } + } + const liveness = await slotLiveness(options, orcadSlotDir(options, before.active)) + if (liveness !== 'DEAD') { + return { outcome: liveness === 'LIVE' ? 'serving' : 'unverifiable' } + } + if (!(await orcadActivationFenceExists(options))) { + // The fence this client left is gone by some other route; any later one belongs to another run. + interruptedWakes.delete(host) + } else if (!(await releaseOwnInterruptedWakeFence(options, host))) { + return { outcome: 'fenced' } + } + // Claimed before the fence and written by the command that creates it: a drop at any point + // after the fence lands leaves one this client can prove its own. Kept on any failure, since a + // release over a dropped connection fails quietly; the next wake re-proves it on the host. + const token = randomUUID() + interruptedWakes.set(host, token) + const result = await withOrcadActivationLock( + options, + async (): Promise => { + // Re-read under the fence: another client may have activated or started a slot meanwhile. + const active = (await readOrcadActivationRecord(options)).active + if (!active) { + return { outcome: 'not-activated' } + } + const identity = await resolveOrcadSlotIdentity(options, active) + onStarting() + return { outcome: 'started', readiness: await ensureOrcadSlotServing(options, identity) } + }, + (): OrcadManagedWake => ({ outcome: 'fenced' }), + token + ) + // Released, or never acquired. + if (interruptedWakes.get(host) === token) { + interruptedWakes.delete(host) + } + return result +} + +const runningWakes = new Map>() + +// The owner token of a fence this client's own wake may have left when its connection dropped. +const interruptedWakes = new Map() + +function wakeOwnerPath(options: OrcadSlotOptions): string { + return joinRemotePath( + options.host, + orcadActivationTransactionRoot(options.host, options.remoteHome), + RELAY_INSTALL_LOCK_NAME, + ORCAD_FENCE_OWNER_FILENAME + ) +} + +function wakeHostKey(options: OrcadSlotOptions): string { + return `${options.conn.getTarget().id}\0${options.remoteHome}` +} + +/** + * Releases only the fence carrying this client's own interrupted wake token and no journal; the + * slot was just proven exited and orcad's instance lock bars a double start. + */ +async function releaseOwnInterruptedWakeFence( + options: OrcadSlotOptions, + host: string +): Promise { + const token = interruptedWakes.get(host) + if (!token || (await readOrcadActivationTransaction(options))) { + return false + } + // Only a fence carrying this process's own token: a fresh fence another client took has none yet. + const owner = await readBoundedOrcadRemoteRecord(options, wakeOwnerPath(options), 64) + if (owner.state !== 'present' || owner.raw.trim() !== token) { + interruptedWakes.delete(host) + return false + } + await releaseOrcadActivationFence(options, token) + interruptedWakes.delete(host) + return true +} diff --git a/src/main/ssh/orcad-migration-catalog-client.test.ts b/src/main/ssh/orcad-migration-catalog-client.test.ts new file mode 100644 index 00000000000..2aa44cb7549 --- /dev/null +++ b/src/main/ssh/orcad-migration-catalog-client.test.ts @@ -0,0 +1,345 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import { computeOrcadMigrationManifestSha256 } from '../orcad/orcad-migration-manifest-digest' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' + +const sendRequest = vi.hoisted(() => vi.fn()) +const status = vi.hoisted(() => ({ capabilities: ['orcad.migration-catalog.v1'] })) +// Runs the status preflight, then hands the request to the old single-call shape the cases assert. +vi.mock('../../shared/remote-runtime-client', () => ({ + sendRemoteRuntimeRequestWithStatusPreflight: async ( + pairing: unknown, + method: string, + params: unknown, + timeoutMs: number, + validate: (response: unknown) => void, + envelope: unknown, + capabilities: unknown, + signal: unknown + ) => { + validate({ + ok: true, + _meta: { runtimeId: 'runtime' }, + result: { capabilities: status.capabilities } + }) + return sendRequest(pairing, method, params, timeoutMs, envelope, signal, capabilities) + } +})) + +const { + abortRemoteOrcadMigrationCatalog, + commitRemoteOrcadMigrationCatalog, + readRemoteOrcadMigrationCatalogState, + stageRemoteOrcadMigrationCatalog, + stageRemoteOrcadMigrationSnapshotChunk +} = await import('./orcad-migration-catalog-client') + +function manifest(): OrcadMigrationManifest { + const unsigned = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-08-30T12:00:00.000Z', + source: { + sshTargetId: 'ssh-prod', + sshTargetGeneration: 7, + targetLabel: 'Production' + }, + payload: { repositories: [], projectGroups: [], folderWorkspaces: [] } + } + return { ...unsigned, manifestSha256: computeOrcadMigrationManifestSha256(unsigned) } +} + +const pairingCode = encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/runtime', + deviceToken: 'device-token', + publicKeyB64: 'public-key', + pairedDeviceId: 'paired-desktop' +}) + +beforeEach(() => { + vi.clearAllMocks() + status.capabilities = ['orcad.migration-catalog.v1'] +}) + +describe('remote orcad migration catalog client', () => { + it.each([ + stageRemoteOrcadMigrationCatalog, + commitRemoteOrcadMigrationCatalog, + readRemoteOrcadMigrationCatalogState, + abortRemoteOrcadMigrationCatalog + ])( + 'refuses another runtime or missing authenticated runtime metadata when pinned', + async (request) => { + for (const runtimeId of ['other-runtime', undefined]) { + sendRequest.mockResolvedValue({ ok: true, _meta: { runtimeId }, result: {} }) + await expect( + request(pairingCode, manifest(), { expectedRuntimeId: 'runtime' }) + ).rejects.toThrow('destination_runtime_mismatch') + } + } + ) + + it('pins snapshot acknowledgments to the admitted destination runtime', async () => { + const request = { + migrationId: 'migration-1', + manifestSha256: manifest().manifestSha256, + ref: `v1-${'1'.repeat(32)}`, + offset: 0, + bytesBase64: 'YQ==' + } + sendRequest.mockResolvedValue({ + ok: true, + _meta: { runtimeId: 'other' }, + result: { ...request, acknowledgedOffset: 1 } + }) + await expect( + stageRemoteOrcadMigrationSnapshotChunk(pairingCode, request, { expectedRuntimeId: 'runtime' }) + ).rejects.toThrow('destination_runtime_mismatch') + sendRequest.mockResolvedValue({ + ok: true, + _meta: { runtimeId: 'runtime' }, + result: { ...request, acknowledgedOffset: 1 } + }) + await expect( + stageRemoteOrcadMigrationSnapshotChunk(pairingCode, request, { expectedRuntimeId: 'runtime' }) + ).resolves.toMatchObject({ acknowledgedOffset: 1 }) + }) + + it.each([ + ['stage', 'orcad.migration.stageCatalog', stageRemoteOrcadMigrationCatalog], + ['commit', 'orcad.migration.commitCatalog', commitRemoteOrcadMigrationCatalog], + ['state', 'orcad.migration.catalogState', readRemoteOrcadMigrationCatalogState] + ] as const)('sends and validates the %s operation', async (_label, method, request) => { + const input = manifest() + const result = { + state: 'staged', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + stagedAt: '2026-08-30T12:01:00.000Z' + } + sendRequest.mockResolvedValue({ ok: true, result }) + const signal = new AbortController().signal + + await expect(request(pairingCode, input, { signal, timeoutMs: 1234 })).resolves.toEqual(result) + expect(sendRequest).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: 'ws://127.0.0.1:46768/runtime' }), + method, + { manifest: input }, + 1234, + undefined, + signal, + expect.any(Object) + ) + }) + + it('validates an abort result without letting it erase committed state', async () => { + const input = manifest() + sendRequest.mockResolvedValue({ + ok: true, + result: { + state: 'absent', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + aborted: true + } + }) + + await expect(abortRemoteOrcadMigrationCatalog(pairingCode, input)).resolves.toMatchObject({ + state: 'absent', + aborted: true + }) + }) + + it.each([undefined, true, false, 'true'])( + 'requires explicit persistence evidence for an already-absent abort (%s)', + async (durableAbsent) => { + const input = manifest() + sendRequest.mockResolvedValue({ + ok: true, + result: { + state: 'absent', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + aborted: false, + ...(durableAbsent === undefined ? {} : { durableAbsent }) + } + }) + const result = abortRemoteOrcadMigrationCatalog(pairingCode, input) + if (durableAbsent === true) { + await expect(result).resolves.toMatchObject({ durableAbsent: true, aborted: false }) + return + } + await expect(result).rejects.toThrow(/durability/) + } + ) + + it('validates a committed receipt against the requested manifest', async () => { + const input = manifest() + const receipt = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + source: input.source, + importedAt: '2026-08-30T12:02:00.000Z', + repositoryIds: [], + projectGroupIds: [], + folderWorkspaceIds: [] + } + sendRequest.mockResolvedValue({ + ok: true, + result: { + state: 'committed', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + receipt + } + }) + + await expect(commitRemoteOrcadMigrationCatalog(pairingCode, input)).resolves.toMatchObject({ + state: 'committed', + receipt + }) + }) + + it('uploads a snapshot chunk and requires the exact acknowledgment', async () => { + const input = manifest() + const request = { + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + ref: `v1-${'1'.repeat(32)}`, + offset: 4, + bytesBase64: 'YQ==' + } + sendRequest.mockResolvedValueOnce({ + ok: true, + result: { ...request, acknowledgedOffset: 5 } + }) + await expect( + stageRemoteOrcadMigrationSnapshotChunk(pairingCode, request) + ).resolves.toMatchObject({ acknowledgedOffset: 5 }) + expect(sendRequest).toHaveBeenLastCalledWith( + expect.any(Object), + 'orcad.migration.stageSnapshotChunk', + request, + 15_000, + undefined, + undefined, + expect.any(Object) + ) + + sendRequest.mockResolvedValueOnce({ + ok: true, + result: { ...request, acknowledgedOffset: 4 } + }) + await expect(stageRemoteOrcadMigrationSnapshotChunk(pairingCode, request)).rejects.toThrow( + 'orcad_migration_snapshot_chunk_result_invalid' + ) + }) + + it('fails closed when a staged old host omits required snapshot upload state', async () => { + const input = manifest() + input.payload.dormantState = { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [], + terminalScrollbackSnapshots: [ + { + tabId: 'tab-1', + leafId: 'leaf-1', + ref: `v1-${'1'.repeat(32)}`, + sha256: 'b'.repeat(64), + byteLength: 1 + } + ] + } + sendRequest.mockResolvedValue({ + ok: true, + result: { + state: 'staged', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + stagedAt: '2026-08-30T12:01:00.000Z' + } + }) + + await expect(readRemoteOrcadMigrationCatalogState(pairingCode, input)).rejects.toThrow( + 'orcad_migration_snapshot_transfer_unsupported' + ) + }) + + it('rejects malformed stage timestamps and abort envelopes', async () => { + const input = manifest() + sendRequest.mockResolvedValueOnce({ + ok: true, + result: { + state: 'staged', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256, + stagedAt: 'not-a-date' + } + }) + await expect(stageRemoteOrcadMigrationCatalog(pairingCode, input)).rejects.toThrow( + 'orcad_migration_catalog_state_staged_at_invalid' + ) + + sendRequest.mockResolvedValueOnce({ + ok: true, + result: { + state: 'absent', + migrationId: input.migrationId, + manifestSha256: input.manifestSha256 + } + }) + await expect(abortRemoteOrcadMigrationCatalog(pairingCode, input)).rejects.toThrow( + 'orcad_migration_catalog_abort_result_invalid' + ) + }) + + it('fails closed for method absence, malformed identity, and invalid pairing', async () => { + const input = manifest() + sendRequest.mockResolvedValueOnce({ ok: false, error: { message: 'method not found' } }) + await expect(stageRemoteOrcadMigrationCatalog(pairingCode, input)).rejects.toThrow( + 'orcad_migration_stage_failed:method not found' + ) + + sendRequest.mockResolvedValueOnce({ + ok: true, + result: { state: 'absent', migrationId: 'other', manifestSha256: input.manifestSha256 } + }) + await expect(readRemoteOrcadMigrationCatalogState(pairingCode, input)).rejects.toThrow( + 'orcad_migration_catalog_state_identity_mismatch' + ) + + await expect(stageRemoteOrcadMigrationCatalog('invalid', input)).rejects.toThrow( + 'orcad_migration_pairing_code_invalid' + ) + expect(sendRequest).toHaveBeenCalledTimes(2) + }) + + it('refuses a destination without the capability before sending any migration request', async () => { + status.capabilities = [] + await expect(stageRemoteOrcadMigrationCatalog(pairingCode, manifest())).rejects.toThrow( + 'orcad_migration_destination_unsupported' + ) + expect(sendRequest).not.toHaveBeenCalled() + }) + + it('reads method-not-found as the same refusal, not as a lost answer', async () => { + sendRequest.mockResolvedValue({ + ok: false, + _meta: { runtimeId: 'runtime' }, + error: { code: 'method_not_found', message: 'Unknown method' } + }) + await expect(commitRemoteOrcadMigrationCatalog(pairingCode, manifest())).rejects.toThrow( + 'orcad_migration_destination_unsupported' + ) + }) +}) diff --git a/src/main/ssh/orcad-migration-catalog-client.ts b/src/main/ssh/orcad-migration-catalog-client.ts new file mode 100644 index 00000000000..a069720a058 --- /dev/null +++ b/src/main/ssh/orcad-migration-catalog-client.ts @@ -0,0 +1,173 @@ +import { + parseOrcadMigrationCatalogAbortResult, + parseOrcadMigrationCatalogState +} from '../../shared/orcad-migration-catalog-state' +import type { + OrcadMigrationCatalogAbortResult, + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import { + parseOrcadMigrationSnapshotChunkResult, + type OrcadMigrationSnapshotChunkRequest, + type OrcadMigrationSnapshotChunkResult +} from '../../shared/orcad-migration-scrollback' +import { parsePairingCode } from '../../shared/pairing' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/electron-remote-runtime-client-capabilities' +import { ORCAD_MIGRATION_CATALOG_RUNTIME_CAPABILITY } from '../../shared/orcad-runtime-capabilities' +import { sendRemoteRuntimeRequestWithStatusPreflight } from '../../shared/remote-runtime-client' +import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' + +type OrcadCatalogMigrationOperation = 'abort' | 'commit' | 'stage' | 'state' + +/** + * A destination that does not offer catalog migration: an older orcad without the capability or + * the method. This is a definite refusal; any other failure, including a lost answer, is not, and + * the caller re-reads the catalog state before deciding anything. + */ +export const ORCAD_MIGRATION_DESTINATION_UNSUPPORTED = 'orcad_migration_destination_unsupported' + +type CatalogRequestOptions = { + signal?: AbortSignal + timeoutMs?: number + expectedRuntimeId?: string +} + +const METHOD_BY_OPERATION: Record = { + abort: 'orcad.migration.abortCatalog', + commit: 'orcad.migration.commitCatalog', + stage: 'orcad.migration.stageCatalog', + state: 'orcad.migration.catalogState' +} + +export function stageRemoteOrcadMigrationCatalog( + pairingCode: string, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions = {} +): Promise { + return requestCatalogState(pairingCode, 'stage', manifest, options) +} + +export function commitRemoteOrcadMigrationCatalog( + pairingCode: string, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions = {} +): Promise { + return requestCatalogState(pairingCode, 'commit', manifest, options) +} + +export function readRemoteOrcadMigrationCatalogState( + pairingCode: string, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions = {} +): Promise { + return requestCatalogState(pairingCode, 'state', manifest, options) +} + +export async function abortRemoteOrcadMigrationCatalog( + pairingCode: string, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions = {} +): Promise { + const result = await request(pairingCode, 'abort', manifest, options) + const parsed = parseOrcadMigrationCatalogAbortResult(result, manifest) + // Older hosts flush real aborts, but their already-absent responses prove no persistence. + if (parsed.state === 'absent' && !parsed.aborted && parsed.durableAbsent !== true) { + throw new Error('orcad_migration_abort_durability_unverifiable:destination_update_required') + } + return parsed +} + +export async function stageRemoteOrcadMigrationSnapshotChunk( + pairingCode: string, + request: OrcadMigrationSnapshotChunkRequest, + options: CatalogRequestOptions = {} +): Promise { + const pairing = parsePairingCode(pairingCode) + if (!pairing) { + throw new Error('orcad_migration_pairing_code_invalid') + } + const response = await sendSupportedMigrationRequest( + pairing, + 'orcad.migration.stageSnapshotChunk', + request, + options + ) + if (!response.ok) { + throw new Error(`orcad_migration_snapshot_failed:${response.error.message}`) + } + return parseOrcadMigrationSnapshotChunkResult(response.result, request) +} + +async function requestCatalogState( + pairingCode: string, + operation: Exclude, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions +): Promise { + const result = await request(pairingCode, operation, manifest, options) + return parseOrcadMigrationCatalogState(result, manifest) +} + +async function request( + pairingCode: string, + operation: OrcadCatalogMigrationOperation, + manifest: OrcadMigrationManifest, + options: CatalogRequestOptions +): Promise { + const pairing = parsePairingCode(pairingCode) + if (!pairing) { + throw new Error('orcad_migration_pairing_code_invalid') + } + const response = await sendSupportedMigrationRequest( + pairing, + METHOD_BY_OPERATION[operation], + { manifest }, + options + ) + if (!response.ok) { + throw new Error(`orcad_migration_${operation}_failed:${response.error.message}`) + } + return response.result +} + +async function sendSupportedMigrationRequest( + pairing: NonNullable>, + method: string, + params: unknown, + options: CatalogRequestOptions +): Promise> { + const response = await sendRemoteRuntimeRequestWithStatusPreflight( + pairing, + method, + params, + options.timeoutMs ?? 15_000, + (status) => { + if (!status.ok) { + throw new Error(`orcad_migration_status_failed:${status.error.message}`) + } + if ( + options.expectedRuntimeId !== undefined && + status._meta?.runtimeId !== options.expectedRuntimeId + ) { + throw new Error('orcad_migration_destination_runtime_mismatch') + } + if (!status.result.capabilities?.includes(ORCAD_MIGRATION_CATALOG_RUNTIME_CAPABILITY)) { + throw new Error(ORCAD_MIGRATION_DESTINATION_UNSUPPORTED) + } + }, + undefined, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES, + options.signal + ) + if (!response.ok && response.error.code === 'method_not_found') { + throw new Error(ORCAD_MIGRATION_DESTINATION_UNSUPPORTED) + } + if ( + options.expectedRuntimeId !== undefined && + response._meta?.runtimeId !== options.expectedRuntimeId + ) { + throw new Error('orcad_migration_destination_runtime_mismatch') + } + return response +} diff --git a/src/main/ssh/orcad-migration-cutover-coordinator.test.ts b/src/main/ssh/orcad-migration-cutover-coordinator.test.ts new file mode 100644 index 00000000000..e7ca374b18e --- /dev/null +++ b/src/main/ssh/orcad-migration-cutover-coordinator.test.ts @@ -0,0 +1,257 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { + abortOrcadMigrationCutover, + commitOrcadMigrationDestination, + stageOrcadMigrationDestination, + type OrcadMigrationCutoverContext +} from './orcad-migration-cutover-coordinator' +import { ORCAD_MIGRATION_DESTINATION_UNSUPPORTED } from './orcad-migration-catalog-client' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' +import { fenceOrcadMigrationSource } from './orcad-migration-source-fence' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { fakeOrcadMigrationDestination as fakeDestination } from './orcad-migration-destination-fake' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} + +const directories: string[] = [] +afterEach(async () => { + await closeTestStores() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +async function setup() { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-cutover-coordinator-')) + directories.push(userDataPath) + const store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + const claims = new SshTargetOrcadClaims(store) + const fenced = await fenceOrcadMigrationSource({ + userDataPath, + store, + claims, + targetId: TARGET.id, + destinationEnvironmentId: 'env-1', + destinationName: 'Managed', + terminalProof: { verdict: 'exited', provenPtyIds: [] }, + hasDirectSshAuthority: () => false + }) + if (fenced.outcome !== 'fenced') { + throw new Error('expected a fence') + } + const destination = fakeDestination() + const context: OrcadMigrationCutoverContext = { userDataPath, store, claims, destination } + const journal = () => listOrcadMigrationSourceCutovers(userDataPath)[0] + const owner = () => getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id)) + return { context, destination, store, migrationId: fenced.cutover.migrationId, journal, owner } +} + +describe('migration cutover coordinator', () => { + it('stages then commits once, journaling each phase the destination proves', async () => { + const h = await setup() + await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({ + phase: 'destination-staged' + }) + expect(h.journal()?.phase).toBe('destination-staged') + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({ + phase: 'destination-committed' + }) + expect(h.journal()?.phase).toBe('destination-committed') + await commitOrcadMigrationDestination(h.context, h.migrationId) + expect(h.destination.commits).toBe(1) + }) + + it('reads a lost commit reply back from the destination instead of failing or retrying blind', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + const commit = h.destination.commit.getMockImplementation()! + h.destination.commit.mockImplementationOnce(async (manifest) => { + await commit(manifest) + throw new Error('socket closed') + }) + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({ + phase: 'destination-committed' + }) + expect(h.destination.commits).toBe(1) + }) + + it('never journals a commit the server holds only in memory, until its flush succeeds', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + const commit = h.destination.commit.getMockImplementation()! + let diskFull = true + // The receipt lands in memory, so reads say committed, but the flush keeps failing. + h.destination.commit.mockImplementation(async (manifest) => { + const state = await commit(manifest) + if (diskFull) { + throw new Error('disk full') + } + return state + }) + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow( + 'disk full' + ) + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow( + 'disk full' + ) + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).rejects.toThrow('disk full') + expect(h.journal()?.phase).toBe('destination-staged') + expect(h.owner()).toBe('env-1') + + diskFull = false + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).resolves.toMatchObject({ + phase: 'destination-committed' + }) + }) + + it('keeps the journal at staged when the commit reply and the re-read are both lost', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + h.destination.commit.mockRejectedValueOnce(new Error('socket closed')) + h.destination.readState.mockRejectedValueOnce(new Error('socket closed')) + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow( + 'socket closed' + ) + expect(h.journal()?.phase).toBe('destination-staged') + expect(h.owner()).toBe('env-1') + }) + + it.each([ + [ + 'a source row changed', + (store: Store) => store.updateRepo('repo-1', { displayName: 'Renamed' }), + 'orcad_migration_source_changed' + ], + [ + 'a terminal started on the source', + (store: Store) => + store.upsertSshRemotePtyLease({ targetId: TARGET.id, ptyId: 'late', state: 'attached' }), + 'orcad_migration_source_terminals_live' + ] + ])('refuses to stage or commit after %s', async (_label, change, code) => { + const h = await setup() + change(h.store) + await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow(code) + expect(h.destination.stage).not.toHaveBeenCalled() + }) + + it('refuses a commit whose source changed after staging', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + h.store.updateRepo('repo-1', { displayName: 'Renamed' }) + await expect(commitOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow( + 'orcad_migration_source_changed' + ) + expect(h.destination.commit).not.toHaveBeenCalled() + }) + + it('aborts a stage and releases the fence only once the destination proves it absent', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toEqual({ + outcome: 'released', + evidence: 'catalog-absent' + }) + expect(h.owner()).toBeNull() + expect(h.journal()).toBeUndefined() + }) + + it('keeps the fence when the abort answer is lost and the destination cannot be read', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + h.destination.abort.mockRejectedValueOnce(new Error('socket closed')) + h.destination.readState.mockResolvedValueOnce({ + state: 'absent', + migrationId: h.migrationId, + manifestSha256: h.journal()!.manifestSha256 + }) + h.destination.readState.mockRejectedValueOnce(new Error('socket closed')) + h.destination.abort.mockRejectedValueOnce(new Error('socket closed')) + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).rejects.toThrow( + 'socket closed' + ) + expect(h.owner()).toBe('env-1') + expect(h.journal()?.phase).toBe('destination-staged') + }) + + it('never releases a committed destination, even when the journal lags behind it', async () => { + const h = await setup() + await stageOrcadMigrationDestination(h.context, h.migrationId) + await h.destination.commit(h.journal()!.manifest) + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toMatchObject({ + outcome: 'refused', + code: 'orcad_migration_committed_source_cannot_be_released' + }) + expect(h.journal()?.phase).toBe('destination-committed') + expect(h.owner()).toBe('env-1') + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).resolves.toMatchObject({ + outcome: 'refused' + }) + }) + + it('releases on an unsupported destination only before anything was staged', async () => { + const unsupported = new Error(ORCAD_MIGRATION_DESTINATION_UNSUPPORTED) + const fresh = await setup() + fresh.destination.readState.mockRejectedValueOnce(unsupported) + await expect(abortOrcadMigrationCutover(fresh.context, fresh.migrationId)).resolves.toEqual({ + outcome: 'released', + evidence: 'destination-unsupported' + }) + expect(fresh.owner()).toBeNull() + + const staged = await setup() + await stageOrcadMigrationDestination(staged.context, staged.migrationId) + staged.destination.readState.mockRejectedValueOnce(unsupported) + await expect(abortOrcadMigrationCutover(staged.context, staged.migrationId)).rejects.toThrow( + ORCAD_MIGRATION_DESTINATION_UNSUPPORTED + ) + expect(staged.owner()).toBe('env-1') + }) + + it('treats any other read failure at abort as unverifiable and keeps the fence', async () => { + const h = await setup() + h.destination.readState.mockRejectedValueOnce(new Error('socket closed')) + await expect(abortOrcadMigrationCutover(h.context, h.migrationId)).rejects.toThrow( + 'socket closed' + ) + expect(h.owner()).toBe('env-1') + }) + + it('refuses a destination answer for another manifest', async () => { + const h = await setup() + h.destination.stage.mockResolvedValueOnce({ + state: 'staged', + migrationId: h.migrationId, + manifestSha256: 'f'.repeat(64), + stagedAt: '2026-10-01T00:00:00.000Z' + }) + await expect(stageOrcadMigrationDestination(h.context, h.migrationId)).rejects.toThrow( + 'orcad_migration_destination_state_mismatch' + ) + expect(h.journal()?.phase).toBe('source-fenced') + }) +}) diff --git a/src/main/ssh/orcad-migration-cutover-coordinator.ts b/src/main/ssh/orcad-migration-cutover-coordinator.ts new file mode 100644 index 00000000000..1ccef9dac92 --- /dev/null +++ b/src/main/ssh/orcad-migration-cutover-coordinator.ts @@ -0,0 +1,262 @@ +/** + * Staging, committing and aborting a dormant migration on its destination orcad. + * + * The source stays authoritative until the destination proves a commit: before every stage and + * commit the fenced source must still export the journaled manifest and carry no state the + * manifest cannot. A lost answer is never read as success or as absence; the destination's + * catalog state is re-read, and only that observation moves the journal, which is on disk before + * anything is returned. A committed destination is never released. + */ +import type { + OrcadMigrationCatalogAbortResult, + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { + OrcadMigrationSnapshotChunkRequest, + OrcadMigrationSnapshotChunkResult +} from '../../shared/orcad-migration-scrollback' +import type { + OrcadMigrationSourceCutover, + OrcadMigrationSourceCutoverPhase +} from '../../shared/orcad-migration-source-cutover' +import { resolveDurableOrcadCatalogMutation } from './orcad-catalog-durable-mutation' +import { ORCAD_MIGRATION_DESTINATION_UNSUPPORTED } from './orcad-migration-catalog-client' +import { + listOrcadMigrationSourceCutovers, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' +import { releaseOrcadMigrationFence } from './orcad-migration-source-fence' +import { + transferOrcadMigrationSnapshots, + type OrcadMigrationSnapshotSource +} from './orcad-migration-snapshot-coordinator' +import { assertOrcadMigrationSourceUnchanged } from './orcad-migration-source-assertions' +import type { OrcadMigrationPreflightStore } from './ssh-target-orcad-preflight' +import type { SshTargetOrcadClaims } from './ssh-target-orcad-claims' + +/** The destination's catalog operations, served by the T6-9 orcad.migration.* client. */ +export type OrcadMigrationDestinationCatalog = { + readState: (manifest: OrcadMigrationManifest) => Promise + stage: (manifest: OrcadMigrationManifest) => Promise + commit: (manifest: OrcadMigrationManifest) => Promise + abort: (manifest: OrcadMigrationManifest) => Promise + stageChunk: ( + request: OrcadMigrationSnapshotChunkRequest + ) => Promise +} + +export type OrcadMigrationCutoverContext = { + userDataPath: string + store: OrcadMigrationPreflightStore & OrcadMigrationSnapshotSource + claims: SshTargetOrcadClaims + destination: OrcadMigrationDestinationCatalog + /** Rebuilt for every check when `store` is a copy that would never see a later edit. */ + freshSource?: () => OrcadMigrationPreflightStore + now?: () => Date +} + +export async function stageOrcadMigrationDestination( + context: OrcadMigrationCutoverContext, + migrationId: string +): Promise { + let cutover = requireCutover(context.userDataPath, migrationId) + if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') { + return cutover + } + assertSourceUnchanged(context, cutover) + const { destination } = context + const read = await destination.readState(cutover.manifest) + const state = + read.state === 'committed' + ? await confirmDurableCommit(destination, cutover.manifest) + : await resolveDurableOrcadCatalogMutation( + () => destination.stage(cutover.manifest), + () => destination.readState(cutover.manifest), + (observed) => observed.state !== 'absent' + ) + cutover = recordObservedState(context, cutover, state) + if (state.state === 'staged') { + await transferOrcadMigrationSnapshots({ + source: context.store, + manifest: cutover.manifest, + state, + destination: { readState: destination.readState, stageChunk: destination.stageChunk } + }) + } + return cutover +} + +export async function commitOrcadMigrationDestination( + context: OrcadMigrationCutoverContext, + migrationId: string +): Promise { + const staged = await stageOrcadMigrationDestination(context, migrationId) + if (staged.phase !== 'destination-staged') { + return staged + } + assertSourceUnchanged(context, staged) + const { destination } = context + // Why re-read: a lost commit reply may hide a commit that landed; the read decides. + const state = await resolveDurableOrcadCatalogMutation( + () => destination.commit(staged.manifest), + () => destination.readState(staged.manifest), + (observed) => observed.state === 'committed' + ) + if (state.state !== 'committed') { + throw new Error(`orcad_migration_commit_not_committed:${state.state}`) + } + return recordObservedState(context, staged, state) +} + +export type OrcadMigrationAbortResult = + | { outcome: 'released'; evidence: 'catalog-absent' | 'destination-unsupported' } + | { outcome: 'refused'; code: string; reason: string } + +/** + * Releases the source only on proof the destination holds nothing of this migration. `release` + * defaults to dropping the fence and journal; a delta move keeps the fence instead. + */ +export async function abortOrcadMigrationCutover( + context: OrcadMigrationCutoverContext, + migrationId: string, + release: (cutover: OrcadMigrationSourceCutover) => Promise = (cutover) => + releaseOrcadMigrationFence(context, cutover) +): Promise { + const cutover = requireCutover(context.userDataPath, migrationId) + if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') { + return committedRefusal() + } + const { destination } = context + let state: OrcadMigrationCatalogState + try { + state = await destination.readState(cutover.manifest) + } catch (error) { + // Unsupported means nothing could have been staged, but only if nothing ever was. + if (isDestinationUnsupported(error) && cutover.phase === 'source-fenced') { + await release(cutover) + return { outcome: 'released', evidence: 'destination-unsupported' } + } + throw error + } + if (state.state !== 'committed') { + state = await abortWithRecovery(destination, cutover.manifest) + } + if (state.state === 'committed') { + recordObservedState(context, cutover, await confirmDurableCommit(destination, cutover.manifest)) + return committedRefusal() + } + if (state.state !== 'absent') { + throw new Error(`orcad_migration_abort_not_absent:${state.state}`) + } + await release(cutover) + return { outcome: 'released', evidence: 'catalog-absent' } +} + +/** + * A committed read may come from memory the server never flushed; only commit()'s acknowledgement, + * which flushes before it answers, may move the journal to committed. + */ +async function confirmDurableCommit( + destination: OrcadMigrationDestinationCatalog, + manifest: OrcadMigrationManifest +): Promise { + const state = await destination.commit(manifest) + if (state.state !== 'committed') { + throw new Error(`orcad_migration_commit_not_committed:${state.state}`) + } + return state +} + +function assertSourceUnchanged( + context: OrcadMigrationCutoverContext, + cutover: OrcadMigrationSourceCutover +): void { + const store = context.freshSource?.() ?? context.store + assertOrcadMigrationSourceUnchanged({ userDataPath: context.userDataPath, store }, cutover) +} + +/** Moves the journal to what the destination reported; durable before it returns. */ +function recordObservedState( + context: OrcadMigrationCutoverContext, + cutover: OrcadMigrationSourceCutover, + state: OrcadMigrationCatalogState +): OrcadMigrationSourceCutover { + if (state.state === 'absent') { + throw new Error('orcad_migration_destination_catalog_absent') + } + if ( + state.migrationId !== cutover.migrationId || + state.manifestSha256 !== cutover.manifestSha256 + ) { + throw new Error('orcad_migration_destination_state_mismatch') + } + if ( + state.state === 'committed' && + (state.receipt.migrationId !== cutover.migrationId || + state.receipt.manifestSha256 !== cutover.manifestSha256) + ) { + throw new Error('orcad_migration_destination_receipt_mismatch') + } + const phase: OrcadMigrationSourceCutoverPhase = + state.state === 'committed' ? 'destination-committed' : 'destination-staged' + if (phase === cutover.phase) { + return cutover + } + if (cutover.phase === 'destination-committed' || cutover.phase === 'source-retired') { + // Why: a journal never moves back from a proven commit. + return cutover + } + const next: OrcadMigrationSourceCutover = { + ...cutover, + phase, + updatedAt: (context.now ?? (() => new Date()))().toISOString() + } + writeOrcadMigrationSourceCutover(context.userDataPath, next) + return next +} + +async function abortWithRecovery( + destination: OrcadMigrationDestinationCatalog, + manifest: OrcadMigrationManifest +): Promise { + try { + return await destination.abort(manifest) + } catch (abortError) { + try { + const observed = await destination.readState(manifest) + if (observed.state === 'committed') { + return observed + } + if (observed.state === 'absent') { + // An absent read may predate the flush; a repeated abort proves it durable. + return await destination.abort(manifest) + } + } catch { + // Still unverifiable: keep the first failure and the fence. + } + throw abortError + } +} + +function requireCutover(userDataPath: string, migrationId: string): OrcadMigrationSourceCutover { + const cutover = listOrcadMigrationSourceCutovers(userDataPath).find( + (entry) => entry.migrationId === migrationId + ) + if (!cutover) { + throw new Error('orcad_migration_source_cutover_not_found') + } + return cutover +} + +function committedRefusal(): OrcadMigrationAbortResult { + return { + outcome: 'refused', + code: 'orcad_migration_committed_source_cannot_be_released', + reason: 'The managed server already holds this migration; it can only move forward.' + } +} + +function isDestinationUnsupported(error: unknown): boolean { + return error instanceof Error && error.message === ORCAD_MIGRATION_DESTINATION_UNSUPPORTED +} diff --git a/src/main/ssh/orcad-migration-cutover-fixture.ts b/src/main/ssh/orcad-migration-cutover-fixture.ts new file mode 100644 index 00000000000..df56e7a5814 --- /dev/null +++ b/src/main/ssh/orcad-migration-cutover-fixture.ts @@ -0,0 +1,36 @@ +import { ORCAD_MIGRATION_MANIFEST_VERSION } from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import { computeOrcadMigrationManifestSha256 } from '../orcad/orcad-migration-manifest-digest' + +/** A valid journal entry whose manifest is bound to its target, generation and destination. */ +export function orcadMigrationCutoverFixture( + migrationId = 'migration-1', + sshTargetId = 'ssh-1', + binding: { generation?: number; environmentId?: string; name?: string } = {} +): OrcadMigrationSourceCutover { + const generation = binding.generation ?? 2 + const environmentId = binding.environmentId ?? 'env-1' + const unsigned = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId, + createdAt: '2026-10-01T00:00:00.000Z', + source: { sshTargetId, sshTargetGeneration: generation, targetLabel: 'Prod' }, + payload: { repositories: [], projectGroups: [], folderWorkspaces: [] }, + destinationEnvironmentId: environmentId + } + const manifest = { ...unsigned, manifestSha256: computeOrcadMigrationManifestSha256(unsigned) } + return { + version: 1, + migrationId, + phase: 'source-fenced', + startedAt: '2026-10-01T00:00:00.000Z', + updatedAt: '2026-10-01T00:00:00.000Z', + destinationEnvironmentId: environmentId, + destinationName: binding.name ?? 'Managed', + sshTargetId, + sshTargetGeneration: generation, + manifestSha256: manifest.manifestSha256, + provenPtyIds: [], + manifest + } +} diff --git a/src/main/ssh/orcad-migration-cutover-journal.test.ts b/src/main/ssh/orcad-migration-cutover-journal.test.ts new file mode 100644 index 00000000000..b18741d6ac0 --- /dev/null +++ b/src/main/ssh/orcad-migration-cutover-journal.test.ts @@ -0,0 +1,93 @@ +import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { orcadMigrationCutoverFixture as cutover } from './orcad-migration-cutover-fixture' +import { + findOrcadMigrationSourceCutoverForTarget, + listOrcadMigrationSourceCutovers, + orcadMigrationCutoverJournalDirectory, + removeOrcadMigrationSourceCutover, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' + +let userDataPath: string +beforeEach(() => { + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-cutover-journal-')) +}) +afterEach(() => rmSync(userDataPath, { recursive: true, force: true })) + +const journalPath = (id: string) => + join(orcadMigrationCutoverJournalDirectory(userDataPath), `${id}.json`) + +describe('migration cutover journal sidecar', () => { + it('round-trips a cutover in an owner-only file', () => { + writeOrcadMigrationSourceCutover(userDataPath, cutover()) + expect(findOrcadMigrationSourceCutoverForTarget(userDataPath, 'ssh-1')).toEqual(cutover()) + if (process.platform !== 'win32') { + expect(statSync(journalPath('migration-1')).mode & 0o077).toBe(0) + } + removeOrcadMigrationSourceCutover(userDataPath, 'migration-1') + expect(listOrcadMigrationSourceCutovers(userDataPath)).toEqual([]) + }) + + it.each([ + ['corrupt JSON', '{not json'], + ['a manifest bound to another target', JSON.stringify({ ...cutover(), sshTargetId: 'ssh-2' })], + ['a tampered manifest', JSON.stringify({ ...cutover(), manifestSha256: 'f'.repeat(64) })], + ['an unknown version', JSON.stringify({ ...cutover(), version: 2 })] + ])('fails closed on %s', (_label, contents) => { + mkdirSync(orcadMigrationCutoverJournalDirectory(userDataPath), { recursive: true }) + writeFileSync(journalPath('migration-1'), contents) + expect(() => listOrcadMigrationSourceCutovers(userDataPath)).toThrow('stays fenced') + }) + + it("reads a newer build's journal that adds an optional field", () => { + mkdirSync(orcadMigrationCutoverJournalDirectory(userDataPath), { recursive: true }) + writeFileSync(journalPath('migration-1'), JSON.stringify({ ...cutover(), addedLater: true })) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toEqual([cutover()]) + }) + + it('serves repeat reads from cache yet sees a file rewritten behind its back', () => { + writeOrcadMigrationSourceCutover(userDataPath, cutover()) + const first = listOrcadMigrationSourceCutovers(userDataPath) + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]).toBe(first[0]) + writeFileSync( + journalPath('migration-1'), + JSON.stringify({ ...cutover(), phase: 'destination-staged' }) + ) + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]?.phase).toBe('destination-staged') + }) + + it('fails closed on a file whose name disagrees with its migration', () => { + mkdirSync(orcadMigrationCutoverJournalDirectory(userDataPath), { recursive: true }) + writeFileSync(journalPath('other'), JSON.stringify(cutover())) + expect(() => listOrcadMigrationSourceCutovers(userDataPath)).toThrow('stays fenced') + }) + + it('ignores durable-write temporaries but refuses two journals for one target', () => { + writeOrcadMigrationSourceCutover(userDataPath, cutover()) + writeFileSync(join(orcadMigrationCutoverJournalDirectory(userDataPath), 'x.json.tmp'), '?') + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + writeOrcadMigrationSourceCutover(userDataPath, cutover('migration-2')) + expect(() => findOrcadMigrationSourceCutoverForTarget(userDataPath, 'ssh-1')).toThrow( + 'journals name SSH target' + ) + }) + + it('bounds concurrent migrations but rewrites an existing one', () => { + for (const index of [1, 2, 3, 4]) { + writeOrcadMigrationSourceCutover(userDataPath, cutover(`m-${index}`, `ssh-${index}`)) + } + expect(() => writeOrcadMigrationSourceCutover(userDataPath, cutover('m-5', 'ssh-5'))).toThrow( + 'capacity' + ) + writeOrcadMigrationSourceCutover(userDataPath, { + ...cutover('m-1', 'ssh-1'), + phase: 'destination-staged' + }) + expect(findOrcadMigrationSourceCutoverForTarget(userDataPath, 'ssh-1')?.phase).toBe( + 'destination-staged' + ) + }) +}) diff --git a/src/main/ssh/orcad-migration-cutover-journal.ts b/src/main/ssh/orcad-migration-cutover-journal.ts new file mode 100644 index 00000000000..bdf498cc1dd --- /dev/null +++ b/src/main/ssh/orcad-migration-cutover-journal.ts @@ -0,0 +1,212 @@ +/** + * The cutover journal sidecar: one durable, owner-only file per migration beside the profile. + * + * Why not the profile store: shipped builds rewrite orca-data.json and the profile database with + * schemas that drop unknown fields, so a journal kept there would vanish across a downgrade while + * the target's `orcadFence`, which shipped builds keep, survived it. A missing journal must + * never look like "no migration"; an unreadable one fails closed. + */ +import { existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { writeSecureJsonFileWithinLimit } from '../../shared/bounded-secure-json-file' +import { readNodeFileSyncWithinLimit } from '../../shared/node-bounded-file-reader' +import { + MAX_ORCAD_MIGRATION_SOURCE_CUTOVERS, + isRetainedOrcadMigrationSourceCutover, + parseOrcadMigrationSourceCutover, + type OrcadMigrationSourceCutover +} from '../../shared/orcad-migration-source-cutover' +import { syncDirectoryDurablySync } from '../durable-file-write' +import { errorMessage } from '../../shared/error-message' + +const JOURNAL_DIRECTORY = 'orcad-migration-cutovers' +const JOURNAL_FILE = /^[A-Za-z0-9_-]{1,128}\.json$/ +// A manifest carries catalog rows and dormant session state, never scrollback bytes. +const MAX_JOURNAL_FILE_BYTES = 16 * 1024 * 1024 + +export class OrcadMigrationCutoverJournalUnreadableError extends Error { + constructor(detail: string) { + super(`The migration journal cannot be read, so the SSH host stays fenced: ${detail}`) + this.name = 'OrcadMigrationCutoverJournalUnreadableError' + } +} + +type JournalChangeListener = (userDataPath: string) => void +let journalChangeListener: JournalChangeListener | null = null + +/** Persistence state that follows the journal (scrollback retention) re-syncs on every change. */ +export function setOrcadMigrationJournalChangeListener( + listener: JournalChangeListener | null +): void { + journalChangeListener = listener +} + +function notifyJournalChanged(userDataPath: string): void { + try { + journalChangeListener?.(userDataPath) + } catch (error) { + // The journal write is already durable; a follower failing must not undo or fail it. + console.warn('[orcad-migration] Journal change follower failed:', error) + } +} + +// Why: hidden-row checks run on every list call, and each journal embeds a full manifest. +const parsedJournals = new Map() + +export function orcadMigrationCutoverJournalDirectory(userDataPath: string): string { + return join(userDataPath, JOURNAL_DIRECTORY) +} + +/** Every journaled cutover; throws rather than skipping a file it cannot trust. */ +export function listOrcadMigrationSourceCutovers( + userDataPath: string +): OrcadMigrationSourceCutover[] { + const directory = orcadMigrationCutoverJournalDirectory(userDataPath) + if (!existsSync(directory)) { + return [] + } + let names: string[] + try { + names = readdirSync(directory) + } catch (error) { + throw new OrcadMigrationCutoverJournalUnreadableError(errorMessage(error)) + } + const files: { name: string; path: string }[] = [] + for (const name of names) { + if (!name.endsWith('.json')) { + continue // Durable-write temporaries and foreign files carry no journal state. + } + if (!JOURNAL_FILE.test(name)) { + throw new OrcadMigrationCutoverJournalUnreadableError(`unexpected entry ${name}`) + } + files.push({ name, path: join(directory, name) }) + } + const key = files.map((file) => `${file.name}:${fileVersion(file.path)}`).join('|') + const cached = parsedJournals.get(directory) + if (cached?.key === key) { + return [...cached.cutovers] + } + const cutovers = files.map((file) => + readJournalFile(file.path, file.name.slice(0, -'.json'.length)) + ) + parsedJournals.set(directory, { key, cutovers }) + return [...cutovers] +} + +/** The target's current cutover: the head of its chain, which no later delta move supersedes. */ +export function findOrcadMigrationSourceCutoverForTarget( + userDataPath: string, + sshTargetId: string +): OrcadMigrationSourceCutover | null { + const chain = listOrcadMigrationCutoverChainForTarget(userDataPath, sshTargetId) + return chain.at(-1) ?? null +} + +/** Oldest first; throws unless the target's journals form one chain of delta moves. */ +export function listOrcadMigrationCutoverChainForTarget( + userDataPath: string, + sshTargetId: string +): OrcadMigrationSourceCutover[] { + const matches = listOrcadMigrationSourceCutovers(userDataPath).filter( + (cutover) => cutover.sshTargetId === sshTargetId + ) + const superseded = new Set(matches.map((cutover) => cutover.supersedesMigrationId)) + const heads = matches.filter((cutover) => !superseded.has(cutover.migrationId)) + if (heads.length > 1) { + throw new OrcadMigrationCutoverJournalUnreadableError( + `${heads.length} journals name SSH target ${sshTargetId}` + ) + } + const byId = new Map(matches.map((cutover) => [cutover.migrationId, cutover])) + const chain: OrcadMigrationSourceCutover[] = [] + let entry: OrcadMigrationSourceCutover | undefined = heads[0] + while (entry && chain.length <= matches.length) { + chain.unshift(entry) + entry = entry.supersedesMigrationId ? byId.get(entry.supersedesMigrationId) : undefined + } + if (chain.length !== matches.length) { + throw new OrcadMigrationCutoverJournalUnreadableError( + `the journals naming SSH target ${sshTargetId} do not form one chain` + ) + } + return chain +} + +/** Durable before it returns: a fence written after this always has its journal on disk. */ +export function writeOrcadMigrationSourceCutover( + userDataPath: string, + cutover: OrcadMigrationSourceCutover +): void { + const parsed = parseOrcadMigrationSourceCutover(cutover) + const existing = listOrcadMigrationSourceCutovers(userDataPath) + // Why in flight only: a retained cutover is finished and may wait two releases for retirement. + const inFlight = existing.filter( + (entry) => entry.phase !== 'source-retired' && !isRetainedOrcadMigrationSourceCutover(entry) + ) + if ( + !existing.some((entry) => entry.migrationId === parsed.migrationId) && + inFlight.length >= MAX_ORCAD_MIGRATION_SOURCE_CUTOVERS + ) { + throw new Error('orcad_migration_cutover_capacity_exceeded') + } + const directory = orcadMigrationCutoverJournalDirectory(userDataPath) + mkdirSync(directory, { recursive: true, mode: 0o700 }) + writeSecureJsonFileWithinLimit( + join(directory, `${parsed.migrationId}.json`), + parsed, + MAX_JOURNAL_FILE_BYTES, + { durable: true } + ) + syncDirectoryDurablySync(directory) + parsedJournals.delete(directory) + notifyJournalChanged(userDataPath) +} + +export function removeOrcadMigrationSourceCutover(userDataPath: string, migrationId: string): void { + if (!JOURNAL_FILE.test(`${migrationId}.json`)) { + throw new Error('orcad_migration_cutover_id_invalid') + } + const directory = orcadMigrationCutoverJournalDirectory(userDataPath) + rmSync(join(directory, `${migrationId}.json`), { force: true }) + parsedJournals.delete(directory) + if (existsSync(directory)) { + syncDirectoryDurablySync(directory) + } + notifyJournalChanged(userDataPath) +} + +/** Every journal a host's stopped or never-registered server leaves behind; they grant nothing. */ +export function removeOrcadMigrationJournalsForDestination( + userDataPath: string, + sshTargetId: string, + environmentId: string +): void { + for (const cutover of listOrcadMigrationSourceCutovers(userDataPath)) { + if (cutover.sshTargetId === sshTargetId && cutover.destinationEnvironmentId === environmentId) { + removeOrcadMigrationSourceCutover(userDataPath, cutover.migrationId) + } + } +} + +/** Durable writes replace the file, so size, mtime and inode change with every rewrite. */ +function fileVersion(path: string): string { + try { + const stat = statSync(path, { bigint: true }) + return `${stat.ino}:${stat.size}:${stat.mtimeNs}` + } catch (error) { + throw new OrcadMigrationCutoverJournalUnreadableError(errorMessage(error)) + } +} + +function readJournalFile(path: string, migrationId: string): OrcadMigrationSourceCutover { + try { + const raw = readNodeFileSyncWithinLimit(path, MAX_JOURNAL_FILE_BYTES).buffer.toString('utf8') + const cutover = parseOrcadMigrationSourceCutover(JSON.parse(raw)) + if (cutover.migrationId !== migrationId) { + throw new Error('file name does not match its migration id') + } + return cutover + } catch (error) { + throw new OrcadMigrationCutoverJournalUnreadableError(errorMessage(error)) + } +} diff --git a/src/main/ssh/orcad-migration-delta-move.test.ts b/src/main/ssh/orcad-migration-delta-move.test.ts new file mode 100644 index 00000000000..f84baa16a73 --- /dev/null +++ b/src/main/ssh/orcad-migration-delta-move.test.ts @@ -0,0 +1,660 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { Repo } from '../../shared/repo-types' +import type { SshTarget } from '../../shared/ssh-types' +import type { WorkspaceSessionState } from '../../shared/workspace-session-state-types' +import { folderWorkspaceKey } from '../../shared/workspace-scope' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' +import { fakeOrcadMigrationDestination } from './orcad-migration-destination-fake' +import { keepOrcadServerVersion, runOrcadDeltaMove } from './orcad-migration-delta-move' +import { planOrcadDeltaMove } from './orcad-migration-delta-plan' +import { latestOrcadMigrationInto } from './orcad-migration-rollback-mark' +import { retainOrcadMigrationSource } from './orcad-migration-source-retention' +import { reconcileManagedOrcadSshTargets, visibleRepos } from './orcad-retained-source' +import { SshConnectionStore } from './ssh-connection-store' +import { resolveHostServerOnConnect } from './ssh-host-server-on-connect' +import { hostServerDepsStub } from './ssh-host-server-on-connect-test-deps' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' + +const mocks = vi.hoisted(() => { + const state: { targetStore: unknown } = { targetStore: null } + return { state, deploy: vi.fn(), ensureTunnel: vi.fn() } +}) +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({}), + getSshTargetRegistryStore: () => mocks.state.targetStore, + hasRegisteredDirectSshAuthority: () => false +})) +vi.mock('./orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensureTunnel })) + +const { convertSshTargetToManagedOrcad } = await import('./orcad-runtime-conversion') + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} + +let userDataPath: string +let store: Store +let sshStore: SshConnectionStore +let destination: ReturnType + +function repo(id: string, path: string): Repo { + return { + id, + path, + displayName: id, + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + } +} + +beforeEach(() => { + vi.resetAllMocks() + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-delta-')) + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo(repo('repo-1', '/srv/app')) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + sshStore = new SshConnectionStore(store as never) + mocks.state.targetStore = sshStore + destination = fakeOrcadMigrationDestination() + mocks.ensureTunnel.mockResolvedValue(undefined) + mocks.deploy.mockImplementation(async (path: string, args: { name: string }) => { + const id = getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))! + if (!listEnvironments(path).some((entry) => entry.id === id)) { + addManagedOrcadEnvironment(path, { + id, + name: args.name, + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/', + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: TARGET.id, + sshTargetGeneration: 2, + localPort: 46_768, + remotePort: 6_768 + } + }) + } + return { outcome: 'created', environment: {}, activeVersion: '1.0.0' } + }) +}) + +afterEach(async () => { + await closeTestStores() + rmSync(userDataPath, { recursive: true, force: true }) +}) + +const now = () => new Date('2026-10-03T00:00:00.000Z') + +/** Converted with source retirement off, then an older build adds repo-2 and renames repo-1. */ +async function convertedThenChangedOnOlderBuild(): Promise { + await expect( + convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: TARGET.id, + name: 'Managed', + listRelayPtyIds: Object.assign(async () => [], { previous: async () => [] }), + censusHost: async () => ({ verdict: 'exited', count: 0 }), + destinationFor: () => destination, + releaseDirectSession: async () => {}, + now + }) + ).resolves.toMatchObject({ outcome: 'converted' }) + store.addRepo(repo('repo-2', '/srv/tool')) + store.updateRepo('repo-1', { displayName: 'app-renamed' }) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() +} + +function deltaMove(at: () => Date = now) { + const target = store.getSshTarget(TARGET.id)! + return runOrcadDeltaMove({ + userDataPath, + store, + claims: sshStore.getOrcadRuntimeClaims(), + target, + environment: listEnvironments(userDataPath)[0]!, + destination, + // This relay and every earlier-build relay answer that nothing runs. + listRelayPtyIds: Object.assign(async () => [], { previous: async () => [] }), + censusHost: async () => ({ verdict: 'exited', count: 0 }), + releaseDirectSession: async () => {}, + ensureTunnel: async () => {}, + runTargetLifecycle, + now: at + }) +} + +const LEAF = '11111111-1111-4111-8111-111111111111' + +/** What v1.4.218 leaves after a downgrade: a terminal in what it added, and client focus there. */ +function olderBuildSessionAfterDowngrade(): void { + const hostId = `ssh:${TARGET.id}` as const + const group = store.createProjectGroup({ + name: 'downgrade-added', + parentPath: '/srv/folders', + connectionId: TARGET.id, + createdFrom: 'manual' + }) + const folder = store.createFolderWorkspace({ + projectGroupId: group.id, + name: 'downgrade-added workspace', + folderPath: '/srv/folders/added', + connectionId: TARGET.id + }) + const folderKey = folderWorkspaceKey(folder.id) + const environmentId = getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))! + const focus: Partial = { + activeRepoId: null, + activeWorktreeId: folderKey, + activeWorkspaceKey: folderKey, + activeWorkspaceExecutionHostId: hostId, + activeTabId: 'tab-term', + activeConnectionIdsAtShutdown: [TARGET.id] + } + store.setWorkspaceSession({ ...store.getWorkspaceSession(), ...focus }) + store.setWorkspaceSession( + { + ...store.getWorkspaceSession(hostId), + ...focus, + tabsByWorktree: { + [folderKey]: [ + { + id: 'tab-term', + ptyId: `${hostId}@@pty2:relay:1`, + worktreeId: folderKey, + title: 'Terminal 1', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'tab-term': { + root: { type: 'leaf', leafId: LEAF }, + activeLeafId: LEAF, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF]: `${hostId}@@pty2:relay:1` } + } + }, + terminalPtyIncarnationsByPaneKey: { [`tab-term:${LEAF}`]: 'incarnation-1' }, + terminalTopologyRevisionByRepoId: { [folderKey]: 1 }, + activeWorktreeIdsOnShutdown: [folderKey], + unifiedTabs: { + // The managed build stamped repo-1's editor tab with its server before the downgrade. + 'repo-1::/srv/app': [ + { + id: 'tab-editor', + entityId: '/srv/app/README.md', + groupId: 'group-editor', + worktreeId: 'repo-1::/srv/app', + executionHostId: `runtime:${environmentId}`, + contentType: 'editor', + label: 'README.md', + customLabel: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + } + }, + hostId + ) + store.upsertSshRemotePtyLease({ + targetId: TARGET.id, + ptyId: 'pty2:relay:1', + worktreeId: folderKey, + tabId: 'tab-term', + leafId: LEAF, + state: 'expired' + }) +} + +/** The delta's commit and every read after it fail, as when the tunnel drops mid-commit. */ +function loseContactAtDeltaCommit(): () => void { + const read = destination.readState.getMockImplementation()! + let lost = false + destination.commit.mockImplementationOnce(async () => { + lost = true + throw new Error('socket closed') + }) + destination.readState.mockImplementation(async (manifest) => { + if (lost) { + throw new Error('socket closed') + } + return read(manifest) + }) + return () => { + lost = false + } +} + +describe('moving what an older build added to a converted host', () => { + it('previews additions and what the server keeps, then moves only the additions', async () => { + await convertedThenChangedOnOlderBuild() + const plan = planOrcadDeltaMove(userDataPath, store, store.getSshTarget(TARGET.id)!) + expect(plan.added.map((row) => row.id)).toEqual(['repo-2']) + expect(plan.notReflected.edited.map((row) => row.id)).toEqual(['repo-1']) + expect(plan.notReflected.removed).toEqual([]) + + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + expect(destination.commits).toBe(2) + const journals = listOrcadMigrationSourceCutovers(userDataPath) + const [first, delta] = [...journals].sort((a) => (a.supersedesMigrationId ? 1 : -1)) + expect(delta?.supersedesMigrationId).toBe(first?.migrationId) + expect(delta?.manifest.payload.repositories.map((row) => row.id)).toEqual(['repo-2']) + expect(journals.every((journal) => journal.sourceRetainedAt)).toBe(true) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + expect(visibleRepos(store, () => userDataPath)).toEqual([]) + // Back to managed, and a start with nothing new keeps it there. + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + }) + + it('fails the whole delta when the server already holds a colliding row', async () => { + await convertedThenChangedOnOlderBuild() + destination.stage.mockRejectedValueOnce( + new Error('orcad_migration_repository_id_conflict:repo-2') + ) + await expect(deltaMove()).resolves.toMatchObject({ + outcome: 'refused', + code: 'orcad_delta_refused_by_server', + reason: 'orcad_migration_repository_id_conflict:repo-2' + }) + expect(destination.commits).toBe(1) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + expect( + visibleRepos(store, () => userDataPath) + .map((row) => row.id) + .sort() + ).toEqual(['repo-1', 'repo-2']) + }) + + it('keeps the server version: back to managed, the older build changes stay unshown', async () => { + await convertedThenChangedOnOlderBuild() + await keepOrcadServerVersion({ + userDataPath, + store, + claims: sshStore.getOrcadRuntimeClaims(), + target: store.getSshTarget(TARGET.id)! + }) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + expect(visibleRepos(store, () => userDataPath)).toEqual([]) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + expect(destination.commits).toBe(1) + }) + + it('survives a second downgrade and re-upgrade after the delta move', async () => { + await convertedThenChangedOnOlderBuild() + await deltaMove() + // A second trip to an older build adds another project. + store.addRepo(repo('repo-3', '/srv/docs')) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + const plan = planOrcadDeltaMove(userDataPath, store, store.getSshTarget(TARGET.id)!) + expect(plan.added.map((row) => row.id)).toEqual(['repo-3']) + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + expect(destination.commits).toBe(3) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(3) + expect(visibleRepos(store, () => userDataPath)).toEqual([]) + }) + + it('moves what a downgrade added despite its exited terminal, tabs and client focus', async () => { + await convertedThenChangedOnOlderBuild() + olderBuildSessionAfterDowngrade() + await store.upsertSshPtyConsumerRecovery({ + targetId: TARGET.id, + clientInstanceId: 'client-1', + serverBuildId: '0.1.0', + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'lease' + }) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + + const plan = planOrcadDeltaMove(userDataPath, store, store.getSshTarget(TARGET.id)!) + expect(plan.blockers).toEqual([]) + expect(plan.added.map((row) => row.kind).sort()).toEqual([ + 'folder-workspace', + 'project-group', + 'repository' + ]) + // The relay answers with no terminals, so the expired lease is proven exited. + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + const delta = listOrcadMigrationSourceCutovers(userDataPath).find( + (journal) => journal.supersedesMigrationId + ) + const session = delta?.manifest.payload.dormantState?.workspaceSession + expect(Object.values(session?.tabsByWorktree ?? {}).flat()).toMatchObject([ + { id: 'tab-term', ptyId: null } + ]) + expect(session?.unifiedTabs?.['repo-1::/srv/app']).toBeUndefined() + }) + + it('resumes a delta whose commit lost contact, keeping the host marked meanwhile', async () => { + await convertedThenChangedOnOlderBuild() + const reconnect = loseContactAtDeltaCommit() + await expect(deltaMove()).rejects.toThrow('socket closed') + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(2) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + + reconnect() + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + expect(destination.commits).toBe(2) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(2) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + }) + + it('backs out an interrupted delta the source has since outgrown, then moves afresh', async () => { + await convertedThenChangedOnOlderBuild() + const reconnect = loseContactAtDeltaCommit() + await expect(deltaMove()).rejects.toThrow('socket closed') + reconnect() + await expect( + keepOrcadServerVersion({ + userDataPath, + store, + claims: sshStore.getOrcadRuntimeClaims(), + target: store.getSshTarget(TARGET.id)! + }) + ).rejects.toThrow('orcad_delta_move_unfinished') + + store.addRepo(repo('repo-3', '/srv/docs')) + await expect(deltaMove()).resolves.toMatchObject({ + outcome: 'refused', + reason: 'orcad_migration_source_changed' + }) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + expect(destination.commits).toBe(2) + }) + + it('refuses a delta whose source changes while it stages', async () => { + await convertedThenChangedOnOlderBuild() + const stage = destination.stage.getMockImplementation()! + destination.stage.mockImplementationOnce(async (manifest) => { + store.addRepo(repo('repo-3', '/srv/docs')) + return stage(manifest) + }) + await expect(deltaMove()).resolves.toMatchObject({ + outcome: 'refused', + reason: 'orcad_migration_source_changed' + }) + expect(destination.commits).toBe(1) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + }) + + it('runs one of two concurrent moves; the other finds it superseded', async () => { + await convertedThenChangedOnOlderBuild() + const results = await Promise.all([deltaMove(), deltaMove()]) + expect(results.map((result) => result.outcome).sort()).toEqual(['moved', 'refused']) + expect(destination.commits).toBe(2) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(2) + }) + + it('gives a delta a crash interrupted its mark back on the next start', async () => { + await convertedThenChangedOnOlderBuild() + const reconnect = loseContactAtDeltaCommit() + await expect(deltaMove()).rejects.toThrow('socket closed') + reconnect() + // What a crash before the mark came back leaves: the delta journal, a fence without its mark. + const environmentId = store.getSshTarget(TARGET.id)!.orcadFence!.environmentId + store.updateSshTarget(TARGET.id, { orcadFence: { environmentId } }) + expect(visibleRepos(store, () => userDataPath)).toHaveLength(2) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + }) +}) + +const HOST_ID = `ssh:${TARGET.id}` as const + +/** An unsaved editor draft in the host's session partition, as either build would save it. */ +function saveDraft(worktreeId: string, content: string): void { + store.setWorkspaceSession( + { + ...store.getWorkspaceSession(HOST_ID), + openFilesByWorktree: { + [worktreeId]: [ + { + filePath: '/srv/notes.md', + relativePath: 'notes.md', + worktreeId, + language: 'markdown', + dirtyDraftContent: content + } + ] + } + }, + HOST_ID + ) +} + +function savedDraft(worktreeId: string): string | undefined { + return store.getWorkspaceSession(HOST_ID).openFilesByWorktree?.[worktreeId]?.[0] + ?.dirtyDraftContent +} + +async function convertKeepingSource(): Promise { + await expect( + convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: TARGET.id, + name: 'Managed', + listRelayPtyIds: Object.assign(async () => [], { previous: async () => [] }), + censusHost: async () => ({ verdict: 'exited', count: 0 }), + destinationFor: () => destination, + releaseDirectSession: async () => {}, + now + }) + ).resolves.toMatchObject({ outcome: 'converted' }) +} + +/** Everything of the host's that this build keeps for a downgrade, as stored. */ +function retainedRows(): string { + return JSON.stringify({ + repos: store.getRepos(), + folderWorkspaces: store.getFolderWorkspaces(), + projectGroups: store.getProjectGroups(), + hostSession: store.getWorkspaceSession(HOST_ID), + localSession: store.getWorkspaceSession() + }) +} + +const expectAllRetained = () => + expect(listOrcadMigrationSourceCutovers(userDataPath).every((j) => j.sourceRetainedAt)).toBe(true) + +/** What a connect does with a committed head; there is no retirement step any more. */ +async function connectAgain(): Promise { + const target = store.getSshTarget(TARGET.id)! + const environmentId = target.orcadFence!.environmentId + await resolveHostServerOnConnect(target, { + ...hostServerDepsStub(), + managedEnvironmentId: () => environmentId, + retainCommittedSource: (host) => { + const head = listOrcadMigrationSourceCutovers(userDataPath).find( + (journal) => journal.sshTargetId === host.id && !journal.sourceRetainedAt + ) + if (head?.phase === 'destination-committed') { + retainOrcadMigrationSource(userDataPath, head.migrationId, now) + } + } + }) +} + +/** A restart: the profile and journals are re-read from disk by a fresh store. */ +async function restart(): Promise { + await store.flushPendingOrThrowAsync() + await closeTestStores() + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + sshStore = new SshConnectionStore(store as never) + mocks.state.targetStore = sshStore + reconcileManagedOrcadSshTargets(userDataPath, store, now) +} + +describe('a retained source is never deleted', () => { + it.each([ + ['a repository worktree', () => 'repo-1::/srv/app'], + [ + 'a folder workspace on a folder-only host', + () => { + store.removeProject('repo-1') + const group = store.createProjectGroup({ + name: 'folders', + parentPath: '/srv/folders', + connectionId: TARGET.id, + createdFrom: 'manual' + }) + const folder = store.createFolderWorkspace({ + projectGroupId: group.id, + folderPath: '/srv/folders/notes', + connectionId: TARGET.id + }) + return folderWorkspaceKey(folder.id) + } + ] + ])('in %s, across connects and a restart', async (_label, workspace) => { + const worktreeId = workspace() + saveDraft(worktreeId, 'draft before migration') + await convertKeepingSource() + // A reload only fills defaults; the snapshot is taken as stored. + await restart() + const kept = retainedRows() + + await connectAgain() + expect(retainedRows()).toBe(kept) + await restart() + await connectAgain() + expect(retainedRows()).toBe(kept) + expect(savedDraft(worktreeId)).toBe('draft before migration') + expectAllRetained() + }) + + // The trade-off: identity drives "changed", so an edit inside a moved project only stays kept. + it('keeps an older build’s draft edit without marking the host changed', async () => { + saveDraft('repo-1::/srv/app', 'draft before migration') + await convertKeepingSource() + saveDraft('repo-1::/srv/app', 'draft after downgrade') + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + + await connectAgain() + await restart() + await connectAgain() + expect(savedDraft('repo-1::/srv/app')).toBe('draft after downgrade') + expectAllRetained() + }) + + it('keeps every row through a delta move and through keeping the server version', async () => { + await convertedThenChangedOnOlderBuild() + await restart() + const kept = retainedRows() + await expect(deltaMove()).resolves.toMatchObject({ outcome: 'moved' }) + expect(retainedRows()).toBe(kept) + await restart() + await connectAgain() + expect(retainedRows()).toBe(kept) + + store.addRepo(repo('repo-3', '/srv/docs')) + reconcileManagedOrcadSshTargets(userDataPath, store, now) + const changed = retainedRows() + await keepOrcadServerVersion({ + userDataPath, + store, + claims: sshStore.getOrcadRuntimeClaims(), + target: store.getSshTarget(TARGET.id)! + }) + expect(retainedRows()).toBe(changed) + await restart() + await connectAgain() + expect(retainedRows()).toBe(changed) + expectAllRetained() + }) +}) + +describe('a delta move against a rollback of an update taken before it', () => { + const later = () => new Date('2026-10-05T00:00:00.000Z') + // An update activated after the conversion and before the delta: its snapshot lacks the delta. + const updateActivatedAt = Date.parse('2026-10-04T00:00:00.000Z') + const rollbackCrossesMigration = () => + updateActivatedAt < + Date.parse(latestOrcadMigrationInto(userDataPath, listEnvironments(userDataPath)[0]!) ?? '') + + it('marks the server before the delta commits, so the rollback is refused', async () => { + await convertedThenChangedOnOlderBuild() + expect(rollbackCrossesMigration()).toBe(false) + const commit = destination.commit.getMockImplementation()! + destination.commit.mockImplementationOnce(async (manifest) => { + expect(listEnvironments(userDataPath)[0]?.orcadMigratedAt).toBe(later().toISOString()) + return commit(manifest) + }) + await expect(deltaMove(later)).resolves.toMatchObject({ outcome: 'moved' }) + expect(rollbackCrossesMigration()).toBe(true) + }) + + it('keeps the mark when the commit lands but its reply is lost', async () => { + await convertedThenChangedOnOlderBuild() + const commit = destination.commit.getMockImplementation()! + const read = destination.readState.getMockImplementation()! + let lost = false + destination.commit.mockImplementationOnce(async (manifest) => { + await commit(manifest) + lost = true + throw new Error('socket closed') + }) + destination.readState.mockImplementation(async (manifest) => { + if (lost) { + throw new Error('socket closed') + } + return read(manifest) + }) + await expect(deltaMove(later)).rejects.toThrow('socket closed') + expect(destination.commits).toBe(2) + expect(rollbackCrossesMigration()).toBe(true) + }) + + it('protects a folder-only delta the same way', async () => { + await convertedThenChangedOnOlderBuild() + store.removeProject('repo-2') + const group = store.createProjectGroup({ + name: 'folders', + parentPath: '/srv/folders', + connectionId: TARGET.id, + createdFrom: 'manual' + }) + store.createFolderWorkspace({ + projectGroupId: group.id, + folderPath: '/srv/folders/notes', + connectionId: TARGET.id + }) + const plan = planOrcadDeltaMove(userDataPath, store, store.getSshTarget(TARGET.id)!) + expect(plan.added.map((row) => row.kind).sort()).toEqual(['folder-workspace', 'project-group']) + await expect(deltaMove(later)).resolves.toMatchObject({ outcome: 'moved' }) + expect(rollbackCrossesMigration()).toBe(true) + }) +}) diff --git a/src/main/ssh/orcad-migration-delta-move.ts b/src/main/ssh/orcad-migration-delta-move.ts new file mode 100644 index 00000000000..e0180fd7dae --- /dev/null +++ b/src/main/ssh/orcad-migration-delta-move.ts @@ -0,0 +1,243 @@ +/** + * The delta move: a fresh, journaled conversion of only what an older build added to a converted + * host, committed to the same managed server. Its journal supersedes the previous head of the + * host's chain; both stay, and the source rows of every manifest in the chain are kept. + * + * A row the server already holds under another identity fails the whole move at stage, before + * anything is committed: the journal goes, and the host keeps its "changed" mark and the relay. + * A move whose outcome the server can't confirm keeps its journal, and the next move resumes it. + */ +import { + ORCAD_MIGRATION_SOURCE_CUTOVER_VERSION, + type OrcadMigrationSourceCutover +} from '../../shared/orcad-migration-source-cutover' +import type { OrcadDeltaMoveResult } from '../../shared/orcad-managed-runtime' +import { recordManagedOrcadMigration } from '../../shared/runtime-environment-managed-orcad-store' +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import type { SshTarget } from '../../shared/ssh-types' +import type { Store } from '../persistence' +import { + abortOrcadMigrationCutover, + commitOrcadMigrationDestination, + type OrcadMigrationCutoverContext, + type OrcadMigrationDestinationCatalog +} from './orcad-migration-cutover-coordinator' +import { + findOrcadMigrationSourceCutoverForTarget, + listOrcadMigrationCutoverChainForTarget, + removeOrcadMigrationSourceCutover, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' +import { + committedOrcadMigrationChain, + orcadDeltaSourceStore, + planOrcadDeltaMove, + unfinishedOrcadDelta, + type OrcadDeltaMovePlan +} from './orcad-migration-delta-plan' +import { retainOrcadMigrationSource } from './orcad-migration-source-retention' +import { + assessOrcadMigrationTerminals, + retireProvenExitedLeases, + type CensusHostRelayTerminals, + type ListRelayPtyIds +} from './orcad-migration-terminal-gate' +import { currentOrcadSourceFingerprint } from './orcad-retained-source' +import type { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { orcadMigrationRefusalReason } from './orcad-migration-refusal-reason' +import { errorMessage } from '../../shared/error-message' + +export type OrcadDeltaMoveArgs = { + userDataPath: string + store: Store + claims: SshTargetOrcadClaims + target: SshTarget + environment: KnownRuntimeEnvironment + destination: OrcadMigrationDestinationCatalog + listRelayPtyIds: ListRelayPtyIds | null + /** With no relay session to ask, the census of the host's relay endpoints that must prove exit. */ + censusHost?: CensusHostRelayTerminals | null + /** Releases the relay session after the terminal check, as a conversion does. */ + releaseDirectSession: (sshTargetId: string) => Promise + ensureTunnel: () => Promise + /** Serializes the journal write through the commit with every other change to this host. */ + runTargetLifecycle: (targetId: string, operation: () => Promise) => Promise + now?: () => Date +} + +export async function runOrcadDeltaMove(args: OrcadDeltaMoveArgs): Promise { + const { userDataPath, store, target } = args + const now = args.now ?? (() => new Date()) + const plan = planOrcadDeltaMove(userDataPath, store, target, { now }) + // Taken with the manifest, before any await: the baseline must describe what the server receives. + const planned = currentOrcadSourceFingerprint(store, target) + if (!plan.resumes && plan.added.length === 0) { + return refuse('orcad_delta_nothing_new', 'An older build added nothing new to move.') + } + if (plan.blockers.length > 0) { + return { + ...refuse('orcad_migration_preflight_blocked', orcadMigrationRefusalReason(plan.blockers)), + blockers: plan.blockers + } + } + const terminals = await assessOrcadMigrationTerminals( + store, + target.id, + args.listRelayPtyIds, + args.censusHost + ) + if (terminals.verdict !== 'exited') { + return refuse('orcad_migration_terminals', terminals.reason) + } + retireProvenExitedLeases(store, target.id, terminals) + await args.releaseDirectSession(target.id) + return args.runTargetLifecycle(target.id, async () => { + // Why re-checked: another move of this host may have journaled or finished while this waited. + const changedAt = store.getSshTarget(target.id)?.orcadFence?.sourceChangedAt + const head = findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) + if (!changedAt || head?.migrationId !== (plan.resumes ?? plan.head).migrationId) { + return refuse('orcad_delta_superseded', 'Another move of this host ran first.') + } + // The source stays writable until the fence below: a draft typed while the terminals were + // checked is newer than the manifest, so it must not become the baseline the server is held to. + if (!plan.resumes && planned !== currentOrcadSourceFingerprint(store, target)) { + return refuse( + 'orcad_delta_source_changed', + 'This host changed while the move was starting. Try the move again.' + ) + } + const cutover = plan.resumes ?? journalDelta(args, plan, planned, terminals.provenPtyIds, now) + return commitDelta(args, plan, cutover, changedAt, now) + }) +} + +function journalDelta( + args: OrcadDeltaMoveArgs, + plan: OrcadDeltaMovePlan, + planned: string, + provenPtyIds: string[], + now: () => Date +): OrcadMigrationSourceCutover { + const timestamp = now().toISOString() + const cutover: OrcadMigrationSourceCutover = { + version: ORCAD_MIGRATION_SOURCE_CUTOVER_VERSION, + migrationId: plan.manifest.migrationId, + phase: 'source-fenced', + startedAt: timestamp, + updatedAt: timestamp, + destinationEnvironmentId: plan.environmentId, + destinationName: plan.head.destinationName, + sshTargetId: args.target.id, + sshTargetGeneration: plan.head.sshTargetGeneration, + manifestSha256: plan.manifest.manifestSha256, + provenPtyIds, + supersedesMigrationId: plan.head.migrationId, + // The whole source as the manifest saw it: what the retained rows must keep matching. + sourceBaselineFingerprint: planned, + manifest: plan.manifest + } + writeOrcadMigrationSourceCutover(args.userDataPath, cutover) + return cutover +} + +async function commitDelta( + args: OrcadDeltaMoveArgs, + plan: OrcadDeltaMovePlan, + cutover: OrcadMigrationSourceCutover, + changedAt: string, + now: () => Date +): Promise { + const { userDataPath, store, target } = args + // The fence goes back without its "changed" mark: the source freezes for the move. + store.updateSshTarget(target.id, { orcadFence: { environmentId: plan.environmentId } }) + await args.claims.flush() + const context: OrcadMigrationCutoverContext = { + userDataPath, + store: plan.source, + freshSource: () => orcadDeltaSourceStore(store, target, plan.moved), + claims: args.claims, + destination: args.destination, + now + } + // Before any commit can land, resumed ones included: a rollback must not cross this move. + recordManagedOrcadMigration(userDataPath, plan.environmentId, now().toISOString()) + try { + await args.ensureTunnel() + const committed = await commitOrcadMigrationDestination(context, cutover.migrationId) + if (committed.phase !== 'destination-committed' && committed.phase !== 'source-retired') { + throw new Error(`orcad_migration_commit_not_proven:${committed.phase}`) + } + } catch (error) { + const released = await releaseUncommittedDelta(args, context, cutover, changedAt) + if (released === 'released') { + return refuse('orcad_delta_refused_by_server', errorMessage(error)) + } + if (released === 'kept') { + throw error + } + } + retainOrcadMigrationSource(userDataPath, cutover.migrationId, now) + return { outcome: 'moved', migrationId: cutover.migrationId } +} + +/** + * Undoes a delta only once the server provably holds nothing of it, so no row moves twice. One it + * may hold stays journaled for the next move to resume; either way the host gets its mark back. + */ +async function releaseUncommittedDelta( + args: OrcadDeltaMoveArgs, + context: OrcadMigrationCutoverContext, + cutover: OrcadMigrationSourceCutover, + changedAt: string +): Promise<'released' | 'committed' | 'kept'> { + const restoreMark = async (): Promise => { + args.store.updateSshTarget(cutover.sshTargetId, { + orcadFence: { environmentId: cutover.destinationEnvironmentId, sourceChangedAt: changedAt } + }) + await args.claims.flush() + } + try { + const result = await abortOrcadMigrationCutover(context, cutover.migrationId, async () => { + // Mark first: a crash before the journal goes leaves a marked host that resumes this delta. + await restoreMark() + removeOrcadMigrationSourceCutover(args.userDataPath, cutover.migrationId) + }) + return result.outcome === 'released' ? 'released' : 'committed' + } catch { + await restoreMark() + return 'kept' + } +} + +/** "Keep the server's version": the older build's changes stay only in the retained profile rows. */ +export async function keepOrcadServerVersion(args: { + userDataPath: string + store: Store + claims: SshTargetOrcadClaims + target: SshTarget +}): Promise { + const environmentId = args.target.orcadFence?.environmentId + const chain = listOrcadMigrationCutoverChainForTarget(args.userDataPath, args.target.id) + if (unfinishedOrcadDelta(chain)) { + // Its rows may be on the server already; only finishing or undoing the move can say. + throw new Error('orcad_delta_move_unfinished') + } + const head = committedOrcadMigrationChain(chain).at(-1) + if (!environmentId || !args.target.orcadFence?.sourceChangedAt || !head) { + throw new Error('orcad_delta_not_changed') + } + writeOrcadMigrationSourceCutover(args.userDataPath, { + ...head, + // Keeping the server's version is the explicit reconcile: the source as it is now is the baseline. + sourceBaselineFingerprint: currentOrcadSourceFingerprint(args.store, args.target) + }) + args.store.updateSshTarget(args.target.id, { orcadFence: { environmentId } }) + await args.claims.flush() +} + +function refuse( + code: string, + reason: string +): Extract { + return { outcome: 'refused', code, reason } +} diff --git a/src/main/ssh/orcad-migration-delta-plan.ts b/src/main/ssh/orcad-migration-delta-plan.ts new file mode 100644 index 00000000000..da10ab59c51 --- /dev/null +++ b/src/main/ssh/orcad-migration-delta-plan.ts @@ -0,0 +1,189 @@ +/** + * Planning a delta move for a host an older build changed after it was converted: what the move + * adds, what the server keeps as it is, and the source view every later check of the move reads. + * + * The view subtracts what the host's earlier migrations already moved, re-exported as it is today, + * so the delta manifest carries only rows and dormant state no earlier migration owns. Nothing + * that overlaps an earlier migration is merged into the server. + */ +import type { OrcadDeltaMovePreview, OrcadDeltaMoveRow } from '../../shared/orcad-managed-runtime' +import type { + OrcadMigrationCatalogPayload, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import { isRetainedOrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import type { SshTarget } from '../../shared/ssh-types' +import type { Store } from '../persistence' +import { collectOrcadMigrationSourceCatalog } from '../persistence/migrating-orcad-catalog/orcad-source-catalog' +import { listOrcadMigrationCutoverChainForTarget } from './orcad-migration-cutover-journal' +import { + createOrcadMigrationManifest, + orcadMigrationCatalogIds +} from './orcad-migration-manifest-export' +import type { OrcadMigrationSnapshotSource } from './orcad-migration-snapshot-coordinator' +import { collectUntransferredDependentBlockers } from './ssh-target-orcad-dependents' +import type { OrcadMigrationPreflightStore } from './ssh-target-orcad-preflight' + +export type OrcadDeltaSourceStore = OrcadMigrationPreflightStore & OrcadMigrationSnapshotSource + +/** A delta move journaled but not yet committed and kept: the next move resumes it. */ +export function unfinishedOrcadDelta( + chain: readonly OrcadMigrationSourceCutover[] +): OrcadMigrationSourceCutover | null { + const head = chain.at(-1) + return head?.supersedesMigrationId && + head.phase !== 'source-retired' && + !isRetainedOrcadMigrationSourceCutover(head) + ? head + : null +} + +/** The committed migrations a delta extends, oldest first, excluding any delta in flight. */ +export function committedOrcadMigrationChain( + chain: readonly OrcadMigrationSourceCutover[] +): OrcadMigrationSourceCutover[] { + return (unfinishedOrcadDelta(chain) ? chain.slice(0, -1) : chain).filter( + (cutover) => cutover.phase === 'destination-committed' || cutover.phase === 'source-retired' + ) +} + +/** The real store, except its catalog and dormant state: those come from the delta view. */ +export function orcadDeltaSourceStore( + store: Store, + target: SshTarget, + moved: readonly OrcadMigrationSourceCutover[] +): OrcadDeltaSourceStore { + const movedNow = createOrcadMigrationManifest(store, target, { + destinationEnvironmentId: moved.at(-1)?.destinationEnvironmentId, + onlyCatalog: orcadMigrationCatalogIds(moved.map((cutover) => cutover.manifest.payload)) + }) + const view = store.createOrcadMigrationDeltaView(movedNow) + return { + ...view, + getSshTarget: (id) => store.getSshTarget(id), + getSshRemotePtyLeases: (id) => store.getSshRemotePtyLeases(id), + readOrcadMigrationSourceSnapshotChunk: (...args) => + store.readOrcadMigrationSourceSnapshotChunk(...args) + } +} + +export type OrcadDeltaMovePlan = OrcadDeltaMovePreview & { + manifest: OrcadMigrationManifest + moved: OrcadMigrationSourceCutover[] + /** The retained migration a new delta supersedes. */ + head: OrcadMigrationSourceCutover + /** An interrupted delta this move resumes from its journaled manifest instead of a new one. */ + resumes: OrcadMigrationSourceCutover | null + source: OrcadDeltaSourceStore +} + +/** Throws when the host is not a converted host an older build changed. */ +export function planOrcadDeltaMove( + userDataPath: string, + store: Store, + target: SshTarget, + options: { migrationId?: string; now?: () => Date } = {} +): OrcadDeltaMovePlan { + const environmentId = target.orcadFence?.environmentId + if (!environmentId || !target.orcadFence?.sourceChangedAt) { + throw new Error('orcad_delta_not_changed') + } + const chain = listOrcadMigrationCutoverChainForTarget(userDataPath, target.id) + const resumes = unfinishedOrcadDelta(chain) + const moved = committedOrcadMigrationChain(chain) + const head = chain.at(resumes ? -2 : -1) + if ( + !head || + !isRetainedOrcadMigrationSourceCutover(head) || + moved.at(-1)?.migrationId !== head.migrationId + ) { + throw new Error('orcad_delta_no_retained_migration') + } + const source = orcadDeltaSourceStore(store, target, moved) + const manifest = + resumes?.manifest ?? + createOrcadMigrationManifest(source, target, { + migrationId: options.migrationId, + now: options.now, + destinationEnvironmentId: environmentId + }) + const current = collectOrcadMigrationSourceCatalog(store, target) + return { + sshTargetId: target.id, + environmentId, + added: catalogRows(manifest.payload), + notReflected: notReflected( + current, + moved.map((cutover) => cutover.manifest.payload) + ), + blockers: collectUntransferredDependentBlockers(source, manifest), + manifest, + moved, + head, + resumes, + source + } +} + +function catalogRows(catalog: OrcadMigrationCatalogPayload): OrcadDeltaMoveRow[] { + return [ + ...catalog.repositories.map((row) => ({ + kind: 'repository' as const, + id: row.id, + label: row.displayName + })), + ...catalog.folderWorkspaces.map((row) => ({ + kind: 'folder-workspace' as const, + id: row.id, + label: row.name + })), + ...catalog.projectGroups.map((row) => ({ + kind: 'project-group' as const, + id: row.id, + label: row.name + })) + ] +} + +/** Rows earlier migrations moved that an older build has since changed or removed. */ +function notReflected( + current: OrcadMigrationCatalogPayload, + movedPayloads: readonly OrcadMigrationCatalogPayload[] +): OrcadDeltaMovePreview['notReflected'] { + const latestMoved = new Map() + for (const payload of movedPayloads) { + for (const row of catalogRows(payload)) { + latestMoved.set(`${row.kind}:${row.id}`, { row, identity: identityOf(payload, row) }) + } + } + const currentByKey = new Map( + catalogRows(current).map((row) => [`${row.kind}:${row.id}`, identityOf(current, row)]) + ) + const edited: OrcadDeltaMoveRow[] = [] + const removed: OrcadDeltaMoveRow[] = [] + for (const [key, moved] of latestMoved) { + const identity = currentByKey.get(key) + if (identity === undefined) { + removed.push(moved.row) + } else if (identity !== moved.identity) { + edited.push(moved.row) + } + } + return { edited, removed } +} + +function identityOf(catalog: OrcadMigrationCatalogPayload, row: OrcadDeltaMoveRow): string { + switch (row.kind) { + case 'repository': { + const repo = catalog.repositories.find((entry) => entry.id === row.id) + return `${repo?.path}\0${repo?.displayName}` + } + case 'folder-workspace': { + const folder = catalog.folderWorkspaces.find((entry) => entry.id === row.id) + return `${folder?.folderPath}\0${folder?.name}` + } + case 'project-group': + return catalog.projectGroups.find((entry) => entry.id === row.id)?.name ?? '' + } +} diff --git a/src/main/ssh/orcad-migration-delta-snapshot.test.ts b/src/main/ssh/orcad-migration-delta-snapshot.test.ts new file mode 100644 index 00000000000..58533298c31 --- /dev/null +++ b/src/main/ssh/orcad-migration-delta-snapshot.test.ts @@ -0,0 +1,230 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { Repo } from '../../shared/repo-types' +import type { SshTarget } from '../../shared/ssh-types' +import { folderWorkspaceKey } from '../../shared/workspace-scope' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' +import { fakeOrcadMigrationDestination } from './orcad-migration-destination-fake' +import { runOrcadDeltaMove } from './orcad-migration-delta-move' +import { reconcileManagedOrcadSshTargets } from './orcad-retained-source' +import { SshConnectionStore } from './ssh-connection-store' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' + +const mocks = vi.hoisted(() => { + const state: { targetStore: unknown } = { targetStore: null } + return { state, deploy: vi.fn(), ensureTunnel: vi.fn() } +}) +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({}), + getSshTargetRegistryStore: () => mocks.state.targetStore, + hasRegisteredDirectSshAuthority: () => false +})) +vi.mock('./orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensureTunnel })) + +const { convertSshTargetToManagedOrcad } = await import('./orcad-runtime-conversion') + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} + +let userDataPath: string +let store: Store +let sshStore: SshConnectionStore +let destination: ReturnType + +function repo(id: string, path: string): Repo { + return { + id, + path, + displayName: id, + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + } +} + +beforeEach(() => { + vi.resetAllMocks() + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-delta-snapshot-')) + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo(repo('repo-1', '/srv/app')) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + sshStore = new SshConnectionStore(store as never) + mocks.state.targetStore = sshStore + destination = fakeOrcadMigrationDestination() + mocks.ensureTunnel.mockResolvedValue(undefined) + mocks.deploy.mockImplementation(async (path: string, args: { name: string }) => { + const id = getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))! + if (!listEnvironments(path).some((entry) => entry.id === id)) { + addManagedOrcadEnvironment(path, { + id, + name: args.name, + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/', + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: TARGET.id, + sshTargetGeneration: 2, + localPort: 46_768, + remotePort: 6_768 + } + }) + } + return { outcome: 'created', environment: {}, activeVersion: '1.0.0' } + }) +}) + +afterEach(async () => { + await closeTestStores() + rmSync(userDataPath, { recursive: true, force: true }) +}) + +const now = () => new Date('2026-10-03T00:00:00.000Z') + +const HOST_ID = `ssh:${TARGET.id}` as const + +/** An unsaved editor draft in the host's session partition, as the renderer saves it. */ +function saveDraft(worktreeId: string, content: string): void { + store.setWorkspaceSession( + { + ...store.getWorkspaceSession(HOST_ID), + openFilesByWorktree: { + [worktreeId]: [ + { + filePath: '/srv/notes.md', + relativePath: 'notes.md', + worktreeId, + language: 'markdown', + dirtyDraftContent: content + } + ] + } + }, + HOST_ID + ) +} + +function savedDraft(worktreeId: string): string | undefined { + return store.getWorkspaceSession(HOST_ID).openFilesByWorktree?.[worktreeId]?.[0] + ?.dirtyDraftContent +} + +/** Converted with source retirement off, then an older build adds what `addOnOlderBuild` adds. */ +async function convertedThenAdded(addOnOlderBuild: () => string): Promise { + await expect( + convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: TARGET.id, + name: 'Managed', + listRelayPtyIds: Object.assign(async () => [], { previous: async () => [] }), + censusHost: async () => ({ verdict: 'exited', count: 0 }), + destinationFor: () => destination, + releaseDirectSession: async () => {}, + now + }) + ).resolves.toMatchObject({ outcome: 'converted' }) + const worktreeId = addOnOlderBuild() + saveDraft(worktreeId, 'draft before the move') + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeDefined() + return worktreeId +} + +/** The relay answers only after `whileChecking` ran, as a user typing during the check would. */ +function deltaMoveTypingDuringTerminalCheck(whileChecking: () => void) { + const list = async (): Promise => { + whileChecking() + return [] + } + return runOrcadDeltaMove({ + userDataPath, + store, + claims: sshStore.getOrcadRuntimeClaims(), + target: store.getSshTarget(TARGET.id)!, + environment: listEnvironments(userDataPath)[0]!, + destination, + listRelayPtyIds: Object.assign(list, { previous: async () => [] }), + censusHost: async () => ({ verdict: 'exited', count: 0 }), + releaseDirectSession: async () => {}, + ensureTunnel: async () => {}, + runTargetLifecycle, + now + }) +} + +describe('a draft typed while a delta move checks terminals', () => { + it.each([ + [ + 'a repository the older build added', + () => { + store.addRepo(repo('repo-2', '/srv/tool')) + return 'repo-2::/srv/tool' + } + ], + [ + 'a folder workspace the older build added', + () => { + const group = store.createProjectGroup({ + name: 'folders', + parentPath: '/srv/folders', + connectionId: TARGET.id, + createdFrom: 'manual' + }) + const folder = store.createFolderWorkspace({ + projectGroupId: group.id, + folderPath: '/srv/folders/notes', + connectionId: TARGET.id + }) + return folderWorkspaceKey(folder.id) + } + ] + ])('in %s moves as planned and keeps the newer draft in the source', async (_label, add) => { + const worktreeId = await convertedThenAdded(add) + + await expect( + deltaMoveTypingDuringTerminalCheck(() => + saveDraft(worktreeId, 'typed while the terminal check awaited') + ) + ).resolves.toMatchObject({ outcome: 'moved' }) + + // The server gets the draft the plan saw; the newer one stays in the retained rows. + expect(destination.commits).toBe(2) + expect(listOrcadMigrationSourceCutovers(userDataPath).every((j) => j.sourceRetainedAt)).toBe( + true + ) + expect(savedDraft(worktreeId)).toBe('typed while the terminal check awaited') + }) + + it('journals the baseline the manifest saw when nothing changed during the check', async () => { + const worktreeId = await convertedThenAdded(() => { + store.addRepo(repo('repo-2', '/srv/tool')) + return 'repo-2::/srv/tool' + }) + + await expect(deltaMoveTypingDuringTerminalCheck(() => {})).resolves.toMatchObject({ + outcome: 'moved' + }) + expect(savedDraft(worktreeId)).toBe('draft before the move') + // Back to managed, and the next start finds the source as the delta baseline recorded it. + reconcileManagedOrcadSshTargets(userDataPath, store, now) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + }) +}) diff --git a/src/main/ssh/orcad-migration-destination-fake.ts b/src/main/ssh/orcad-migration-destination-fake.ts new file mode 100644 index 00000000000..a5824edf021 --- /dev/null +++ b/src/main/ssh/orcad-migration-destination-fake.ts @@ -0,0 +1,72 @@ +import { vi, type Mock } from 'vitest' +import type { + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationDestinationCatalog } from './orcad-migration-cutover-coordinator' + +type Catalog = OrcadMigrationDestinationCatalog + +export type FakeOrcadMigrationDestination = { commits: number } & { + [Method in keyof Catalog]: Mock +} + +/** + * An in-memory destination with the T6-9 semantics: idempotent stage, receipt-keyed commit, one + * catalog state per migration so a delta move after a first migration starts absent. + */ +export function fakeOrcadMigrationDestination(): FakeOrcadMigrationDestination { + const states = new Map() + const stateOf = (manifest: OrcadMigrationManifest) => states.get(manifest.migrationId) ?? 'absent' + const view = (manifest: OrcadMigrationManifest): OrcadMigrationCatalogState => { + const state = stateOf(manifest) + const base = { migrationId: manifest.migrationId, manifestSha256: manifest.manifestSha256 } + if (state === 'committed') { + return { + ...base, + state, + receipt: { + version: 1, + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256, + source: manifest.source, + importedAt: '2026-10-01T00:00:00.000Z', + repositoryIds: [], + projectGroupIds: [], + folderWorkspaceIds: [] + } + } + } + return state === 'staged' + ? { ...base, state, stagedAt: '2026-10-01T00:00:00.000Z', snapshotUploads: [] } + : { ...base, state } + } + const destination: FakeOrcadMigrationDestination = { + commits: 0, + readState: vi.fn(async (manifest: OrcadMigrationManifest) => + view(manifest) + ), + stage: vi.fn(async (manifest: OrcadMigrationManifest) => { + if (stateOf(manifest) === 'absent') { + states.set(manifest.migrationId, 'staged') + } + return view(manifest) + }), + commit: vi.fn(async (manifest: OrcadMigrationManifest) => { + if (stateOf(manifest) === 'staged') { + states.set(manifest.migrationId, 'committed') + destination.commits += 1 + } + return view(manifest) + }), + abort: vi.fn(async (manifest: OrcadMigrationManifest) => { + const aborted = stateOf(manifest) === 'staged' + if (aborted) { + states.set(manifest.migrationId, 'absent') + } + return { ...view(manifest), aborted, ...(aborted ? {} : { durableAbsent: true as const }) } + }), + stageChunk: vi.fn() + } + return destination +} diff --git a/src/main/ssh/orcad-migration-manifest-export.test.ts b/src/main/ssh/orcad-migration-manifest-export.test.ts new file mode 100644 index 00000000000..8637dca12b2 --- /dev/null +++ b/src/main/ssh/orcad-migration-manifest-export.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../shared/constants' +import type { FolderWorkspace } from '../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../shared/project-group-types' +import type { Repo } from '../../shared/repo-types' +import type { SshTarget } from '../../shared/ssh-types' +import { computeOrcadMigrationManifestSha256 } from '../orcad/orcad-migration-manifest-digest' +import { emptyDormantPayload } from '../persistence/migrating-orcad-catalog/orcad-source-dormant-state' +import { createOrcadMigrationManifest } from './orcad-migration-manifest-export' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 7 +} + +function repo(id: string, connectionId: string, projectGroupId?: string): Repo { + return { + id, + path: `/srv/${id}`, + displayName: id, + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId, + ...(projectGroupId ? { projectGroupId } : {}) + } +} + +function group( + id: string, + connectionId: string | null, + parentGroupId: string | null = null +): ProjectGroup { + return { + id, + name: id, + parentPath: `/srv/${id}`, + connectionId, + parentGroupId, + createdFrom: 'manual', + tabOrder: 1, + isCollapsed: false, + color: null, + createdAt: 1, + updatedAt: 1 + } +} + +function folder(id: string, projectGroupId: string, connectionId?: string): FolderWorkspace { + return { + id, + projectGroupId, + name: id, + folderPath: `/srv/${id}`, + ...(connectionId ? { connectionId } : {}), + linkedTask: null, + comment: '', + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 1, + lastActivityAt: 0, + createdAt: 1, + updatedAt: 1 + } +} + +describe('orcad migration manifest export', () => { + it('exports only the target catalog plus referenced group ancestry', () => { + const groups = [ + group('parent', null), + group('repo-group', 'ssh-prod', 'parent'), + group('folder-group', 'ssh-prod'), + group('other-group', 'ssh-other') + ] + const manifest = createOrcadMigrationManifest( + { + collectOrcadMigrationSourceDormantState: emptyDormantPayload, + getRepos: () => [ + repo('repo-prod', 'ssh-prod', 'repo-group'), + repo('repo-other', 'ssh-other') + ], + getProjectGroups: () => groups, + getFolderWorkspaces: () => [ + folder('folder-inherited', 'folder-group'), + folder('folder-explicit', 'folder-group', 'ssh-prod'), + folder('folder-other', 'other-group', 'ssh-other') + ] + }, + TARGET, + { migrationId: 'migration-1', now: () => new Date('2026-08-30T12:00:00.000Z') } + ) + + expect(manifest.payload.repositories.map((entry) => entry.id)).toEqual(['repo-prod']) + expect(manifest.payload.projectGroups.map((entry) => entry.id)).toEqual([ + 'parent', + 'repo-group', + 'folder-group' + ]) + expect(manifest.payload.folderWorkspaces.map((entry) => entry.id)).toEqual([ + 'folder-inherited', + 'folder-explicit' + ]) + expect(manifest.source).toEqual({ + sshTargetId: 'ssh-prod', + sshTargetGeneration: 7, + targetLabel: 'Production' + }) + const { manifestSha256, ...unsigned } = manifest + expect(manifestSha256).toBe(computeOrcadMigrationManifestSha256(unsigned)) + }) + + it('records a null generation for a legacy registration without mutating it', () => { + const target = { ...TARGET, generation: undefined } + const manifest = createOrcadMigrationManifest( + { + collectOrcadMigrationSourceDormantState: emptyDormantPayload, + getRepos: () => [], + getProjectGroups: () => [], + getFolderWorkspaces: () => [] + }, + target, + { migrationId: 'migration-legacy', now: () => new Date('2026-08-30T12:00:00.000Z') } + ) + + expect(manifest.source.sshTargetGeneration).toBeNull() + expect(target.generation).toBeUndefined() + }) + + it('includes a session-only dormant payload', () => { + const dormant = emptyDormantPayload() + dormant.workspaceSession = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + 'repo-prod::/srv/worktree': [ + { + id: 'tab-dormant', + ptyId: null, + worktreeId: 'repo-prod::/srv/worktree', + title: 'Dormant', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + } + } + const manifest = createOrcadMigrationManifest( + { + collectOrcadMigrationSourceDormantState: () => dormant, + getRepos: () => [repo('repo-prod', TARGET.id)], + getProjectGroups: () => [], + getFolderWorkspaces: () => [] + }, + TARGET, + { migrationId: 'migration-session-only' } + ) + + expect(manifest.payload.dormantState?.workspaceSession?.tabsByWorktree).toHaveProperty( + 'repo-prod::/srv/worktree' + ) + }) +}) diff --git a/src/main/ssh/orcad-migration-manifest-export.ts b/src/main/ssh/orcad-migration-manifest-export.ts new file mode 100644 index 00000000000..9983be807a6 --- /dev/null +++ b/src/main/ssh/orcad-migration-manifest-export.ts @@ -0,0 +1,117 @@ +/** + * The signed manifest a relay-hosted SSH target's state is exported as. + * + * Reads only: the catalog rows the target owns and the dormant state that references them, copied + * out of the profile-state store. The source keeps every row; retiring it happens only after the + * destination has verified and imported this exact manifest. + */ +import { randomUUID } from 'node:crypto' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + parseOrcadMigrationManifest, + type OrcadMigrationCatalogPayload, + type OrcadMigrationDormantStatePayload, + type OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { SshTarget } from '../../shared/ssh-types' +import type { Store } from '../persistence' +import { collectOrcadMigrationSourceCatalog } from '../persistence/migrating-orcad-catalog/orcad-source-catalog' +import { computeOrcadMigrationManifestSha256 } from '../orcad/orcad-migration-manifest-digest' + +export type OrcadMigrationExportStore = Pick< + Store, + | 'collectOrcadMigrationSourceDormantState' + | 'getFolderWorkspaces' + | 'getProjectGroups' + | 'getRepos' +> + +export function createOrcadMigrationManifest( + store: OrcadMigrationExportStore, + target: SshTarget, + options: { + migrationId?: string + now?: () => Date + destinationEnvironmentId?: string + /** Only these rows: a re-export of what earlier migrations of the host already moved. */ + onlyCatalog?: OrcadMigrationCatalogIds + } = {} +): OrcadMigrationManifest { + const payload = restrictCatalog( + collectOrcadMigrationSourceCatalog(store, target), + options.onlyCatalog + ) + const source = { + sshTargetId: target.id, + sshTargetGeneration: target.generation ?? null, + targetLabel: target.label + } + const dormantState = store.collectOrcadMigrationSourceDormantState( + source, + payload, + options.destinationEnvironmentId + ) + const unsigned = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: options.migrationId ?? randomUUID(), + createdAt: (options.now ?? (() => new Date()))().toISOString(), + source, + payload: { + ...payload, + ...(hasDormantState(dormantState) ? { dormantState } : {}) + }, + ...(options.destinationEnvironmentId + ? { destinationEnvironmentId: options.destinationEnvironmentId } + : {}) + } + return parseOrcadMigrationManifest({ + ...unsigned, + manifestSha256: computeOrcadMigrationManifestSha256(unsigned) + }) +} + +export type OrcadMigrationCatalogIds = { + repositoryIds: ReadonlySet + folderWorkspaceIds: ReadonlySet + projectGroupIds: ReadonlySet +} + +export function orcadMigrationCatalogIds( + payloads: readonly OrcadMigrationCatalogPayload[] +): OrcadMigrationCatalogIds { + return { + repositoryIds: new Set(payloads.flatMap((p) => p.repositories.map((row) => row.id))), + folderWorkspaceIds: new Set(payloads.flatMap((p) => p.folderWorkspaces.map((row) => row.id))), + projectGroupIds: new Set(payloads.flatMap((p) => p.projectGroups.map((row) => row.id))) + } +} + +function restrictCatalog( + payload: OrcadMigrationCatalogPayload, + ids: OrcadMigrationCatalogIds | undefined +): OrcadMigrationCatalogPayload { + if (!ids) { + return payload + } + return { + repositories: payload.repositories.filter((row) => ids.repositoryIds.has(row.id)), + folderWorkspaces: payload.folderWorkspaces.filter((row) => ids.folderWorkspaceIds.has(row.id)), + projectGroups: payload.projectGroups.filter((row) => ids.projectGroupIds.has(row.id)) + } +} + +function hasDormantState(state: OrcadMigrationDormantStatePayload): boolean { + return ( + state.worktreeMeta.length > 0 || + state.worktreeLineage.length > 0 || + state.workspaceLineage.length > 0 || + state.sparsePresets.length > 0 || + state.retiredWorktreeNames.length > 0 || + state.retiredWorktreeNamespaces.length > 0 || + state.workspaceSession !== undefined || + (state.terminalScrollbackSnapshots?.length ?? 0) > 0 || + (state.automations?.length ?? 0) > 0 || + (state.automationRuns?.length ?? 0) > 0 || + (state.clientState !== undefined && Object.keys(state.clientState).length > 0) + ) +} diff --git a/src/main/ssh/orcad-migration-refusal-reason.test.ts b/src/main/ssh/orcad-migration-refusal-reason.test.ts new file mode 100644 index 00000000000..8c0b41c8f9d --- /dev/null +++ b/src/main/ssh/orcad-migration-refusal-reason.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import { orcadMigrationRefusalReason } from './orcad-migration-refusal-reason' + +describe('orcadMigrationRefusalReason', () => { + it('names each blocking kind once, in plain words', () => { + expect( + orcadMigrationRefusalReason([ + { + code: 'orcad_migration_direct_ssh_repositories', + category: 'drainable-static-state', + repositories: [] + }, + { + code: 'orcad_migration_dependent_state', + category: 'client-owned-state', + dependencies: [ + { kind: 'workspace-session', count: 7 }, + { kind: 'worktree-lineage', count: 1 }, + { kind: 'workspace-lineage', count: 1 }, + { kind: 'automation', count: 2 } + ] + }, + { + code: 'orcad_migration_direct_ssh_terminal_leases', + category: 'live-or-unverifiable', + terminalLeases: [] + } + ]) + ).toBe( + 'This SSH host cannot move yet: saved tabs and panes, workspace history, automations, terminals still running.' + ) + }) + + it('keeps the bare refusal when nothing that blocks is named', () => { + expect(orcadMigrationRefusalReason([])).toBe('This SSH host cannot move yet.') + }) +}) diff --git a/src/main/ssh/orcad-migration-refusal-reason.ts b/src/main/ssh/orcad-migration-refusal-reason.ts new file mode 100644 index 00000000000..4f4681c6961 --- /dev/null +++ b/src/main/ssh/orcad-migration-refusal-reason.ts @@ -0,0 +1,50 @@ +import type { + OrcadMigrationBlocker, + OrcadMigrationDependencyKind +} from '../../shared/orcad-migration-preflight' + +const DEPENDENCY_WORDS: Record = { + 'saved-port-forward': 'saved port forwards', + 'terminal-lease': 'terminals still running', + 'terminal-recovery': 'terminals waiting to reconnect', + 'workspace-session': 'saved tabs and panes', + 'worktree-metadata': 'saved workspace details', + 'worktree-lineage': 'workspace history', + 'workspace-lineage': 'workspace history', + 'sparse-preset': 'sparse checkout presets', + 'retired-worktree-name': 'retired workspace names', + automation: 'automations', + 'automation-run': 'automation runs', + 'mobile-tab-selection': 'mobile tab selections', + 'ui-routing': 'sidebar settings' +} + +function blockerWords(blocker: OrcadMigrationBlocker): string[] { + switch (blocker.code) { + case 'orcad_migration_target_not_found': + return ['the host is no longer saved'] + case 'orcad_migration_target_owned': + return ['another server holds it'] + case 'orcad_migration_owner_unrecorded': + return ['its server record is missing'] + case 'orcad_migration_direct_ssh_terminal_leases': + return ['terminals still running'] + case 'orcad_migration_dependent_state': + return blocker.dependencies.map(({ kind }) => DEPENDENCY_WORDS[kind]) + case 'orcad_migration_dependency_unverifiable': + return ['saved state Orca could not read'] + case 'orcad_migration_direct_ssh_repositories': + case 'orcad_migration_direct_ssh_folder_workspaces': + case 'orcad_migration_saved_port_forwards': + // These move with the host or stay behind; they never refuse a conversion. + return [] + } +} + +/** The refusal names what blocks, in plain words; the status line shows only this text. */ +export function orcadMigrationRefusalReason(blockers: readonly OrcadMigrationBlocker[]): string { + const words = [...new Set(blockers.flatMap(blockerWords))] + return words.length > 0 + ? `This SSH host cannot move yet: ${words.join(', ')}.` + : 'This SSH host cannot move yet.' +} diff --git a/src/main/ssh/orcad-migration-relay-pty-lister.test.ts b/src/main/ssh/orcad-migration-relay-pty-lister.test.ts new file mode 100644 index 00000000000..8882b0133ba --- /dev/null +++ b/src/main/ssh/orcad-migration-relay-pty-lister.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('../ipc/pty/provider/registry', () => ({ getSshPtyProvider: vi.fn(() => undefined) })) + +import { toAppSshPtyId } from '../providers/ssh-pty-id' +import { assessOrcadMigrationTerminals } from './orcad-migration-terminal-gate' +import { + ORCAD_MIGRATION_RELAY_LIST_BUDGET_MS, + orcadMigrationRelayPtyLister +} from './orcad-migration-relay-pty-lister' + +const TARGET = 'ssh-win' +const noLeases = { getSshRemotePtyLeases: () => [] } + +describe('the terminal gate asking a relay what it still runs', () => { + it('answers in the relay spelling the leases use, within a bounded deadline', async () => { + const listProcesses = vi.fn(async () => [ + { id: toAppSshPtyId(TARGET, 'pty-7'), cwd: '', title: 'cmd.exe' } + ]) + const lister = orcadMigrationRelayPtyLister(TARGET, { listProcesses }, () => 1_000) + expect(await lister?.()).toEqual(['pty-7']) + expect(listProcesses).toHaveBeenCalledWith({ + deadlineMs: 1_000 + ORCAD_MIGRATION_RELAY_LIST_BUDGET_MS + }) + }) + + it('lets the gate prove exit only when every relay answers with nothing running', async () => { + const running = orcadMigrationRelayPtyLister(TARGET, { + listProcesses: async () => [{ id: toAppSshPtyId(TARGET, 'pty-7'), cwd: '', title: 'pwsh' }] + }) + expect(await assessOrcadMigrationTerminals(noLeases, TARGET, running)).toMatchObject({ + verdict: 'live', + ptyIds: ['pty-7'] + }) + const idle = orcadMigrationRelayPtyLister( + TARGET, + { listProcesses: async () => [] }, + Date.now, + async () => [] + ) + const hostIdle = async () => ({ verdict: 'exited' as const, count: 0 }) + expect(await assessOrcadMigrationTerminals(noLeases, TARGET, idle, hostIdle)).toEqual({ + verdict: 'exited', + provenPtyIds: [] + }) + // Earlier relays that cannot be asked leave it unverifiable, even with nothing leased here. + const unasked = orcadMigrationRelayPtyLister( + TARGET, + { listProcesses: async () => [] }, + Date.now, + async () => null + ) + expect(await assessOrcadMigrationTerminals(noLeases, TARGET, unasked)).toMatchObject({ + verdict: 'unverifiable' + }) + }) + + it('has no lister without a connected relay session', () => { + expect(orcadMigrationRelayPtyLister(TARGET)).toBeNull() + }) + + it("asks earlier-build relays in the leases' spelling, keeping an unknown answer null", async () => { + const provider = { listProcesses: async () => [] } + const held = orcadMigrationRelayPtyLister(TARGET, provider, Date.now, async () => [ + toAppSshPtyId(TARGET, 'pty-old') + ]) + expect(await held?.previous?.()).toEqual(['pty-old']) + const unknown = orcadMigrationRelayPtyLister(TARGET, provider, Date.now, async () => null) + expect(await unknown?.previous?.()).toBeNull() + }) +}) diff --git a/src/main/ssh/orcad-migration-relay-pty-lister.ts b/src/main/ssh/orcad-migration-relay-pty-lister.ts new file mode 100644 index 00000000000..a8f5e9c19eb --- /dev/null +++ b/src/main/ssh/orcad-migration-relay-pty-lister.ts @@ -0,0 +1,38 @@ +/** + * The terminal gate's question to the relay: which PTYs does it still run for this target? + * + * Asked through the SSH PTY provider's `pty.listProcesses`, which the pinned relay answers the + * same way on Windows and POSIX hosts, so a Windows relay-hosted target proves its terminals + * exited exactly as a Linux one does. Ids come back in the relay's own spelling, which is how + * the target's PTY leases name them. + */ +import { getSshPtyProvider } from '../ipc/pty/provider/registry' +import type { IPtyProvider } from '../providers/types' +import { toRelaySshPtyId } from '../providers/ssh-pty-id' +import type { ListRelayPtyIds } from './orcad-migration-terminal-gate' +import { listPreviousRelayPtyIds } from './ssh-legacy-relay-routing' + +/** Long enough for a Windows relay's first process-table read, short enough to block a click. */ +export const ORCAD_MIGRATION_RELAY_LIST_BUDGET_MS = 10_000 + +/** Null when no relay session is connected: the gate then cannot clear expired leases. */ +export function orcadMigrationRelayPtyLister( + targetId: string, + provider: Pick | undefined = getSshPtyProvider(targetId), + now: () => number = Date.now, + listPrevious: (targetId: string) => Promise = listPreviousRelayPtyIds +): ListRelayPtyIds | null { + if (!provider) { + return null + } + const list: ListRelayPtyIds = async () => { + const processes = await provider.listProcesses({ + deadlineMs: now() + ORCAD_MIGRATION_RELAY_LIST_BUDGET_MS + }) + return processes.map((process) => toRelaySshPtyId(targetId, process.id)) + } + // Earlier-build relays answer through their own bridge, POSIX only; null keeps the gate blocked. + list.previous = async () => + (await listPrevious(targetId))?.map((id) => toRelaySshPtyId(targetId, id)) ?? null + return list +} diff --git a/src/main/ssh/orcad-migration-rollback-mark.ts b/src/main/ssh/orcad-migration-rollback-mark.ts new file mode 100644 index 00000000000..443b3ad8a1a --- /dev/null +++ b/src/main/ssh/orcad-migration-rollback-mark.ts @@ -0,0 +1,27 @@ +/** When a managed server last took in migrated state; a rollback to an older snapshot loses it. */ +import type { KnownRuntimeEnvironment } from '../../shared/runtime-environments' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' + +/** + * The latest migration into this server: its durable mark, or a journal's start for a delta + * moved before deltas wrote that mark. An unreadable journal fails closed as "now". + */ +export function latestOrcadMigrationInto( + userDataPath: string, + environment: KnownRuntimeEnvironment +): string | undefined { + let started: string[] + try { + started = listOrcadMigrationSourceCutovers(userDataPath) + .filter((cutover) => cutover.destinationEnvironmentId === environment.id) + .map((cutover) => cutover.startedAt) + } catch { + return new Date().toISOString() + } + return [environment.orcadMigratedAt, ...started] + .filter((at): at is string => at !== undefined) + .reduce( + (latest, at) => (latest === undefined || Date.parse(at) > Date.parse(latest) ? at : latest), + undefined + ) +} diff --git a/src/main/ssh/orcad-migration-scrollback-retention-wiring.test.ts b/src/main/ssh/orcad-migration-scrollback-retention-wiring.test.ts new file mode 100644 index 00000000000..74ca5c06542 --- /dev/null +++ b/src/main/ssh/orcad-migration-scrollback-retention-wiring.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { OrcadMigrationManifest } from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' + +const journal = vi.hoisted(() => { + const state: { + cutovers: Pick[] + listener: ((userDataPath: string) => void) | null + } = { cutovers: [], listener: null } + return state +}) +vi.mock('./orcad-migration-cutover-journal', () => ({ + listOrcadMigrationSourceCutovers: () => journal.cutovers, + setOrcadMigrationJournalChangeListener: (listener: (path: string) => void) => { + journal.listener = listener + } +})) + +const { installOrcadMigrationScrollbackRetention } = + await import('./orcad-migration-scrollback-retention-wiring') + +function manifest(migrationId: string): OrcadMigrationManifest { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the wiring only forwards manifests; the store fake reads the id. + return { migrationId } as OrcadMigrationManifest +} + +afterEach(() => { + journal.cutovers = [] + journal.listener = null +}) + +describe('scrollback retention follows the migration journal', () => { + it('holds from the fence until commit, across journal changes and a restart', () => { + const sync = vi.fn<(pending: readonly OrcadMigrationManifest[]) => void>() + const held = (): string[] => + (sync.mock.calls.at(-1)?.[0] ?? []).map((entry) => entry.migrationId) + journal.cutovers = [{ phase: 'source-fenced', manifest: manifest('m1') }] + + // Startup rebuilds what an unfinished journal holds. + installOrcadMigrationScrollbackRetention('/profile', { + syncOrcadMigrationScrollbackRetention: sync + }) + expect(held()).toEqual(['m1']) + + journal.cutovers = [{ phase: 'destination-staged', manifest: manifest('m1') }] + journal.listener?.('/profile') + expect(held()).toEqual(['m1']) + + journal.cutovers = [{ phase: 'destination-committed', manifest: manifest('m1') }] + journal.listener?.('/profile') + expect(held()).toEqual([]) + }) + + it('keeps what it holds when the journal cannot be read', () => { + const sync = vi.fn() + installOrcadMigrationScrollbackRetention('/profile', { + syncOrcadMigrationScrollbackRetention: sync + }) + journal.cutovers = new Proxy([], { + get: () => { + throw new Error('unreadable') + } + }) + journal.listener?.('/profile') + expect(sync).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/ssh/orcad-migration-scrollback-retention-wiring.ts b/src/main/ssh/orcad-migration-scrollback-retention-wiring.ts new file mode 100644 index 00000000000..afb50ca3d20 --- /dev/null +++ b/src/main/ssh/orcad-migration-scrollback-retention-wiring.ts @@ -0,0 +1,26 @@ +import type { Store } from '../persistence' +import { + listOrcadMigrationSourceCutovers, + setOrcadMigrationJournalChangeListener +} from './orcad-migration-cutover-journal' + +/** Keeps the store's scrollback retention in step with the journal, from startup on. */ +export function installOrcadMigrationScrollbackRetention( + userDataPath: string, + store: Pick +): void { + const sync = (path: string): void => { + try { + // Held until the destination has the bytes; committed or abandoned migrations release them. + const pending = listOrcadMigrationSourceCutovers(path).filter( + (cutover) => cutover.phase === 'source-fenced' || cutover.phase === 'destination-staged' + ) + store.syncOrcadMigrationScrollbackRetention(pending.map((cutover) => cutover.manifest)) + } catch (error) { + // An unreadable journal keeps whatever is held; releasing on a guess could lose bytes. + console.warn('[orcad-migration] Could not sync scrollback retention:', error) + } + } + setOrcadMigrationJournalChangeListener(sync) + sync(userDataPath) +} diff --git a/src/main/ssh/orcad-migration-snapshot-coordinator.test.ts b/src/main/ssh/orcad-migration-snapshot-coordinator.test.ts new file mode 100644 index 00000000000..59aa7365084 --- /dev/null +++ b/src/main/ssh/orcad-migration-snapshot-coordinator.test.ts @@ -0,0 +1,164 @@ +import { createHash } from 'node:crypto' +import { describe, expect, it, vi } from 'vitest' +import { + ORCAD_MIGRATION_MANIFEST_VERSION, + type OrcadMigrationCatalogState, + type OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationTerminalScrollbackSnapshot } from '../../shared/orcad-migration-scrollback' +import { transferOrcadMigrationSnapshots } from './orcad-migration-snapshot-coordinator' + +const BYTES = Buffer.from('resume these bytes', 'utf8') +const SNAPSHOT: OrcadMigrationTerminalScrollbackSnapshot = { + tabId: 'tab-1', + leafId: 'leaf-1', + ref: `v1-${'1'.repeat(32)}`, + sha256: createHash('sha256').update(BYTES).digest('hex'), + byteLength: BYTES.length +} +const MANIFEST: OrcadMigrationManifest = { + version: ORCAD_MIGRATION_MANIFEST_VERSION, + migrationId: 'migration-1', + createdAt: '2026-08-30T12:00:00.000Z', + source: { sshTargetId: 'source', sshTargetGeneration: 1, targetLabel: 'Source' }, + payload: { + repositories: [], + projectGroups: [], + folderWorkspaces: [], + dormantState: { + version: 1, + worktreeMeta: [], + worktreeLineage: [], + workspaceLineage: [], + sparsePresets: [], + retiredWorktreeNames: [], + retiredWorktreeNamespaces: [], + terminalScrollbackSnapshots: [SNAPSHOT] + } + }, + manifestSha256: 'a'.repeat(64) +} +type ChunkReader = ( + manifest: OrcadMigrationManifest, + ref: string, + offset: number +) => { bytesBase64: string; totalBytes: number; eof: boolean } + +function source(read: ChunkReader) { + return { + readOrcadMigrationSourceSnapshotChunk: read + } +} + +const unreachableRead: ChunkReader = () => { + throw new Error('must not read') +} + +describe('orcad migration snapshot coordinator', () => { + it('resumes at the observed offset and reconciles a lost chunk response', async () => { + const initialOffset = 4 + const sourceRead = vi.fn(() => ({ + bytesBase64: BYTES.subarray(initialOffset).toString('base64'), + totalBytes: BYTES.length, + eof: true + })) + const store = source(sourceRead) + const snapshotRequest = vi.fn() + const remoteReads = [staged(BYTES.length), staged(BYTES.length)] + + await transferOrcadMigrationSnapshots({ + source: store, + manifest: MANIFEST, + state: staged(initialOffset), + destination: { + stageChunk: async (request) => { + snapshotRequest(request) + throw new Error('response lost') + }, + readState: async () => nextState(remoteReads) + } + }) + + expect(sourceRead).toHaveBeenCalledWith(MANIFEST, SNAPSHOT.ref, initialOffset) + expect(snapshotRequest).toHaveBeenCalledWith( + expect.objectContaining({ offset: initialOffset, ref: SNAPSHOT.ref }) + ) + expect(remoteReads).toEqual([]) + }) + + it('fails closed when an old host omits snapshot upload state', async () => { + const sourceRead = vi.fn(unreachableRead) + const store = source(sourceRead) + await expect( + transferOrcadMigrationSnapshots({ + source: store, + manifest: MANIFEST, + state: staged(), + destination: { + stageChunk: async () => { + throw new Error('must not upload') + }, + readState: async () => staged() + } + }) + ).rejects.toThrow('orcad_migration_snapshot_transfer_unsupported') + expect(sourceRead).not.toHaveBeenCalled() + }) + + it('refuses a snapshot whose source length changed since export', async () => { + await expect( + transferOrcadMigrationSnapshots({ + source: source(() => ({ + bytesBase64: Buffer.from('changed').toString('base64'), + totalBytes: BYTES.length + 1, + eof: true + })), + manifest: MANIFEST, + state: staged(0), + destination: { + stageChunk: async () => { + throw new Error('must not upload') + }, + readState: async () => staged(0) + } + }) + ).rejects.toThrow('orcad_migration_source_snapshot_changed') + }) + + it('requires complete upload evidence after the final chunk', async () => { + const remoteReads = [staged(BYTES.length - 1)] + await expect( + transferOrcadMigrationSnapshots({ + source: source(unreachableRead), + manifest: MANIFEST, + state: staged(BYTES.length), + destination: { + stageChunk: async () => { + throw new Error('must not upload') + }, + readState: async () => nextState(remoteReads) + } + }) + ).rejects.toThrow('orcad_migration_snapshot_transfer_incomplete') + }) +}) + +function staged(receivedBytes?: number): Extract { + return { + state: 'staged', + migrationId: MANIFEST.migrationId, + manifestSha256: MANIFEST.manifestSha256, + stagedAt: '2026-08-30T12:01:00.000Z', + ...(receivedBytes === undefined ? {} : { snapshotUploads: [{ ...SNAPSHOT, receivedBytes }] }) + } +} + +function nextState( + states: Extract[] +): Extract { + const state = states.shift() + if (!state) { + throw new Error('unexpected remote read') + } + return state +} diff --git a/src/main/ssh/orcad-migration-snapshot-coordinator.ts b/src/main/ssh/orcad-migration-snapshot-coordinator.ts new file mode 100644 index 00000000000..d15a80f1b73 --- /dev/null +++ b/src/main/ssh/orcad-migration-snapshot-coordinator.ts @@ -0,0 +1,114 @@ +/** + * Sending a manifest's scrollback snapshots to the destination in bounded, resumable chunks. + * + * The source side only reads: each chunk comes from the profile-state store, checked against the + * signed manifest's length and digest. The destination's catalog operations are passed in, so + * this driver owns no transport and retires nothing. A chunk whose acknowledgement was lost is + * confirmed by reading the destination's recorded offset, never assumed. + */ +import type { + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import { + decodeOrcadMigrationSnapshotChunk, + type OrcadMigrationSnapshotChunkRequest, + type OrcadMigrationSnapshotChunkResult +} from '../../shared/orcad-migration-scrollback' +import type { Store } from '../persistence' + +export type OrcadMigrationSnapshotSource = Pick + +export type OrcadMigrationSnapshotDestination = { + readState: (manifest: OrcadMigrationManifest) => Promise + stageChunk: ( + request: OrcadMigrationSnapshotChunkRequest + ) => Promise +} + +export async function transferOrcadMigrationSnapshots(args: { + source: OrcadMigrationSnapshotSource + manifest: OrcadMigrationManifest + /** The destination's staged state, whose upload offsets say where each snapshot resumes. */ + state: Extract + destination: OrcadMigrationSnapshotDestination +}): Promise { + const snapshots = args.manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + if (snapshots.length === 0) { + return + } + // The journal holds these bytes until commit or abort, so a closed tab still sends on a retry. + await sendSnapshots(args, snapshots) +} + +async function sendSnapshots( + args: Parameters[0], + snapshots: NonNullable< + NonNullable['terminalScrollbackSnapshots'] + > +): Promise { + const offsets = new Map( + (args.state.snapshotUploads ?? []).map((entry) => [entry.ref, entry.receivedBytes]) + ) + for (const snapshot of snapshots) { + let offset = offsets.get(snapshot.ref) + if (offset === undefined) { + throw new Error('orcad_migration_snapshot_transfer_unsupported') + } + while (offset < snapshot.byteLength) { + const chunk = args.source.readOrcadMigrationSourceSnapshotChunk( + args.manifest, + snapshot.ref, + offset + ) + if (chunk.totalBytes !== snapshot.byteLength || !chunk.bytesBase64) { + throw new Error('orcad_migration_source_snapshot_changed') + } + const request: OrcadMigrationSnapshotChunkRequest = { + migrationId: args.manifest.migrationId, + manifestSha256: args.manifest.manifestSha256, + ref: snapshot.ref, + offset, + bytesBase64: chunk.bytesBase64 + } + const expectedOffset = offset + decodeOrcadMigrationSnapshotChunk(chunk.bytesBase64).length + offset = await stageChunkWithRecovery(args, request, expectedOffset) + } + } + const verified = await args.destination.readState(args.manifest) + if ( + verified.state !== 'staged' || + (verified.snapshotUploads ?? []).length !== snapshots.length || + (verified.snapshotUploads ?? []).some((entry) => entry.receivedBytes !== entry.byteLength) + ) { + throw new Error('orcad_migration_snapshot_transfer_incomplete') + } +} + +async function stageChunkWithRecovery( + args: { manifest: OrcadMigrationManifest; destination: OrcadMigrationSnapshotDestination }, + request: OrcadMigrationSnapshotChunkRequest, + expectedOffset: number +): Promise { + try { + const result = await args.destination.stageChunk(request) + if (result.acknowledgedOffset !== expectedOffset) { + throw new Error('orcad_migration_snapshot_ack_invalid') + } + return result.acknowledgedOffset + } catch (error) { + try { + const observed = await args.destination.readState(args.manifest) + const received = + observed.state === 'staged' + ? observed.snapshotUploads?.find((entry) => entry.ref === request.ref)?.receivedBytes + : undefined + if (received === expectedOffset) { + return received + } + } catch { + // The chunk stays unverifiable; report the first failure. + } + throw error + } +} diff --git a/src/main/ssh/orcad-migration-snapshot-resume.test.ts b/src/main/ssh/orcad-migration-snapshot-resume.test.ts new file mode 100644 index 00000000000..0be0d7b701e --- /dev/null +++ b/src/main/ssh/orcad-migration-snapshot-resume.test.ts @@ -0,0 +1,161 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { getDefaultWorkspaceSession } from '../../shared/constants' +import { toSshExecutionHostId } from '../../shared/execution-host' +import type { + OrcadMigrationCatalogState, + OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationSnapshotChunkRequest } from '../../shared/orcad-migration-scrollback' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { createOrcadMigrationManifest } from './orcad-migration-manifest-export' +import { transferOrcadMigrationSnapshots } from './orcad-migration-snapshot-coordinator' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 1 +} +const WORKTREE_ID = 'repo-1::/srv/app' +const OUTPUT = 'dormant output\r\n' + +const directories: string[] = [] +afterEach(async () => { + await closeTestStores() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function openStore(dataFile: string): Store { + return createSqliteTestStore(Store, { dataFile }) +} + +/** A relay host whose dormant tab keeps its scrollback inline, as `ssh:` partitions do. */ +function relayProfile(): { store: Store; dataFile: string } { + const directory = mkdtempSync(join(tmpdir(), 'orcad-snapshot-resume-')) + directories.push(directory) + const dataFile = join(directory, 'orca-data.json') + const store = openStore(dataFile) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + store.setWorkspaceSession( + { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [WORKTREE_ID]: [ + { + id: 'tab-1', + ptyId: null, + worktreeId: WORKTREE_ID, + title: 'Shell', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: 'leaf-1' }, + activeLeafId: 'leaf-1', + expandedLeafId: null, + buffersByLeafId: { 'leaf-1': OUTPUT } + } + } + }, + toSshExecutionHostId(TARGET.id) + ) + return { store, dataFile } +} + +/** A staged destination that loses the connection on its first chunk. */ +function destination(manifest: OrcadMigrationManifest, failFirst: boolean) { + const [snapshot] = manifest.payload.dormantState?.terminalScrollbackSnapshots ?? [] + let received = 0 + let fail = failFirst + const state = (): Extract => ({ + state: 'staged', + migrationId: manifest.migrationId, + manifestSha256: manifest.manifestSha256, + stagedAt: '2026-10-05T00:00:00.000Z', + snapshotUploads: [{ ...snapshot!, receivedBytes: received }] + }) + return { + readState: async () => state(), + stageChunk: async (request: OrcadMigrationSnapshotChunkRequest) => { + if (fail) { + fail = false + throw new Error('transport lost') + } + received = request.offset + Buffer.from(request.bytesBase64, 'base64').length + return { + migrationId: request.migrationId, + manifestSha256: request.manifestSha256, + ref: request.ref, + acknowledgedOffset: received + } + }, + received: () => received + } +} + +async function transfer( + store: Store, + manifest: OrcadMigrationManifest, + remote: ReturnType +): Promise { + await transferOrcadMigrationSnapshots({ + source: store, + manifest, + state: await remote.readState(), + destination: remote + }) +} + +describe('resuming a scrollback upload the journal still holds', () => { + it('retries after the tab closed and the upload was interrupted', async () => { + const { store } = relayProfile() + const manifest = createOrcadMigrationManifest(store, TARGET) + store.syncOrcadMigrationScrollbackRetention([manifest]) + store.setWorkspaceSession(getDefaultWorkspaceSession(), toSshExecutionHostId(TARGET.id)) + const remote = destination(manifest, true) + + await expect(transfer(store, manifest, remote)).rejects.toThrow('transport lost') + await transfer(store, manifest, remote) + + expect(remote.received()).toBe(Buffer.byteLength(OUTPUT)) + }) + + it('recovers after a restart, from the journal alone', async () => { + const { store, dataFile } = relayProfile() + const manifest = createOrcadMigrationManifest(store, TARGET) + store.syncOrcadMigrationScrollbackRetention([manifest]) + store.setWorkspaceSession(getDefaultWorkspaceSession(), toSshExecutionHostId(TARGET.id)) + await store.flushPendingOrThrowAsync() + await closeTestStores() + + const restarted = openStore(dataFile) + restarted.syncOrcadMigrationScrollbackRetention([manifest]) + const remote = destination(manifest, false) + await transfer(restarted, manifest, remote) + + expect(remote.received()).toBe(Buffer.byteLength(OUTPUT)) + }) +}) diff --git a/src/main/ssh/orcad-migration-source-assertions.ts b/src/main/ssh/orcad-migration-source-assertions.ts new file mode 100644 index 00000000000..384a348eaf7 --- /dev/null +++ b/src/main/ssh/orcad-migration-source-assertions.ts @@ -0,0 +1,64 @@ +/** Checks run before every stage and commit: the fenced source is still what the journal says. */ +import { createHash } from 'node:crypto' +import { + serializeOrcadMigrationValue, + type OrcadMigrationManifest +} from '../../shared/orcad-migration-manifest' +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import { createOrcadMigrationManifest } from './orcad-migration-manifest-export' +import { resolveOrcadMigrationFence } from './orcad-migration-source-fence' +import { confirmOrcadMigrationTerminalsUnderFence } from './orcad-migration-terminal-gate' +import { collectUntransferredDependentBlockers } from './ssh-target-orcad-dependents' +import type { OrcadMigrationPreflightStore } from './ssh-target-orcad-preflight' + +export function assertOrcadMigrationSourceUnchanged( + context: { userDataPath: string; store: OrcadMigrationPreflightStore }, + cutover: OrcadMigrationSourceCutover +): void { + const target = context.store.getSshTarget(cutover.sshTargetId) + const fence = target ? resolveOrcadMigrationFence(context.userDataPath, target) : null + if (!target || fence?.state !== 'fenced' || fence.cutover.migrationId !== cutover.migrationId) { + throw new Error('orcad_migration_source_fence_lost') + } + // Same id, time and destination as the journaled export: any difference is a source change. + const current = createOrcadMigrationManifest(context.store, target, { + migrationId: cutover.migrationId, + destinationEnvironmentId: cutover.destinationEnvironmentId, + now: () => new Date(cutover.manifest.createdAt) + }) + if (frozenSourceDigest(current) !== frozenSourceDigest(cutover.manifest)) { + throw new Error('orcad_migration_source_changed') + } + if (collectUntransferredDependentBlockers(context.store, cutover.manifest).length > 0) { + throw new Error('orcad_migration_source_dependencies_present') + } + const terminals = confirmOrcadMigrationTerminalsUnderFence(context.store, target.id, { + verdict: 'exited', + provenPtyIds: cutover.provenPtyIds + }) + if (terminals.verdict !== 'exited') { + throw new Error(`orcad_migration_source_terminals_${terminals.verdict}`) + } +} + +/** + * The manifest minus what the UI rewrites as the user works (tabs, focus, routing): the server + * gets those as journaled, and a tab reorder mid-conversion must not fail the move. + */ +function frozenSourceDigest(manifest: OrcadMigrationManifest): string { + const { manifestSha256: _digest, payload, ...rest } = manifest + const { + version: _version, + workspaceSession: _session, + clientState: _client, + // Retained for the transfer, so a tab closed mid-move still sends its journaled bytes. + terminalScrollbackSnapshots: _snapshots, + ...dormant + } = payload.dormantState ?? {} + // Export omits an empty dormant payload, so UI state alone can make one appear. + const empty = Object.values(dormant).every( + (value) => value === undefined || (Array.isArray(value) && value.length === 0) + ) + const frozen = { ...rest, payload: { ...payload, dormantState: empty ? undefined : dormant } } + return createHash('sha256').update(serializeOrcadMigrationValue(frozen)).digest('hex') +} diff --git a/src/main/ssh/orcad-migration-source-fence.test.ts b/src/main/ssh/orcad-migration-source-fence.test.ts new file mode 100644 index 00000000000..62a162cff4e --- /dev/null +++ b/src/main/ssh/orcad-migration-source-fence.test.ts @@ -0,0 +1,221 @@ +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { + listOrcadMigrationSourceCutovers, + orcadMigrationCutoverJournalDirectory +} from './orcad-migration-cutover-journal' +import { + fenceOrcadMigrationSource, + resolveOrcadMigrationFence +} from './orcad-migration-source-fence' +import type { OrcadMigrationTerminalVerdict } from './orcad-migration-terminal-gate' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} +const exited: OrcadMigrationTerminalVerdict = { verdict: 'exited', provenPtyIds: [] } + +const directories: string[] = [] +afterEach(async () => { + await closeTestStores() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function setup() { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-migration-fence-')) + directories.push(userDataPath) + const dataFile = join(userDataPath, 'orca-data.json') + const store = createSqliteTestStore(Store, { dataFile }) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + const claims = new SshTargetOrcadClaims(store) + const fence = (overrides: Partial[0]> = {}) => + fenceOrcadMigrationSource({ + userDataPath, + store, + claims, + targetId: TARGET.id, + destinationEnvironmentId: 'env-1', + destinationName: 'Managed', + terminalProof: exited, + hasDirectSshAuthority: () => false, + ...overrides + }) + const target = () => store.getSshTarget(TARGET.id)! + return { userDataPath, dataFile, store, claims, fence, target } +} + +describe('migration source fence', () => { + it('journals, then fences, then flushes, all before returning', async () => { + const harness = setup() + const order: string[] = [] + const journalWrite = vi + .spyOn(harness.claims, 'fenceForMigration') + .mockImplementation((...args) => { + order.push( + listOrcadMigrationSourceCutovers(harness.userDataPath).length ? 'journal' : 'none' + ) + order.push('fence') + return SshTargetOrcadClaims.prototype.fenceForMigration.apply(harness.claims, args) + }) + const flush = vi.spyOn(harness.claims, 'flush').mockImplementation(async () => { + order.push(harness.target().orcadFence ? 'flush-after-fence' : 'flush-before-fence') + }) + const result = await harness.fence() + expect(result).toMatchObject({ outcome: 'fenced', resumed: false }) + expect(order).toEqual(['journal', 'fence', 'flush-after-fence']) + expect(getManagedOrcadFenceEnvironmentId(harness.target())).toBe('env-1') + journalWrite.mockRestore() + flush.mockRestore() + }) + + it('keeps the journal out of the profile, where an older build would strip it', async () => { + const harness = setup() + const result = await harness.fence() + if (result.outcome !== 'fenced') { + throw new Error('expected a fence') + } + await harness.store.flushPendingOrThrowAsync() + const journalDirectory = orcadMigrationCutoverJournalDirectory(harness.userDataPath) + const profileFiles = readdirSync(harness.userDataPath, { recursive: true, encoding: 'utf8' }) + .map((name) => join(harness.userDataPath, name)) + .filter((path) => !path.startsWith(journalDirectory) && statSync(path).isFile()) + expect(profileFiles.length).toBeGreaterThan(0) + for (const path of profileFiles) { + expect(readFileSync(path).includes(result.cutover.migrationId)).toBe(false) + } + const [onDisk] = listOrcadMigrationSourceCutovers(harness.userDataPath) + expect(onDisk).toMatchObject({ phase: 'source-fenced', sshTargetGeneration: 2 }) + expect(onDisk?.manifest.payload.repositories.map((repo) => repo.id)).toEqual(['repo-1']) + }) + + it('resumes the same migration instead of starting a second one', async () => { + const harness = setup() + const first = await harness.fence() + const second = await harness.fence({ + terminalProof: { verdict: 'live', ptyIds: ['p'], reason: 'x' } + }) + expect(second).toMatchObject({ outcome: 'fenced', resumed: true }) + expect( + first.outcome === 'fenced' && second.outcome === 'fenced' && second.cutover.migrationId + ).toBe(first.outcome === 'fenced' && first.cutover.migrationId) + expect(listOrcadMigrationSourceCutovers(harness.userDataPath)).toHaveLength(1) + }) + + it('treats a fence whose journal was lost as unverifiable and never releases it', async () => { + const harness = setup() + await harness.fence() + rmSync(orcadMigrationCutoverJournalDirectory(harness.userDataPath), { recursive: true }) + expect(resolveOrcadMigrationFence(harness.userDataPath, harness.target())).toEqual({ + state: 'fenced-unverifiable', + environmentId: 'env-1' + }) + await expect(harness.fence()).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_migration_fenced_unverifiable' + }) + expect(getManagedOrcadFenceEnvironmentId(harness.target())).toBe('env-1') + }) + + it('treats a journal whose fence an older build removed as stale, granting nothing', async () => { + const harness = setup() + await harness.fence() + harness.store.updateSshTarget(TARGET.id, { orcadFence: undefined }) + expect(resolveOrcadMigrationFence(harness.userDataPath, harness.target()).state).toBe( + 'stale-journal' + ) + await expect(harness.fence()).resolves.toMatchObject({ code: 'orcad_migration_stale_journal' }) + }) + + it('fails closed on an unreadable journal', async () => { + const harness = setup() + await harness.fence() + const [cutover] = listOrcadMigrationSourceCutovers(harness.userDataPath) + writeFileSync( + join( + orcadMigrationCutoverJournalDirectory(harness.userDataPath), + `${cutover!.migrationId}.json` + ), + '{not json' + ) + expect(() => resolveOrcadMigrationFence(harness.userDataPath, harness.target())).toThrow( + 'stays fenced' + ) + await expect(harness.fence()).rejects.toThrow('stays fenced') + }) + + it.each<[OrcadMigrationTerminalVerdict, string]>([ + [{ verdict: 'live', ptyIds: ['p'], reason: 'terminals run' }, 'live'], + [{ verdict: 'unverifiable', ptyIds: ['p'], reason: 'relay silent' }, 'unverifiable'] + ])('refuses before fencing when terminals are %j', async (terminalProof, verdict) => { + const harness = setup() + await expect(harness.fence({ terminalProof })).resolves.toMatchObject({ + outcome: 'refused', + verdict + }) + expect(harness.target().orcadFence).toBeUndefined() + expect(listOrcadMigrationSourceCutovers(harness.userDataPath)).toEqual([]) + }) + + it('refuses while the host is still connected directly', async () => { + const harness = setup() + await expect(harness.fence({ hasDirectSshAuthority: () => true })).resolves.toMatchObject({ + code: 'orcad_migration_direct_ssh_connected' + }) + expect(harness.target().orcadFence).toBeUndefined() + }) + + it('keeps a blocker it could not read unverifiable, never live', async () => { + const harness = setup() + vi.spyOn(harness.store, 'inspectOrcadMigrationUntransferredDependencies').mockImplementation( + () => { + throw new Error('census failed') + } + ) + await expect(harness.fence()).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_migration_preflight_blocked' + }) + }) + + it('releases its own fence when a terminal appeared before the fence took hold', async () => { + const harness = setup() + const flush = vi.spyOn(harness.claims, 'flush').mockImplementation(async () => { + if (harness.target().orcadFence) { + harness.store.upsertSshRemotePtyLease({ + targetId: TARGET.id, + ptyId: 'late-pty', + state: 'detached' + }) + } + }) + await expect(harness.fence()).resolves.toMatchObject({ outcome: 'refused', verdict: 'live' }) + expect(harness.target().orcadFence).toBeUndefined() + expect(listOrcadMigrationSourceCutovers(harness.userDataPath)).toEqual([]) + flush.mockRestore() + }) +}) diff --git a/src/main/ssh/orcad-migration-source-fence.ts b/src/main/ssh/orcad-migration-source-fence.ts new file mode 100644 index 00000000000..4500d3dde69 --- /dev/null +++ b/src/main/ssh/orcad-migration-source-fence.ts @@ -0,0 +1,239 @@ +/** + * Fencing a relay-hosted SSH target for a dormant migration into a managed orcad. + * + * Two durable records describe a fence: the journal sidecar and the target's `orcadFence`. + * Write order is journal, then fence, then the profile flush, all before any remote call, so a + * crash leaves either nothing, a journal without a fence (stale, never authority), or both. + * A fence without a journal is never released here: only the destination can say what happened. + */ +import { randomUUID } from 'node:crypto' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import type { OrcadMigrationBlocker } from '../../shared/orcad-migration-preflight' +import { + ORCAD_MIGRATION_SOURCE_CUTOVER_VERSION, + type OrcadMigrationSourceCutover +} from '../../shared/orcad-migration-source-cutover' +import type { SshTarget } from '../../shared/ssh-types' +import { createOrcadMigrationManifest } from './orcad-migration-manifest-export' +import { + findOrcadMigrationSourceCutoverForTarget, + removeOrcadMigrationSourceCutover, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' +import { + confirmOrcadMigrationTerminalsUnderFence, + type OrcadMigrationTerminalVerdict +} from './orcad-migration-terminal-gate' +import type { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { + isBlockingOrcadMigrationBlocker, + preflightOrcadMigrationExport, + type OrcadMigrationPreflightStore +} from './ssh-target-orcad-preflight' +import { orcadMigrationRefusalReason } from './orcad-migration-refusal-reason' + +export type OrcadMigrationFenceState = + | { state: 'none' } + /** Owner and journal agree: the migration can resume. */ + | { state: 'fenced'; cutover: OrcadMigrationSourceCutover } + /** Owned by an empty-host deploy, which records itself as a provisioning intent instead. */ + | { state: 'owned-by-deploy'; environmentId: string } + /** Owner with no journal (a downgrade dropped it, or it never landed): recover, never release. */ + | { state: 'fenced-unverifiable'; environmentId: string } + /** Journal whose fence is gone or changed: it can never authorize staging. */ + | { state: 'stale-journal'; cutover: OrcadMigrationSourceCutover } + +/** Throws when the journal cannot be read: an unreadable record keeps the host fenced. */ +export function resolveOrcadMigrationFence( + userDataPath: string, + target: SshTarget +): OrcadMigrationFenceState { + const cutover = findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) + const ownerEnvironmentId = getManagedOrcadFenceEnvironmentId(target) + if (cutover) { + return ownerEnvironmentId === cutover.destinationEnvironmentId && + target.generation === cutover.sshTargetGeneration + ? { state: 'fenced', cutover } + : { state: 'stale-journal', cutover } + } + if (!ownerEnvironmentId) { + return { state: 'none' } + } + return target.orcadProvisioning + ? { state: 'owned-by-deploy', environmentId: ownerEnvironmentId } + : { state: 'fenced-unverifiable', environmentId: ownerEnvironmentId } +} + +export type OrcadMigrationFenceResult = + | { outcome: 'fenced'; cutover: OrcadMigrationSourceCutover; resumed: boolean } + | { + outcome: 'refused' + verdict: 'live' | 'unverifiable' + code: string + reason: string + blockers?: OrcadMigrationBlocker[] + } + +export async function fenceOrcadMigrationSource(args: { + userDataPath: string + store: OrcadMigrationPreflightStore + claims: SshTargetOrcadClaims + targetId: string + destinationEnvironmentId: string + destinationName: string + /** Taken by the caller while the relay could still answer; see orcad-migration-terminal-gate. */ + terminalProof: OrcadMigrationTerminalVerdict + hasDirectSshAuthority: (targetId: string) => boolean + now?: () => Date + signal?: AbortSignal +}): Promise { + const target = args.store.getSshTarget(args.targetId) + if (!target) { + return refuse('unverifiable', 'orcad_migration_target_not_found', 'The SSH host is gone.') + } + const existing = resolveOrcadMigrationFence(args.userDataPath, target) + if (existing.state === 'fenced') { + return existing.cutover.destinationEnvironmentId === args.destinationEnvironmentId + ? { outcome: 'fenced', cutover: existing.cutover, resumed: true } + : refuse('live', 'orcad_migration_in_progress', 'Another migration holds this SSH host.') + } + if (existing.state !== 'none') { + return refuse( + 'unverifiable', + `orcad_migration_${existing.state.replaceAll('-', '_')}`, + fenceStateReason(existing) + ) + } + if (args.terminalProof.verdict !== 'exited') { + return refuse( + args.terminalProof.verdict, + 'orcad_migration_terminals', + args.terminalProof.reason + ) + } + if (args.hasDirectSshAuthority(target.id)) { + return refuse('live', 'orcad_migration_direct_ssh_connected', 'Disconnect this SSH host first.') + } + const preflight = preflightOrcadMigrationExport(args.store, target.id) + if (!preflight.claimable) { + return { + ...refuse( + // Only a proven blocker reads live; one that is merely unanswered stays unverifiable. + preflight.blockers + .filter(isBlockingOrcadMigrationBlocker) + .every((blocker) => blocker.category === 'live-or-unverifiable') + ? 'unverifiable' + : 'live', + 'orcad_migration_preflight_blocked', + orcadMigrationRefusalReason(preflight.blockers) + ), + blockers: preflight.blockers + } + } + const generation = args.claims.ensureGeneration(target.id) + const now = (args.now ?? (() => new Date()))().toISOString() + const manifest = createOrcadMigrationManifest( + args.store, + { ...target, generation }, + { migrationId: randomUUID(), destinationEnvironmentId: args.destinationEnvironmentId } + ) + const cutover: OrcadMigrationSourceCutover = { + version: ORCAD_MIGRATION_SOURCE_CUTOVER_VERSION, + migrationId: manifest.migrationId, + phase: 'source-fenced', + startedAt: now, + updatedAt: now, + destinationEnvironmentId: args.destinationEnvironmentId, + destinationName: args.destinationName, + sshTargetId: target.id, + sshTargetGeneration: generation, + manifestSha256: manifest.manifestSha256, + provenPtyIds: args.terminalProof.provenPtyIds, + // Taken with the export, before any commit is possible: the only proof of what the server holds. + manifest + } + writeOrcadMigrationSourceCutover(args.userDataPath, cutover) + args.claims.fenceForMigration(target.id, args.destinationEnvironmentId, generation) + await args.claims.flush(args.signal) + // Why again: a terminal may have started between the proof and the fence. + const underFence = confirmOrcadMigrationTerminalsUnderFence( + args.store, + target.id, + args.terminalProof + ) + if (underFence.verdict !== 'exited') { + await releaseUnstagedFence(args, cutover) + return refuse(underFence.verdict, 'orcad_migration_terminals', underFence.reason) + } + return { outcome: 'fenced', cutover, resumed: false } +} + +type FenceReleaseArgs = { userDataPath: string; claims: SshTargetOrcadClaims; signal?: AbortSignal } + +/** Undoes a fence nothing remote has seen. */ +export async function releaseUnstagedFence( + args: FenceReleaseArgs, + cutover: OrcadMigrationSourceCutover +): Promise { + if (cutover.phase !== 'source-fenced') { + throw new Error('orcad_migration_fence_release_after_stage') + } + await releaseOrcadMigrationFence(args, cutover) +} + +/** + * Owner first, then the journal: a crash between them leaves a stale journal, which grants + * nothing, rather than an owner nothing explains. + */ +export async function releaseOrcadMigrationFence( + args: FenceReleaseArgs, + cutover: OrcadMigrationSourceCutover +): Promise { + args.claims.release(cutover.sshTargetId, cutover.destinationEnvironmentId) + await args.claims.flush(args.signal) + removeOrcadMigrationSourceCutover(args.userDataPath, cutover.migrationId) +} + +/** + * Releases a migration fence whose destination server was never registered: no deploy finished, + * so nothing could have been staged there. A registered destination must answer an abort instead. + */ +export async function releaseUndeployedMigrationFence(args: { + userDataPath: string + claims: SshTargetOrcadClaims + targetId: string + isDestinationRegistered: (environmentId: string) => boolean + signal?: AbortSignal +}): Promise<'released' | 'none'> { + const cutover = findOrcadMigrationSourceCutoverForTarget(args.userDataPath, args.targetId) + if (!cutover) { + return 'none' + } + if (args.isDestinationRegistered(cutover.destinationEnvironmentId)) { + throw new Error('orcad_migration_destination_registered') + } + await releaseUnstagedFence(args, cutover) + return 'released' +} + +function fenceStateReason(state: OrcadMigrationFenceState): string { + switch (state.state) { + case 'owned-by-deploy': + return 'This SSH host already belongs to a managed server.' + case 'fenced-unverifiable': + return 'This SSH host is held for a migration whose record is missing. Recover it from the managed server first.' + case 'stale-journal': + return 'A migration record names this SSH host but its fence is gone. Resolve that migration first.' + case 'fenced': + case 'none': + return 'This SSH host is not in a state a migration can start from.' + } +} + +function refuse( + verdict: 'live' | 'unverifiable', + code: string, + reason: string +): Extract { + return { outcome: 'refused', verdict, code, reason } +} diff --git a/src/main/ssh/orcad-migration-source-retention.test.ts b/src/main/ssh/orcad-migration-source-retention.test.ts new file mode 100644 index 00000000000..e520123fe4c --- /dev/null +++ b/src/main/ssh/orcad-migration-source-retention.test.ts @@ -0,0 +1,57 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + listOrcadMigrationSourceCutovers, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' +import { orcadMigrationCutoverFixture } from './orcad-migration-cutover-fixture' +import { listPendingManagedOrcadMigrations } from './orcad-managed-migration-status' +import { retainOrcadMigrationSource } from './orcad-migration-source-retention' + +let userDataPath: string + +beforeEach(() => { + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-retention-')) +}) + +afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) +}) + +describe('retaining a committed migration source', () => { + it('marks only a committed cutover, idempotently, and stops counting it as pending', () => { + writeOrcadMigrationSourceCutover(userDataPath, orcadMigrationCutoverFixture('m-1')) + expect(() => retainOrcadMigrationSource(userDataPath, 'm-1')).toThrow( + 'orcad_migration_retain_before_commit' + ) + const committed = { + ...orcadMigrationCutoverFixture('m-1'), + phase: 'destination-committed' as const + } + writeOrcadMigrationSourceCutover(userDataPath, committed) + const at = () => new Date('2026-10-03T00:00:00.000Z') + retainOrcadMigrationSource(userDataPath, 'm-1', at) + retainOrcadMigrationSource(userDataPath, 'm-1', () => new Date('2026-12-01T00:00:00.000Z')) + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]?.sourceRetainedAt).toBe( + '2026-10-03T00:00:00.000Z' + ) + expect(listPendingManagedOrcadMigrations(userDataPath)).toEqual([]) + }) + + it('does not count retained cutovers against the in-flight journal capacity', () => { + for (let index = 0; index < 4; index += 1) { + const id = `m-${index}` + writeOrcadMigrationSourceCutover(userDataPath, { + ...orcadMigrationCutoverFixture(id, `ssh-${index}`), + phase: 'destination-committed', + sourceRetainedAt: '2026-10-03T00:00:00.000Z' + }) + } + expect(() => + writeOrcadMigrationSourceCutover(userDataPath, orcadMigrationCutoverFixture('m-new', 'ssh-9')) + ).not.toThrow() + }) +}) diff --git a/src/main/ssh/orcad-migration-source-retention.ts b/src/main/ssh/orcad-migration-source-retention.ts new file mode 100644 index 00000000000..51ca26ad49e --- /dev/null +++ b/src/main/ssh/orcad-migration-source-retention.ts @@ -0,0 +1,32 @@ +/** + * After commit: a migration keeps its source rows, so a downgraded build still sees the host and + * its projects and can reach them over its own relay. New builds hide those rows and serve the host + * from the server. They are never deleted automatically. + */ +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import { + listOrcadMigrationSourceCutovers, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' + +/** Marks a committed migration as finished for this build while its source rows stay. */ +export function retainOrcadMigrationSource( + userDataPath: string, + migrationId: string, + now: () => Date = () => new Date() +): OrcadMigrationSourceCutover { + const cutover = listOrcadMigrationSourceCutovers(userDataPath).find( + (entry) => entry.migrationId === migrationId + ) + if (!cutover || cutover.phase !== 'destination-committed') { + throw new Error('orcad_migration_retain_before_commit') + } + if (cutover.sourceRetainedAt) { + return cutover + } + // Why no baseline here: the source may have changed since the commit (a crash, then an older + // build); only the fence's pre-commit baseline proves what the server holds. + const retained = { ...cutover, sourceRetainedAt: now().toISOString() } + writeOrcadMigrationSourceCutover(userDataPath, retained) + return retained +} diff --git a/src/main/ssh/orcad-migration-terminal-gate-host-wide.test.ts b/src/main/ssh/orcad-migration-terminal-gate-host-wide.test.ts new file mode 100644 index 00000000000..31d03e1f150 --- /dev/null +++ b/src/main/ssh/orcad-migration-terminal-gate-host-wide.test.ts @@ -0,0 +1,159 @@ +// Astra pass 4 §2: with this desktop's relay connected and idle, its own and its earlier relays' +// lists name only this target's instances. Desktop B's live shell, under another target id on the +// same account, must still keep the host from converting. +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshConnection } from './ssh-connection' +import { relaySocketNameForInstanceId } from './ssh-relay-instance-id' +import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints' +import { getRemoteHostPlatform, joinRemotePath } from './ssh-remote-platform' + +const { + execCommand, + probeRelayEndpointIncumbent, + countRelayEndpointPtys, + countRelayPtysOverBridge +} = vi.hoisted(() => ({ + execCommand: vi.fn(), + probeRelayEndpointIncumbent: vi.fn(), + countRelayEndpointPtys: vi.fn(), + countRelayPtysOverBridge: vi.fn() +})) + +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand })) +vi.mock('./ssh-relay-endpoint-incumbent', async (importOriginal) => ({ + ...(await importOriginal>()), + probeRelayEndpointIncumbent +})) +vi.mock('./ssh-relay-endpoint-pty-count', () => ({ + countRelayEndpointPtys, + countRelayPtysOverBridge +})) +vi.mock('./ssh-relay-endpoint-runtime', () => ({ readRelayDaemonRuntimes: async () => new Map() })) +vi.mock('./ssh-remote-commands', async (importOriginal) => ({ + ...(await importOriginal>()), + listRemoteInstallBaseDirsCommand: () => 'LIST' +})) +vi.mock('./ssh-host-relay-windows-inventory', async (importOriginal) => ({ + ...(await importOriginal>()), + windowsRelayInventoryCommand: () => 'INVENTORY', + windowsPipeAccessCommand: () => 'ACCESS' +})) +vi.mock('./ssh-relay-windows-launch-command', () => ({ + windowsRelayConnectCommand: (_h: unknown, _n: string, _d: string, pipe: string) => + `BRIDGE ${pipe}` +})) + +import { + assessOrcadMigrationTerminals, + type ListRelayPtyIds +} from './orcad-migration-terminal-gate' +import { censusHostRelayEndpoints } from './ssh-host-relay-endpoint-census' +import { censusWindowsHostRelays } from './ssh-host-relay-windows-census' +import { hostTerminalProofFromCensus } from './ssh-host-relay-terminals-on-connect' + +// The censuses only hand the connection to the mocked exec, probe and bridge. +const conn: SshConnection = Object.create(null) +const noLeases = { getSshRemotePtyLeases: () => [] } +// Desktop A is connected: its relay and its earlier relays both answer that nothing runs. +const desktopAConnectedIdle = (): ListRelayPtyIds => + Object.assign(async () => [], { previous: async () => [] }) + +describe('a connected desktop converting while another desktop runs a shell on the account', () => { + beforeEach(() => { + execCommand.mockReset() + probeRelayEndpointIncumbent.mockReset() + countRelayEndpointPtys.mockReset() + countRelayPtysOverBridge.mockReset() + }) + + it('never proves exit from this target’s empty lists alone', async () => { + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle()) + ).resolves.toMatchObject({ + verdict: 'unverifiable', + reason: "no census of every relay on this host's account was taken" + }) + }) + + it('stays live on a POSIX host where only desktop B’s socket runs a shell', async () => { + const host = getRemoteHostPlatform('linux-x64') + const dir = '/home/dev/.orca-remote/relay-0.1.0+aaaaaaaaaaaa' + const aSock = `${dir}/${relaySocketNameForInstanceId('desktop-a')}` + const bSock = `${dir}/${relaySocketNameForInstanceId('desktop-b')}` + execCommand.mockResolvedValue(`${aSock}\n${bSock}\n`) + probeRelayEndpointIncumbent.mockResolvedValue({ verdict: 'live', holders: [] }) + countRelayEndpointPtys.mockImplementation(async (_c: unknown, _n: string, endpoint: string) => + endpoint === bSock ? 1 : 0 + ) + const census = async () => + hostTerminalProofFromCensus( + await censusHostRelayEndpoints(conn, { + host, + remoteHome: '/home/dev', + fallbackNodePath: async () => '/usr/bin/node' + }) + ) + + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle(), census) + ).resolves.toMatchObject({ verdict: 'live', hostTerminals: 1 }) + + countRelayEndpointPtys.mockResolvedValue(0) + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle(), census) + ).resolves.toEqual({ verdict: 'exited', provenPtyIds: [] }) + }) + + it('stays live on a Windows host where only desktop B’s pipe runs a shell', async () => { + const host = getRemoteHostPlatform('win32-x64') + const version = 'relay-0.1.0+aaaaaaaaaaaa' + const dir = joinRemotePath(host, 'C:\\Users\\dev', '.orca-remote', version) + const sockA = relaySocketNameForInstanceId('desktop-a') + const sockB = relaySocketNameForInstanceId('desktop-b') + const [aPipe] = windowsRelayPipePathsForSocketName(host, dir, sockA) + const [bPipe] = windowsRelayPipePathsForSocketName(host, dir, sockB) + const bare = (pipe: string): string => pipe.slice('\\\\.\\pipe\\'.length) + execCommand.mockImplementation(async (_c: unknown, command: string) => + command === 'LIST' + ? `${version}\r\n` + : JSON.stringify({ + pipes: [bare(aPipe), bare(bPipe)], + dirs: { + [version]: { + credentials: [`${sockA}.credential`, `${sockB}.credential`], + markers: {} + } + } + }) + ) + countRelayPtysOverBridge.mockImplementation(async (_c: unknown, command: string) => + command === `BRIDGE ${bPipe}` ? 1 : 0 + ) + const census = async () => + hostTerminalProofFromCensus( + await censusWindowsHostRelays(conn, { + host, + remoteHome: 'C:\\Users\\dev', + targetId: 'desktop-a', + nodePath: async () => 'C:\\node\\node.exe' + }) + ) + + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle(), census) + ).resolves.toMatchObject({ verdict: 'live', hostTerminals: 1 }) + + countRelayPtysOverBridge.mockResolvedValue(0) + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle(), census) + ).resolves.toEqual({ verdict: 'exited', provenPtyIds: [] }) + }) + + it('is unverifiable when the host-wide census cannot answer', async () => { + await expect( + assessOrcadMigrationTerminals(noLeases, 'desktop-a', desktopAConnectedIdle(), async () => { + throw new Error('connect refused') + }) + ).resolves.toMatchObject({ verdict: 'unverifiable' }) + }) +}) diff --git a/src/main/ssh/orcad-migration-terminal-gate.test.ts b/src/main/ssh/orcad-migration-terminal-gate.test.ts new file mode 100644 index 00000000000..684783552a8 --- /dev/null +++ b/src/main/ssh/orcad-migration-terminal-gate.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshRemotePtyLease } from '../../shared/ssh-types' +import { + assessOrcadMigrationTerminals, + confirmOrcadMigrationTerminalsUnderFence, + retireProvenExitedLeases, + type ListRelayPtyIds +} from './orcad-migration-terminal-gate' + +function store(leases: Pick[]) { + const full = leases.map((lease) => ({ ...lease, targetId: 'ssh-1', createdAt: 1, updatedAt: 1 })) + return { getSshRemotePtyLeases: () => full } +} + +function relay(current: string[] | null, previous: string[] | null): ListRelayPtyIds { + const list: ListRelayPtyIds = async () => current + list.previous = async () => previous + return list +} + +// Every relay on the account, whichever desktop launched it, holds no work. +const hostIdle = async () => ({ verdict: 'exited' as const, count: 0 }) + +describe('migration terminal gate', () => { + it('proves exit from terminated leases and an empty relay', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'a', state: 'terminated' }]), + 'ssh-1', + relay([], []), + hostIdle + ) + ).resolves.toEqual({ verdict: 'exited', provenPtyIds: ['a'] }) + }) + + // Finding 3: with nothing leased here, a missing inventory must not read as nothing running. + it.each([ + ['this relay did not answer', relay(null, [])], + ['the earlier relays could not be asked', relay([], null)], + ['a Windows earlier relay could not be bridged', relay([], null)] + ])('is unverifiable with no leases when %s', async (_label, list) => { + await expect(assessOrcadMigrationTerminals(store([]), 'ssh-1', list)).resolves.toMatchObject({ + verdict: 'unverifiable' + }) + }) + + it('needs a host census, not silence, when no relay session can be asked', async () => { + await expect(assessOrcadMigrationTerminals(store([]), 'ssh-1', null)).resolves.toMatchObject({ + verdict: 'unverifiable', + reason: "no census of every relay on this host's account was taken" + }) + await expect( + assessOrcadMigrationTerminals(store([]), 'ssh-1', null, async () => ({ + verdict: 'exited', + count: 0 + })) + ).resolves.toEqual({ verdict: 'exited', provenPtyIds: [] }) + for (const verdict of ['live', 'unverifiable'] as const) { + await expect( + assessOrcadMigrationTerminals(store([]), 'ssh-1', null, async () => ({ verdict, count: 1 })) + ).resolves.toMatchObject({ verdict }) + } + await expect( + assessOrcadMigrationTerminals(store([]), 'ssh-1', null, async () => { + throw new Error('connect refused') + }) + ).resolves.toMatchObject({ verdict: 'unverifiable' }) + }) + + it('blocks an attached lease as live', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'a', state: 'attached' }]), + 'ssh-1', + async () => [] + ) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['a'] }) + }) + + // B4 after an app update: a shell a respawn superseded on its tab still runs on the previous + // relay, with no live lease here; only the leased shells were counted before. + it('counts a shell an earlier relay still runs that no lease here knows', async () => { + await expect( + assessOrcadMigrationTerminals( + store([ + { ptyId: 'pty2:8ea088dc:1', state: 'terminated' }, + { ptyId: 'pty2:8ea088dc:2', state: 'attached' } + ]), + 'ssh-1', + relay(['pty2:8ea088dc:2'], ['pty2:8ea088dc:2', 'pty2:8ea088dc:cli']) + ) + ).resolves.toMatchObject({ + verdict: 'live', + ptyIds: ['pty2:8ea088dc:2', 'pty2:8ea088dc:cli'] + }) + await expect( + assessOrcadMigrationTerminals(store([]), 'ssh-1', relay([], ['pty2:8ea088dc:cli'])) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['pty2:8ea088dc:cli'] }) + }) + + it('counts every shell the relay lists alongside the attached leases', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'a', state: 'attached' }]), + 'ssh-1', + async () => ['a', 'cli-shell'] + ) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['a', 'cli-shell'] }) + }) + + it('proves a detached terminal exited once this relay and earlier relays both answer without it', async () => { + const leases = store([{ ptyId: 'a', state: 'detached' }]) + const proof = await assessOrcadMigrationTerminals(leases, 'ssh-1', relay([], []), hostIdle) + expect(proof).toEqual({ verdict: 'exited', provenPtyIds: ['a'] }) + + // Only the move acting on the proof retires the lease; asking alone changes nothing. + const markSshRemotePtyLease = vi.fn() + retireProvenExitedLeases({ ...leases, markSshRemotePtyLease }, 'ssh-1', proof) + expect(markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'a', 'terminated') + }) + + it('blocks a detached terminal an earlier relay still runs as live', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'a', state: 'detached' }]), + 'ssh-1', + relay([], ['a']) + ) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['a'] }) + }) + + it.each([ + ['this relay did not answer', relay(null, [])], + ['earlier relays could not be asked (Windows, no census, unreachable)', relay([], null)], + ['no earlier-relay lister', async () => []] + ])('keeps a detached or expired lease unverifiable when %s', async (_label, list) => { + for (const state of ['detached', 'expired'] as const) { + await expect( + assessOrcadMigrationTerminals(store([{ ptyId: 'a', state }]), 'ssh-1', list) + ).resolves.toMatchObject({ verdict: 'unverifiable', ptyIds: ['a'] }) + } + }) + + it('blocks an expired terminal an earlier relay still runs as live', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'old', state: 'expired' }]), + 'ssh-1', + relay([], ['old']) + ) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['old'] }) + }) + + it('blocks terminals the relay still runs even with no lease for them', async () => { + await expect( + assessOrcadMigrationTerminals(store([]), 'ssh-1', async () => ['x']) + ).resolves.toMatchObject({ verdict: 'live', ptyIds: ['x'] }) + }) + + it.each([ + ['no relay to ask', null], + ['a relay that did not answer', async () => null], + [ + 'a relay that failed', + async () => { + throw new Error('channel closed') + } + ] + ])('reads an expired lease with %s as unverifiable, never as exited', async (_label, list) => { + await expect( + assessOrcadMigrationTerminals(store([{ ptyId: 'old', state: 'expired' }]), 'ssh-1', list) + ).resolves.toMatchObject({ verdict: 'unverifiable', ptyIds: ['old'] }) + }) + + it('accepts an expired lease once every relay answers that nothing runs', async () => { + await expect( + assessOrcadMigrationTerminals( + store([{ ptyId: 'old', state: 'expired' }]), + 'ssh-1', + relay([], []), + hostIdle + ) + ).resolves.toEqual({ verdict: 'exited', provenPtyIds: ['old'] }) + }) + + it('confirms under the fence only when no unproven terminal appeared', () => { + const proof = { verdict: 'exited' as const, provenPtyIds: ['old'] } + expect( + confirmOrcadMigrationTerminalsUnderFence( + store([{ ptyId: 'old', state: 'expired' }]), + 'ssh-1', + proof + ) + ).toBe(proof) + expect( + confirmOrcadMigrationTerminalsUnderFence( + store([{ ptyId: 'new', state: 'expired' }]), + 'ssh-1', + proof + ) + ).toMatchObject({ verdict: 'unverifiable', ptyIds: ['new'] }) + expect( + confirmOrcadMigrationTerminalsUnderFence( + store([{ ptyId: 'old', state: 'attached' }]), + 'ssh-1', + proof + ) + ).toMatchObject({ verdict: 'live' }) + expect( + confirmOrcadMigrationTerminalsUnderFence( + store([{ ptyId: 'new', state: 'terminated' }]), + 'ssh-1', + proof + ) + ).toBe(proof) + }) +}) diff --git a/src/main/ssh/orcad-migration-terminal-gate.ts b/src/main/ssh/orcad-migration-terminal-gate.ts new file mode 100644 index 00000000000..a7e1237f733 --- /dev/null +++ b/src/main/ssh/orcad-migration-terminal-gate.ts @@ -0,0 +1,202 @@ +/** + * The dormant migration's terminal gate: a relay PTY cannot move into orcad, so the source must + * prove that every terminal it ever leased on the target has exited. Loss of contact is never + * exit: an unanswered relay, or a lease the relay cannot account for, blocks as `unverifiable`. + */ +import { isLiveSshPtyLease } from '../../shared/ssh-pty-lease-liveness' +import type { SshRemotePtyLease } from '../../shared/ssh-types' +import type { Store } from '../persistence' + +export type OrcadMigrationTerminalVerdict = + | { verdict: 'exited'; provenPtyIds: string[] } + | { + verdict: 'live' | 'unverifiable' + ptyIds: string[] + reason: string + /** Terminals the host-wide census counted, which it reports without ids. */ + hostTerminals?: number + } + +/** + * The relay's own process list for the target; `null` when it did not answer. `previous` asks the + * relays an earlier Orca build left running the same way, `null` when they cannot be asked. + */ +export type ListRelayPtyIds = (() => Promise) & { + previous?: () => Promise +} + +/** A census of the host's relay endpoints, taken when no relay session could be asked. */ +export type HostRelayTerminalProof = { verdict: 'exited' | 'live' | 'unverifiable'; count: number } + +/** + * Every relay endpoint on the account, whichever desktop's target launched it. The only evidence + * that can prove the host idle; a census that throws proves nothing. + */ +export type CensusHostRelayTerminals = () => Promise + +type LeaseStore = Pick + +/** + * Taken before the fence, while the relay can still be asked. Read-only, so previews may ask. + * This target's relays can prove `live`, but their lists name only this target's instances, so + * `exited` also needs a complete host-wide census: another desktop's relay on the same account runs + * under a different target id. An inventory that is missing or failed is `unverifiable` even when + * this desktop leases nothing. + */ +export async function assessOrcadMigrationTerminals( + store: LeaseStore, + targetId: string, + listRelayPtyIds: ListRelayPtyIds | null, + censusHost?: CensusHostRelayTerminals | null +): Promise { + const leases = store.getSshRemotePtyLeases(targetId) + const attached = leases.filter((lease) => lease.state === 'attached') + // A detached or expired lease may run on this relay or one an earlier build left; both answer. + const unresolved = leases.filter( + (lease) => lease.state === 'detached' || lease.state === 'expired' + ) + if (!listRelayPtyIds) { + return await withoutRelaySession(leases, attached, unresolved, censusHost) + } + // The relays' own listings are the authority on what runs, leased or not: a CLI-created shell, or + // one a respawn superseded on its tab, keeps running with no live lease here. Asking the earlier + // relays only once this relay answered keeps a relay that answered nothing from counting as none. + const relayPtyIds = await ask(listRelayPtyIds) + const previousPtyIds = relayPtyIds ? await ask(listRelayPtyIds.previous) : null + const running = [...(relayPtyIds ?? []), ...(previousPtyIds ?? [])] + if (attached.length > 0 || running.length > 0) { + return { + verdict: 'live', + ptyIds: [...new Set([...attached.map((lease) => lease.ptyId), ...running])], + reason: + attached.length > 0 || (relayPtyIds?.length ?? 0) > 0 + ? 'terminals on this host are still running' + : 'an earlier Orca relay still runs terminals on this host' + } + } + if (relayPtyIds === null) { + return refuse( + 'unverifiable', + unresolved, + 'the SSH relay could not confirm its terminals here exited' + ) + } + if (previousPtyIds === null) { + return refuse('unverifiable', unresolved, 'Orca could not confirm its terminals here exited') + } + return await hostWideVerdict(leases, censusHost) +} + +async function withoutRelaySession( + leases: SshRemotePtyLease[], + attached: SshRemotePtyLease[], + unresolved: SshRemotePtyLease[], + censusHost: CensusHostRelayTerminals | null | undefined +): Promise { + if (attached.length > 0) { + return refuse('live', attached, 'terminals on this host are still running') + } + if (unresolved.length > 0) { + return refuse('unverifiable', unresolved, 'no SSH relay could confirm these terminals exited') + } + return await hostWideVerdict(leases, censusHost) +} + +async function hostWideVerdict( + leases: SshRemotePtyLease[], + censusHost: CensusHostRelayTerminals | null | undefined +): Promise { + const hostProof: HostRelayTerminalProof | null = censusHost + ? await censusHost().catch(() => ({ verdict: 'unverifiable', count: 0 }) as const) + : null + if (hostProof?.verdict === 'exited') { + return { verdict: 'exited', provenPtyIds: leases.map((lease) => lease.ptyId) } + } + if (hostProof) { + return { + verdict: hostProof.verdict, + ptyIds: [], + reason: "the host's relays still run or may run terminals", + hostTerminals: hostProof.count + } + } + return { + verdict: 'unverifiable', + ptyIds: [], + reason: "no census of every relay on this host's account was taken" + } +} + +/** + * Only the host-wide census counted them: no lease or listing of this target names one, so they + * run under another desktop's target or session and stopping this target's terminals can't end them. + */ +export function terminalsRunElsewhere(proof: OrcadMigrationTerminalVerdict): boolean { + return proof.verdict !== 'exited' && proof.ptyIds.length === 0 && (proof.hostTerminals ?? 0) > 0 +} + +/** + * Whoever acts on an exited proof (a move, or a connect whose relay session answered) marks the + * detached and expired leases it covers terminated, so later checks stop reading them as running + * or unverifiable and the next connect can convert. + */ +export function retireProvenExitedLeases( + store: Pick, + targetId: string, + proof: OrcadMigrationTerminalVerdict +): void { + if (proof.verdict !== 'exited') { + return + } + const proven = new Set(proof.provenPtyIds) + for (const lease of store.getSshRemotePtyLeases(targetId)) { + if ((lease.state === 'detached' || lease.state === 'expired') && proven.has(lease.ptyId)) { + store.markSshRemotePtyLease(targetId, lease.ptyId, 'terminated') + } + } +} + +async function ask(list: (() => Promise) | null | undefined) { + try { + return list ? await list() : null + } catch { + return null + } +} + +/** + * Re-checked under the fence, after the relay was let go: the fence stops new leases, so any + * lease the earlier proof did not cover means a terminal started in between. + */ +export function confirmOrcadMigrationTerminalsUnderFence( + store: LeaseStore, + targetId: string, + proof: OrcadMigrationTerminalVerdict +): OrcadMigrationTerminalVerdict { + if (proof.verdict !== 'exited') { + return proof + } + const proven = new Set(proof.provenPtyIds) + const leases = store.getSshRemotePtyLeases(targetId) + const live = leases.filter(isLiveSshPtyLease) + if (live.length > 0) { + return refuse('live', live, 'a terminal started on this host before the fence took hold') + } + const unproven = leases.filter((lease) => !proven.has(lease.ptyId)) + if (unproven.some((lease) => lease.state !== 'terminated')) { + return refuse( + 'unverifiable', + unproven, + 'a terminal lease appeared on this host that the relay was not asked about' + ) + } + return proof +} + +function refuse( + verdict: 'live' | 'unverifiable', + leases: SshRemotePtyLease[], + reason: string +): OrcadMigrationTerminalVerdict { + return { verdict, ptyIds: leases.map((lease) => lease.ptyId), reason } +} diff --git a/src/main/ssh/orcad-recovery-slot.ts b/src/main/ssh/orcad-recovery-slot.ts new file mode 100644 index 00000000000..29b1f6b607e --- /dev/null +++ b/src/main/ssh/orcad-recovery-slot.ts @@ -0,0 +1,196 @@ +/** + * Slot-level verdicts shared by activation, rollback and their crash recovery. + * + * Every verdict here is the execution host's: `exited` only on positive proof, and a probe + * that could not answer is `unverifiable` — never a reason to start a second slot. + */ +import type { ServeReadiness } from '../server/serve-readiness' +import type { OrcadActivationExpectation } from './orcad-activation-gate' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir } from './ssh-relay-versioned-install' +import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { + launchOrcadSlotAndAwaitReadiness, + OrcadActiveReadinessError, + probeActiveOrcadReadiness +} from './orcad-active-readiness' +import { orcadLivenessProbeCommand, parseOrcadLiveness } from './orcad-remote-launch' +import { + parseOrcadStopOutcome, + stopOrcadCommand, + type OrcadStopOutcome +} from './orcad-remote-process-control' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { orcadActivationTransactionRoot } from './orcad-activation-lock' +import { + initialOrcadActivationAdmissionCommand, + parseInitialOrcadActivationAdmission +} from './orcad-initial-activation-admission' + +export const ORCAD_SLOT_STOP_WAIT_SECONDS = 20 + +export type OrcadSlotOptions = OrcadRemoteExecTarget & { + remoteHome: string + /** Host Node for legacy slots only; a slot's own runtime marker wins. */ + nodePath: string + userDataDir: string + bindHost: string + port: number + readinessTimeoutMs?: number + sleep?: (ms: number) => Promise +} + +export type OrcadSlotIdentity = { version: string; remoteDir: string; buildHash: string } + +function expectation(identity: OrcadSlotIdentity): OrcadActivationExpectation { + return { buildHash: identity.buildHash, fullVersion: identity.version } +} + +export function orcadSlotDir(options: OrcadSlotOptions, version: string): string { + return computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, version) +} + +/** Reads the installed bytes' hash, so a later readiness payload can be matched to them. */ +export async function resolveOrcadSlotIdentity( + options: OrcadSlotOptions, + version: string +): Promise { + const remoteDir = orcadSlotDir(options, version) + return { version, remoteDir, buildHash: await readRemoteOrcadBuildHash(options, remoteDir) } +} + +export function launchOrcadSlot( + options: OrcadSlotOptions, + identity: OrcadSlotIdentity +): Promise { + return launchOrcadSlotAndAwaitReadiness( + options, + { + remoteInstallDir: identity.remoteDir, + nodePath: options.nodePath, + fullVersion: identity.version, + userDataDir: options.userDataDir, + bindHost: options.bindHost, + port: options.port, + // Every SSH launch is client-managed, so every one may idle out and be woken on connect. + activationRoot: orcadActivationTransactionRoot(options.host, options.remoteHome) + }, + expectation(identity) + ) +} + +/** Proves the slot serves, starting it only on proven exit. */ +export async function ensureOrcadSlotServing( + options: OrcadSlotOptions, + identity: OrcadSlotIdentity +): Promise { + const liveness = parseOrcadLiveness( + await execOrcadRemote(options, orcadLivenessProbeCommand(options.host, identity.remoteDir)) + ) + if (liveness === 'LIVE') { + return probeActiveOrcadReadiness( + { ...options, remoteInstallDir: identity.remoteDir }, + expectation(identity) + ) + } + if (liveness === 'UNKNOWN') { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${identity.version} process state is unverifiable.` + ) + } + return launchOrcadSlot(options, identity) +} + +/** `justLaunched` only for a slot this run started; otherwise the readiness PID must agree. */ +export async function stopOrcadSlot( + options: OrcadSlotOptions, + remoteDir: string, + justLaunched: boolean +): Promise { + return parseOrcadStopOutcome( + await execOrcadRemote( + options, + stopOrcadCommand( + options.host, + remoteDir, + justLaunched + ? { waitSeconds: ORCAD_SLOT_STOP_WAIT_SECONDS, justLaunched: true } + : { waitSeconds: ORCAD_SLOT_STOP_WAIT_SECONDS, nodePath: options.nodePath } + ) + ) + ) +} + +export type OrcadSlotQuiescence = 'exited' | 'other-slot-serving' + +/** + * Before an interrupted run's state is replaced: the slot it started must be proven exited and + * the slot being restored must not be running, or must already be serving (nothing to undo). + * + * A missing PID file is not proof of exit: an SSH drop can kill the launcher after `nohup` + * but before it records `$!`, so the data root's owner records decide. + */ +export async function quiesceInterruptedOrcadSlot( + options: OrcadSlotOptions, + /** `null` when nothing was launched, so only `otherSlot` needs ruling out. */ + version: string | null, + otherSlot: OrcadSlotIdentity | null +): Promise { + const remoteDir = version === null ? null : orcadSlotDir(options, version) + const stopped = + remoteDir === null ? 'already-exited' : await stopOrcadSlot(options, remoteDir, false) + let exited = stopped === 'stopped' || stopped === 'already-exited' + if ((stopped === 'unknown' || stopped === 'unconfirmed') && remoteDir !== null) { + // No verified answer, e.g. the slot died before readiness; liveness decides. + exited = (await slotLiveness(options, remoteDir)) === 'DEAD' + } + if (otherSlot) { + const other = await slotLiveness(options, otherSlot.remoteDir) + if (other === 'LIVE' && (exited || stopped === 'no-pid')) { + // The serving slot holds the instance lock, so a PID-less launch cannot own the state. + await probeActiveOrcadReadiness( + { ...options, remoteInstallDir: otherSlot.remoteDir }, + expectation(otherSlot) + ) + return 'other-slot-serving' + } + if (other !== 'DEAD') { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${version ?? 'candidate'} (${stopped}) and ${otherSlot.version} (${other}) cannot both be ` + + 'ruled out as owners of the shared state; it was not replaced.' + ) + } + } + if (!exited && stopped === 'no-pid' && remoteDir !== null) { + const admission = parseInitialOrcadActivationAdmission( + await execOrcadRemote( + options, + initialOrcadActivationAdmissionCommand( + options.host, + options.userDataDir, + remoteDir, + options.nodePath + ) + ) + ) + exited = admission.decision === 'proceed' + } + if (!exited) { + throw new OrcadActiveReadinessError( + 'unverifiable', + `orcad ${version ?? 'candidate'} could not be confirmed stopped (${stopped}); replacing its state would be unsafe.` + ) + } + return 'exited' +} + +export async function slotLiveness( + options: OrcadSlotOptions, + remoteDir: string +): Promise<'LIVE' | 'DEAD' | 'UNKNOWN'> { + return parseOrcadLiveness( + await execOrcadRemote(options, orcadLivenessProbeCommand(options.host, remoteDir)) + ) +} diff --git a/src/main/ssh/orcad-remote-build-hash.ts b/src/main/ssh/orcad-remote-build-hash.ts new file mode 100644 index 00000000000..1308c4868f0 --- /dev/null +++ b/src/main/ssh/orcad-remote-build-hash.ts @@ -0,0 +1,42 @@ +/** The installed slot's `orcad.js` identity, the same 16-hex prefix orcad reports in its health. */ +import { shellEscape } from './ssh-connection-utils' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { orcadWindowsBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { ORCAD_BUILD_HASH_MARKER as BUILD_HASH_MARKER } from './orcad-windows-host-script' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export async function readRemoteOrcadBuildHash( + target: OrcadRemoteExecTarget, + remoteInstallDir: string +): Promise { + const output = await execOrcadRemote( + target, + remoteOrcadBuildHashCommand(target.host, remoteInstallDir) + ) + const match = output.match(/__ORCAD_BUILD_HASH__\s+([a-fA-F0-9]{16})/u) + if (!match?.[1]) { + throw new Error('Could not verify the installed orcad build hash.') + } + return match[1].toLowerCase() +} + +export function remoteOrcadBuildHashCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + if (isWindowsRemoteHost(host)) { + return orcadWindowsHostOpCommand( + host, + orcadWindowsBaseDir(host, remoteInstallDir), + 'build-hash', + [joinRemotePath(host, remoteInstallDir, 'orcad.js')] + ) + } + const path = shellEscape(joinRemotePath(host, remoteInstallDir, 'orcad.js')) + // Why both tools: GNU/busybox ship sha256sum, macOS ships shasum; either prints the digest first. + return [ + `orca_hash=$(if command -v sha256sum >/dev/null 2>&1; then sha256sum ${path} | awk '{print $1}';`, + `elif command -v shasum >/dev/null 2>&1; then shasum -a 256 ${path} | awk '{print $1}'; fi);`, + `case "$orca_hash" in [0-9a-fA-F][0-9a-fA-F]*) printf '%s %.16s\\n' ${shellEscape(BUILD_HASH_MARKER)} "$orca_hash";; esac` + ].join(' ') +} diff --git a/src/main/ssh/orcad-remote-context.ts b/src/main/ssh/orcad-remote-context.ts new file mode 100644 index 00000000000..8ed4300c2a6 --- /dev/null +++ b/src/main/ssh/orcad-remote-context.ts @@ -0,0 +1,67 @@ +/** Everything a managed-orcad operation needs to know about one SSH host before it acts. */ +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import type { SshTarget } from '../../shared/ssh-types' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { resolveOrcadRuntimeTarget } from './orcad-runtime-target' +import { prepareWindowsOrcadHost } from './orcad-windows-host-preparation' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import type { SshConnection } from './ssh-connection' +import { readRemoteHomeCommand } from './ssh-remote-commands' +import { + isWindowsRemoteHost, + joinRemotePath, + normalizeRemoteHome, + validateRemoteHome, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' +import { OrcadHostUnsupportedError } from './orcad-host-unavailable' +import { rememberSshHostPlatform } from './ssh-host-platform-memo' + +export type OrcadRemoteContext = { + activationRecord: OrcadActivationRecord + /** A compat runtime on hosts below the default runtime's glibc floor (design D6 rung B). */ + serverTarget: NodeRuntimeTarget + connection: SshConnection + host: RemoteHostPlatform + remoteHome: string + target: SshTarget + userDataDir: string +} + +export async function resolveOrcadRemoteContext( + target: SshTarget, + connection: SshConnection, + signal?: AbortSignal, + detectedHost?: RemoteHostPlatform +): Promise { + const host = detectedHost ?? (await detectRemoteHostPlatform(connection, { signal })) + if (!host) { + throw new OrcadHostUnsupportedError('This SSH host platform is not supported by managed orcad.') + } + const remote = { conn: connection, host, signal } + const remoteHome = normalizeRemoteHome( + await execOrcadRemote(remote, readRemoteHomeCommand(host)), + host + ) + if (!validateRemoteHome(remoteHome, host)) { + throw new Error(`Remote home is not a valid path: ${remoteHome.slice(0, 100)}`) + } + const serverTarget = await resolveOrcadRuntimeTarget({ conn: connection, host, signal }) + rememberSshHostPlatform(target.id, host, serverTarget) + if (isWindowsRemoteHost(host)) { + // Every Windows host op, the activation record read included, runs on the pinned node.exe. + await prepareWindowsOrcadHost({ conn: connection, host, remoteHome, serverTarget, signal }) + } + const activationRecord = await readOrcadActivationRecord({ ...remote, remoteHome }) + return { + activationRecord, + serverTarget, + connection, + host, + remoteHome, + target, + userDataDir: joinRemotePath(host, remoteHome, '.orca') + } +} diff --git a/src/main/ssh/orcad-remote-decommission.test.ts b/src/main/ssh/orcad-remote-decommission.test.ts new file mode 100644 index 00000000000..6c75f66422f --- /dev/null +++ b/src/main/ssh/orcad-remote-decommission.test.ts @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as RecordFile from './orcad-remote-record-file' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn() +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' +import { decommissionRemoteOrcad } from './orcad-remote-stop' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { parseOrcadActivationRecord } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' +import { FakeOrcadHost, OLD, type CrashMode } from './orcad-activation-host-test-harness' + +let host = new FakeOrcadHost() + +const slot = { + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z') +} +const idle: OrcadTerminalCensus = { + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: 3 +} + +function currentRecord() { + const parsed = parseOrcadActivationRecord(host.record) + if (parsed.state !== 'ok') { + throw new Error('fixture record is unreadable') + } + return parsed.record +} + +const decommission = (census: OrcadTerminalCensus = idle) => + decommissionRemoteOrcad({ ...slot, record: currentRecord(), census }) + +beforeEach(() => { + vi.clearAllMocks() + host = FakeOrcadHost.deployedOld() + vi.mocked(execCommand).mockImplementation(async (_conn, command) => host.exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async (_conn, _root, _host, options) => { + host.acquireFence(options) + }) + vi.mocked(writeAtomicOrcadRemoteRecord).mockImplementation(async (_target, path, contents) => + host.write(path, contents) + ) +}) + +function expectExactlyTheRecordedSlot(): void { + const active = host.activeVersion() + expect([...host.alive]).toEqual(active ? [active] : []) + expect(host.journal).toBeNull() + expect(host.fence).toBe(false) +} + +describe('decommissioning a managed orcad', () => { + it('stops the active instance by request, then records that nothing serves', async () => { + expect(await decommission()).toEqual({ + outcome: 'decommissioned', + version: OLD, + retirement: 'retired' + }) + expect(currentRecord()).toMatchObject({ active: null, previous: OLD, snapshot: null }) + expectExactlyTheRecordedSlot() + // Never a signal: the instance-bound request is the only stop path. + expect(host.commands.some((command) => command.includes('kill -TERM'))).toBe(false) + }) + + it.each([ + [{ ...idle, liveSessions: null }, 'unverifiable', 'orcad_decommission_census_unavailable'], + [{ ...idle, liveSessions: 2 }, 'live', 'orcad_decommission_terminals_running'] + ] as const)('refuses while the census says %j', async (census, verdict, code) => { + expect(await decommission(census)).toMatchObject({ outcome: 'refused', verdict, code }) + expect(host.alive.has(OLD)).toBe(true) + expect(writeAtomicOrcadRemoteRecord).not.toHaveBeenCalled() + }) + + it('refuses an orcad that cannot be addressed by request', async () => { + host.alive.clear() + expect(await decommission()).toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_managed_stop_readiness_unverifiable' + }) + expect(writeAtomicOrcadRemoteRecord).not.toHaveBeenCalled() + }) + + it('withdraws a request orcad never acted on and keeps serving', async () => { + host.managedStop = 'stays' + expect(await decommission()).toMatchObject({ + outcome: 'refused', + verdict: 'live', + code: 'orcad_decommission_stop_withdrawn' + }) + expect(currentRecord().active).toBe(OLD) + expectExactlyTheRecordedSlot() + }) + + it('keeps the fence while orcad is still stopping, and recovery finishes once it exits', async () => { + host.managedStop = 'stays-dispatched' + expect(await decommission()).toMatchObject({ + outcome: 'refused', + verdict: 'live', + code: 'orcad_decommission_stop_unsettled' + }) + expect(host.fence).toBe(true) + expect(host.journal).not.toBeNull() + expect(await recoverInterruptedOrcadActivation(slot)).toMatchObject({ + outcome: 'refused', + code: 'orcad_recovery_decommission_unsettled' + }) + host.managedStop = 'exits' + expect(await recoverInterruptedOrcadActivation(slot)).toMatchObject({ + outcome: 'recovered', + resolution: 'committed', + activeVersion: null + }) + expectExactlyTheRecordedSlot() + }) + + it.each(['before', 'after'])( + 'recovers to exactly the recorded slot when interrupted at every mutation (%s)', + async (mode) => { + host = FakeOrcadHost.deployedOld() + await decommission() + const total = host.mutations + expect(total).toBeGreaterThan(3) + for (let crashAt = 1; crashAt <= total; crashAt += 1) { + host = FakeOrcadHost.deployedOld() + host.crashAt = crashAt + host.crashMode = mode + await decommission().catch(() => undefined) + host.crashAt = null + const result = await recoverInterruptedOrcadActivation(slot) + expect(['recovered', 'none'], `mutation ${crashAt}`).toContain(result.outcome) + expectExactlyTheRecordedSlot() + } + } + ) +}) diff --git a/src/main/ssh/orcad-remote-deploy-stop.ts b/src/main/ssh/orcad-remote-deploy-stop.ts deleted file mode 100644 index 8a00a1dfa03..00000000000 --- a/src/main/ssh/orcad-remote-deploy-stop.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { SshConnection } from './ssh-connection' -import { execCommand } from './ssh-relay-deploy-helpers' -import { ORCAD_INSTALL_MODEL } from './remote-install-model' -import { computeRemoteInstallDir } from './ssh-relay-versioned-install' -import { - parseOrcadStopOutcome, - stopOrcadCommand, - type OrcadStopOutcome -} from './orcad-remote-process-control' -import type { RemoteHostPlatform } from './ssh-remote-platform' -import { compareOrcadStateSnapshotCommand, orcadSnapshotIsUnchanged } from './orcad-state-snapshot' - -export type OrcadOutgoingStopOptions = { - conn: SshConnection - host: RemoteHostPlatform - remoteHome: string - nodePath: string - signal?: AbortSignal -} - -/** Stop the outgoing runtime and return its execution-host verdict. */ -export async function stopOutgoingOrcad( - options: OrcadOutgoingStopOptions, - outgoingVersion: string -): Promise { - const outgoingDir = computeRemoteInstallDir( - ORCAD_INSTALL_MODEL, - options.remoteHome, - outgoingVersion - ) - const output = await execCommand( - options.conn, - stopOrcadCommand(options.host, outgoingDir, { waitSeconds: 20, nodePath: options.nodePath }), - { - wrapCommand: options.host.commandDialect !== 'powershell', - signal: options.signal - } - ) - return parseOrcadStopOutcome(output) -} - -/** The caller must confirm candidate exit before inspecting its shared state. */ -export async function rejectedOrcadStateRecoveryRefusal( - options: OrcadOutgoingStopOptions & { userDataDir: string }, - incumbentVersion: string, - snapshotDir: string | undefined -): Promise { - const unchanged = snapshotDir - ? orcadSnapshotIsUnchanged( - await execCommand( - options.conn, - compareOrcadStateSnapshotCommand(options.host, options.userDataDir, snapshotDir), - { wrapCommand: options.host.commandDialect !== 'powershell', signal: options.signal } - ).catch(() => '') - ) - : false - if (unchanged) { - return undefined - } - // RPC was already exposed; a prelaunch census cannot authorize discarding candidate writes. - const retainedSnapshot = snapshotDir ? ` at ${snapshotDir}.` : ', which is unavailable.' - return ( - 'The candidate is stopped, but profile state changed or could not be verified. ' + - `orcad ${incumbentVersion} was not restarted against potentially incompatible state. ` + - 'Current state and daemon terminals are preserved; recovery requires a fresh host ' + - `terminal census before restoring the prelaunch snapshot${retainedSnapshot}` - ) -} diff --git a/src/main/ssh/orcad-remote-deploy.test.ts b/src/main/ssh/orcad-remote-deploy.test.ts index 8308ff52db5..6742623a91a 100644 --- a/src/main/ssh/orcad-remote-deploy.test.ts +++ b/src/main/ssh/orcad-remote-deploy.test.ts @@ -1,4 +1,5 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' +import type * as RecordFile from './orcad-remote-record-file' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -11,12 +12,18 @@ vi.mock('./ssh-relay-deploy-helpers', () => ({ vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) vi.mock('./ssh-relay-install-lock', () => ({ acquireInstallLock: vi.fn().mockResolvedValue(undefined), + isRelayInstallLockStale: vi.fn().mockResolvedValue(false), + RemoteInstallLockBusyError: class extends Error {}, RELAY_INSTALL_LOCK_NAME: '.install-lock' })) vi.mock('./ssh-relay-install-transfers', () => ({ uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), writeRelayFile: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) vi.mock('./orcad-remote-node-runtime', () => ({ ensureRemoteOrcadNodeRuntime: vi.fn().mockResolvedValue(undefined) })) @@ -25,8 +32,9 @@ vi.mock('./orcad-local-build-hash', () => ({ })) import { execCommand } from './ssh-relay-deploy-helpers' -import { acquireInstallLock } from './ssh-relay-install-lock' -import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { acquireInstallLock, isRelayInstallLockStale } from './ssh-relay-install-lock' +import { uploadRelayDirectory } from './ssh-relay-install-transfers' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' import { installOrcadBundle } from './orcad-remote-install' import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' @@ -37,8 +45,12 @@ import { } from './ssh-relay-versioned-install' import { emptyOrcadActivationRecord, withActivatedVersion } from './orcad-activation-record' import { getRemoteHostPlatform } from './ssh-remote-platform' +import { isReadinessRead } from './orcad-activation-host-test-harness' +import { isSnapshotCaptureCommand } from './orcad-snapshot-capture-command' import type { SshConnection } from './ssh-connection' import { NODE_RUNTIME_PIN } from '../../shared/node-runtime-pin' +import { serializeOrcadActivationTransaction } from './orcad-activation-transaction' +import { createOrcadActivationTransaction } from './orcad-activation-transaction-transitions' const mockExec = vi.mocked(execCommand) const NEW_VERSION = '0.2.0+bb0100000000' @@ -53,6 +65,7 @@ vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ })) function readyLine(overrides: { + version?: string buildHash?: string daemonState?: 'live' | 'degraded' | 'absent' selfTestOk?: boolean @@ -67,7 +80,7 @@ function readyLine(overrides: { pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, health: { buildHash: overrides.buildHash ?? 'abc123def4567890', - buildVersion: NEW_VERSION, + buildVersion: overrides.version ?? NEW_VERSION, nodeVersion: '20.11.0', nodeAbi: '115', platform: 'linux', @@ -101,15 +114,30 @@ type HostScript = { comparisonResult?: string candidateStopResult?: string readinessAtMs?: number + orcadLog?: string } function scriptHost(script: HostScript): void { mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) - if (text.startsWith('cat ') && text.includes('orcad-active.json')) { - return script.activationRecord + if (text.startsWith('tail -c')) { + return script.orcadLog ?? '' } - if (text.includes('.orcad-readiness') && text.startsWith('cat ')) { + if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('orcad-active.json')) { + return script.activationRecord + ? `__ORCAD_RECORD_PRESENT__\n${script.activationRecord}` + : '__ORCAD_RECORD_ABSENT__\n' + } + if (text.includes('__ORCAD_RECORD_PRESENT__') && text.includes('transaction.json')) { + return '__ORCAD_RECORD_ABSENT__\n' + } + if (text.includes('__ORCAD_BUILD_HASH__')) { + return '__ORCAD_BUILD_HASH__ abc123def4567890\n' + } + if (text.includes('orcad.lock') && text.includes('orca-runtime.json')) { + return 'CLEAR' + } + if (text.includes('.orcad-readiness') && isReadinessRead(text)) { if (script.readinessAtMs !== undefined && Date.now() < script.readinessAtMs) { return '' } @@ -139,7 +167,7 @@ function scriptHost(script: HostScript): void { script.log.push(`stop:${text.includes(NEW_VERSION) ? NEW_VERSION : OLD_VERSION}`) return text.includes(NEW_VERSION) ? (script.candidateStopResult ?? 'STOPPED') : 'STOPPED' } - if (text.includes('tar -C') && text.includes('-cf')) { + if (isSnapshotCaptureCommand(text)) { script.log.push('snapshot') return script.snapshotResult ?? 'CAPTURED' } @@ -162,7 +190,7 @@ function options(overrides: Partial = {}): OrcadDeployOption userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', port: 7777, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, readinessTimeoutMs: 50, sleep: async () => {}, now: () => new Date('2026-02-02T00:00:00.000Z'), @@ -272,8 +300,8 @@ describe('deployOrcad', () => { expect(script.log).toEqual(['preflight']) expect( vi - .mocked(writeRelayFile) - .mock.calls.some(([, , path]) => path.includes('orcad-active.json')) + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.some(([, path]) => path.includes('orcad-active.json')) ).toBe(false) } ) @@ -307,7 +335,7 @@ describe('deployOrcad', () => { options({ host: getRemoteHostPlatform(platform), target, - census: { liveSessions: 1, startedSinceActivation: 0 } + census: { liveSessions: 1, startedSinceActivation: 0, daemonProtocolVersion: 3 } }) ) const chmod = mockExec.mock.calls.findIndex(([, command]) => @@ -351,12 +379,65 @@ describe('deployOrcad', () => { ) }) + it.each([ + ['a fresh fence with no journal', false, false, 'orcad_activation_fence_busy'], + ['a stale fence over a journal', true, true, 'orcad_activation_recovery_required'], + ['a fresh fence over a live run journal', false, true, 'orcad_activation_fence_busy'] + ])('refuses before uploading on %s', async (_label, stale, journal, code) => { + vi.mocked(isRelayInstallLockStale).mockResolvedValueOnce(stale) + const JOURNAL = serializeOrcadActivationTransaction( + createOrcadActivationTransaction({ + transactionId: '00000000-0000-4000-8000-000000000001', + candidateVersion: NEW_VERSION, + recordBefore: emptyOrcadActivationRecord(), + snapshotDirName: 'pre-1', + now: new Date(0) + }) + ) + mockExec.mockImplementation(async (_conn, command) => { + const text = String(command) + if (text.includes('echo LOCKED || echo OPEN')) { + return 'LOCKED\n' + } + if (text.includes('__ORCAD_RECORD_ABSENT__') && text.includes('transaction.json')) { + return journal ? `__ORCAD_RECORD_PRESENT__\n${JOURNAL}` : '__ORCAD_RECORD_ABSENT__\n' + } + return text.includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : '' + }) + await expect(deployOrcad(options())).resolves.toMatchObject({ + outcome: 'installed-not-activated', + code + }) + expect(uploadRelayDirectory).not.toHaveBeenCalled() + }) + + // BUG-21: a bare stale fence (a wake cut short) failed every update with "Recover it first". + it('clears a stale fence no journal backs and goes on with the update', async () => { + vi.mocked(isRelayInstallLockStale).mockResolvedValueOnce(true) + let fenced = true + mockExec.mockImplementation(async (_conn, command) => { + const text = String(command) + if (text.includes('echo LOCKED || echo OPEN')) { + return fenced ? 'LOCKED\n' : 'OPEN\n' + } + if (text.includes('echo RELEASED')) { + fenced = false + } + return text.includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : '' + }) + await deployOrcad(options()).catch(() => undefined) + expect(vi.mocked(acquireInstallLock).mock.calls[0]?.[3]).toMatchObject({ + allowStaleTakeover: true + }) + expect(uploadRelayDirectory).toHaveBeenCalled() + }) + it('leaves an upload incomplete when the remote cannot make search executable', async () => { mockExec.mockImplementation(async (_conn, command) => { if (String(command).startsWith('chmod 755 ')) { throw new Error('chmod failed') } - return '' + return String(command).includes('__ORCAD_RECORD_ABSENT__') ? '__ORCAD_RECORD_ABSENT__\n' : '' }) await expect(deployOrcad(options())).rejects.toThrow('chmod failed') expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled() @@ -441,9 +522,9 @@ describe('deployOrcad', () => { const result = await deployOrcad(options()) expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: NEW_VERSION }) const written = vi - .mocked(writeRelayFile) - .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) - expect(JSON.parse(String(written?.[3]))).toMatchObject({ + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[2]))).toMatchObject({ active: NEW_VERSION, previous: OLD_VERSION }) @@ -470,7 +551,7 @@ describe('deployOrcad', () => { } scriptHost(script) const result = await deployOrcad( - options({ census: { liveSessions: 2, startedSinceActivation: 0 } }) + options({ census: { liveSessions: 2, startedSinceActivation: 0, daemonProtocolVersion: 3 } }) ) expect(result).toMatchObject({ outcome: 'installed-not-activated', @@ -485,15 +566,20 @@ describe('deployOrcad', () => { const script: HostScript = { activationRecord: ACTIVE_OLD, readiness: { [NEW_VERSION]: readyLine({ daemonState: 'degraded' }) }, - log: [] + log: [], + orcadLog: 'daemon: starting\ndaemon: socket bind failed: EACCES\n' } scriptHost(script) const result = await deployOrcad(options()) expect(result).toMatchObject({ code: 'orcad_activation_daemon_degraded' }) + // The error carries why the candidate failed, not only where its log is. + expect(result).toMatchObject({ + reason: expect.stringMatching(/Last lines of orcad\.log:\ndaemon: starting\n.*EACCES$/u) + }) expect( vi - .mocked(writeRelayFile) - .mock.calls.some((call) => String(call[2]).endsWith('orcad-active.json')) + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.some((call) => String(call[1]).endsWith('orcad-active.json')) ).toBe(false) }) @@ -502,7 +588,7 @@ describe('deployOrcad', () => { activationRecord: ACTIVE_OLD, readiness: { [NEW_VERSION]: readyLine({ selfTestOk: false }), - [OLD_VERSION]: readyLine({}) + [OLD_VERSION]: readyLine({ version: OLD_VERSION }) }, log: [] } @@ -546,7 +632,7 @@ describe('deployOrcad', () => { 'compare-state' ]) expect(result.outcome === 'installed-not-activated' && result.reason).toContain( - 'recovery requires a fresh host terminal census' + 'Recover to restore the prelaunch snapshot' ) expect(result.outcome === 'installed-not-activated' && result.reason).toContain( '/home/u/.orca-remote/orcad-state-snapshots/' @@ -560,7 +646,7 @@ describe('deployOrcad', () => { it('restarts the incumbent when a quiescent snapshot cannot be captured', async () => { const script: HostScript = { activationRecord: ACTIVE_OLD, - readiness: { [OLD_VERSION]: readyLine({}) }, + readiness: { [OLD_VERSION]: readyLine({ version: OLD_VERSION }) }, log: [], snapshotResult: 'tar: write failed' } @@ -603,7 +689,7 @@ describe('deployOrcad', () => { activationRecord: ACTIVE_OLD, readiness: { [NEW_VERSION]: readyLine({ buildHash: 'deadbeefdeadbeef' }), - [OLD_VERSION]: readyLine({}) + [OLD_VERSION]: readyLine({ version: OLD_VERSION }) }, log: [] } diff --git a/src/main/ssh/orcad-remote-deploy.ts b/src/main/ssh/orcad-remote-deploy.ts index f296e08bd36..28dcc69602a 100644 --- a/src/main/ssh/orcad-remote-deploy.ts +++ b/src/main/ssh/orcad-remote-deploy.ts @@ -2,52 +2,34 @@ * Activate installed bytes only after the candidate proves healthy. A rejected candidate * allows restarting the incumbent only when profile state is provably unchanged; otherwise * preserve current state and the prelaunch snapshot for explicit recovery. + * + * Every activation runs under the host's activation fence and journal + * (`orcad-activation-lock.ts`), so an interrupted one is recoverable to exactly one slot. */ -import type { SshConnection } from './ssh-connection' -import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' -import { execCommand } from './ssh-relay-deploy-helpers' -import { ORCAD_INSTALL_MODEL } from './remote-install-model' -import { writeRelayFile } from './ssh-relay-install-transfers' -import { computeRemoteInstallDir, readLocalFullVersion } from './ssh-relay-versioned-install' -import { RELAY_REMOTE_DIR } from './relay-protocol' -import { - ORCAD_STATE_SNAPSHOT_DIR, - serializeOrcadActivationRecord, - withActivatedVersion, - type OrcadActivationRecord, - type OrcadStateSnapshot -} from './orcad-activation-record' -import { orcadActivationPath, readOrcadActivationRecord } from './orcad-activation-record-store' -import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' -import { planOrcadUpdate, type OrcadTerminalCensus } from './orcad-update-plan' -import { - ORCAD_LOG_FILENAME, - orcadLaunchCommand, - parseOrcadReadinessOutput, - readOrcadReadinessCommand -} from './orcad-remote-launch' -import { rejectedOrcadStateRecoveryRefusal, stopOutgoingOrcad } from './orcad-remote-deploy-stop' -import { - captureOrcadStateSnapshotCommand, - orcadSnapshotDirName, - parseOrcadSnapshotCapture -} from './orcad-state-snapshot' -import { - orcadStopFreedTheHost, - parseOrcadStopOutcome, - stopOrcadCommand -} from './orcad-remote-process-control' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' -import { computeLocalOrcadBuildHash } from './orcad-local-build-hash' -import { preflightInstalledOrcad } from './orcad-remote-preflight' -import { assertPosixOrcadHost } from './orcad-remote-host-support' -import { installOrcadBundle } from './orcad-remote-install' +import { logOrcadActivationOutcome } from './orcad-activation-outcome-log' import { join } from 'node:path' +import type { SshConnection } from './ssh-connection' +import { ORCAD_INSTALL_MODEL } from './remote-install-model' +import { computeRemoteInstallDir, readLocalFullVersion } from './ssh-relay-versioned-install' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import type { OrcadActivationVerdict } from './orcad-activation-gate' +import type { OrcadTerminalCensus } from './orcad-update-plan' +import type { RemoteHostPlatform } from './ssh-remote-platform' +import { installOrcadBundle } from './orcad-remote-install' import { getAppEnvironment } from '../../shared/app-environment' -import type { ServerTarget } from '../../shared/node-runtime-pin' +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' import { materializeOrcadArtifact } from './orcad-artifact-materializer' -import { readOrcadBundleTarget, resolveOrcadDeploymentTarget } from './orcad-deployment-target' +import { readOrcadBundleTarget } from './orcad-deployment-target' +import { resolveOrcadRuntimeTarget } from './orcad-runtime-target' import { materializeNodeRuntimeArchive } from './pinned-runtime-materializer' +import { + orcadActivationFenceExists, + resolveOrcadActivationReadinessTimeout, + withOrcadActivationLock +} from './orcad-activation-lock' +import { activateInstalledOrcad } from './orcad-installed-activation' +import { orcadActivationFenceRefusal } from './orcad-activation-fence-hold' export type OrcadDeployOptions = { conn: SshConnection @@ -55,8 +37,8 @@ export type OrcadDeployOptions = { remoteHome: string /** An already assembled bundle; otherwise materialize the packaged template for this host. */ localOrcadDir?: string - /** The bundle's server target; read from `localOrcadDir` or probed when absent. */ - target?: ServerTarget + /** The bundle's runtime target; read from `localOrcadDir` or probed when absent. */ + target?: NodeRuntimeTarget /** Where the pinned runtime archive is cached; defaults beside the orcad artifact cache. */ runtimeCacheRoot?: string nodePath: string @@ -70,6 +52,8 @@ export type OrcadDeployOptions = { */ census: OrcadTerminalCensus force?: boolean + /** The Orca app version activating, recorded so an older client never downgrades the host. */ + appVersion?: string readinessTimeoutMs?: number now?: () => Date sleep?: (ms: number) => Promise @@ -81,140 +65,23 @@ export type OrcadDeployResult = | { outcome: 'already-active'; fullVersion: string } | { outcome: 'installed-not-activated'; fullVersion: string; code: string; reason: string } -const READINESS_POLL_MS = 500 -const STOP_WAIT_SECONDS = 20 - -function exec(options: OrcadDeployOptions, command: string): Promise { - return execCommand(options.conn, command, { - wrapCommand: options.host.commandDialect !== 'powershell', - signal: options.signal - }) -} - -function baseDir(options: OrcadDeployOptions): string { - return joinRemotePath(options.host, options.remoteHome, RELAY_REMOTE_DIR) -} - -async function captureSnapshot( - options: OrcadDeployOptions, - fullVersion: string, - outgoingVersion: string | null, - takenAt: Date -): Promise { - // The caller has already stopped the outgoing runtime. This is required once profile state - // includes SQLite: a tar of a live WAL, main database, and SHM file is not a SQLite backup. - const dirName = orcadSnapshotDirName(fullVersion, takenAt.getTime()) - const snapshotDir = joinRemotePath( - options.host, - baseDir(options), - ORCAD_STATE_SNAPSHOT_DIR, - dirName - ) - const capture = parseOrcadSnapshotCapture( - await exec( - options, - captureOrcadStateSnapshotCommand(options.host, options.userDataDir, snapshotDir) - ) - ) - if (capture === 'failed') { - throw new Error( - `Could not snapshot ${options.userDataDir} before activating ${fullVersion}. Orca's ` + - 'persisted state carries no schema version, so without a snapshot a rollback has no ' + - 'way back. Refusing to activate.' - ) - } - // Empty profiles need no rollback snapshot. - if (capture === 'empty') { - return null - } - return { - dirName, - takenBeforeVersion: fullVersion, - readableByVersion: outgoingVersion, - takenAt: takenAt.toISOString() - } -} - -async function launchAndAwaitReadiness( - options: OrcadDeployOptions, - remoteInstallDir: string, - fullVersion: string -): Promise> { - await exec( - options, - orcadLaunchCommand(options.host, { ...options, remoteInstallDir, fullVersion }) - ) - const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS) - const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) - let last = parseOrcadReadinessOutput('') - while (Date.now() < deadline) { - options.signal?.throwIfAborted() - last = parseOrcadReadinessOutput( - await exec(options, readOrcadReadinessCommand(options.host, remoteInstallDir)) - ) - if (last.state !== 'pending') { - return last - } - await sleep(READINESS_POLL_MS) - } - return last -} - -/** Restart the incumbent only when the candidate left shared state unchanged. */ -async function restoreIncumbent( - options: OrcadDeployOptions, - record: OrcadActivationRecord, - candidateDir?: string, - snapshot?: OrcadStateSnapshot | null -): Promise { - if (candidateDir) { - const stopped = parseOrcadStopOutcome( - await exec( - options, - stopOrcadCommand(options.host, candidateDir, { - waitSeconds: STOP_WAIT_SECONDS, - justLaunched: true - }) - ) - ) - if (!orcadStopFreedTheHost(stopped)) { - return `The candidate itself did not stop (${stopped}); the host may still be serving the rejected build.` - } - } - if (!record.active) { - return 'No previous version was active, so this host is now serving nothing.' - } - if (candidateDir) { - const snapshotDir = snapshot - ? joinRemotePath(options.host, baseDir(options), ORCAD_STATE_SNAPSHOT_DIR, snapshot.dirName) - : undefined - const refusal = await rejectedOrcadStateRecoveryRefusal(options, record.active, snapshotDir) - if (refusal) { - return refusal - } - } - const incumbentDir = computeRemoteInstallDir( - ORCAD_INSTALL_MODEL, - options.remoteHome, - record.active - ) - const parsed = await launchAndAwaitReadiness(options, incumbentDir, record.active) - return parsed.state === 'ready' - ? `orcad ${record.active} was restarted and is serving again.` - : `orcad ${record.active} was relaunched but has not published readiness; this host may be down.` -} - /** Activate on a healthy verdict; retain changed candidate state for explicit recovery. */ -export async function deployOrcad(input: OrcadDeployOptions): Promise { - assertPosixOrcadHost(input.host) +export async function deployOrcad( + input: OrcadDeployOptions, + retryClearedFence = true +): Promise { const target = input.target ?? (input.localOrcadDir ? readOrcadBundleTarget(input.localOrcadDir) - : await resolveOrcadDeploymentTarget(input)) + : await resolveOrcadRuntimeTarget(input)) const options = { ...input, target, + readinessTimeoutMs: resolveOrcadActivationReadinessTimeout( + input.readinessTimeoutMs, + ORCAD_STARTUP_READINESS_TIMEOUT_MS + ), nodeRuntimeArchive: () => materializeNodeRuntimeArchive( target, @@ -224,108 +91,33 @@ export async function deployOrcad(input: OrcadDeployOptions): Promise new Date()) const fullVersion = readLocalFullVersion(options.localOrcadDir) const remoteDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, fullVersion) - const record = await readOrcadActivationRecord(options) + const held = async (): Promise => { + const { cleared, ...refusal } = await orcadActivationFenceRefusal(options, 'update') + // Once: a second abandoned fence means something keeps leaving them, so report it. + if (cleared && retryClearedFence) { + return deployOrcad({ ...input, target, localOrcadDir: options.localOrcadDir }, false) + } + return { outcome: 'installed-not-activated', fullVersion, ...refusal } + } + // Fail fast before upload; the activation re-reads both under the fence. + await readOrcadActivationRecord(options) + // An unanswered probe only skips this shortcut: the fence acquisition itself still decides. + if (await orcadActivationFenceExists(options).catch(() => false)) { + return held() + } await installOrcadBundle(options, fullVersion, remoteDir) - const plan = planOrcadUpdate({ - record, - candidateVersion: fullVersion, - census: options.census, - ...(options.force !== undefined ? { force: options.force } : {}) - }) - if (plan.action === 'noop') { - return { outcome: 'already-active', fullVersion } - } - if (plan.action === 'defer') { - return { - outcome: 'installed-not-activated', - fullVersion, - code: plan.code, - reason: plan.reason - } - } - - try { - await preflightInstalledOrcad({ - ...options, - remoteInstallDir: remoteDir, - fullVersion - }) - } catch (error) { - options.signal?.throwIfAborted() - return { - outcome: 'installed-not-activated', - fullVersion, - code: 'orcad_candidate_preflight_failed', - reason: `Candidate profile preflight failed; the incumbent was not stopped: ${ - error instanceof Error ? error.message : String(error) - }` - } - } - - if (record.active) { - const stopped = await stopOutgoingOrcad(options, record.active) - if (!orcadStopFreedTheHost(stopped)) { - return { - outcome: 'installed-not-activated', - fullVersion, - code: 'orcad_outgoing_stop_incomplete', - reason: - `Could not verify that orcad ${record.active} exited (${stopped}). ` + - 'No snapshot was taken and the candidate was not started. Orca requires matching ' + - 'runtime readiness before signaling an incumbent and confirmed exit before snapshotting.' - } - } - } - - // A live SQLite WAL is not a backup boundary: tar can observe the main file, WAL and SHM - // at different points and restore a set SQLite cannot recover. Stop the incumbent first so - // its final durable flush has completed before capturing the pre-activation state. - let snapshot: OrcadStateSnapshot | null = null - if (record.active) { - try { - snapshot = await captureSnapshot(options, fullVersion, record.active, now()) - } catch (error) { - const restored = await restoreIncumbent(options, record).catch( - (restartError: unknown) => - `The incumbent could not be restarted: ${ - restartError instanceof Error ? restartError.message : String(restartError) - }` - ) - throw new Error( - `${error instanceof Error ? error.message : String(error)} The incumbent was stopped ` + - `before snapshotting; ${restored}` - ) - } - } - - const parsed = await launchAndAwaitReadiness(options, remoteDir, fullVersion) - const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { - buildHash: computeLocalOrcadBuildHash(options.localOrcadDir), - fullVersion - }) - if (verdict.decision === 'reject') { - const restored = await restoreIncumbent(options, record, remoteDir, snapshot) - return { - outcome: 'installed-not-activated', - fullVersion, - code: verdict.code, - reason: - `${verdict.reason} Candidate stderr is at ` + - `${joinRemotePath(options.host, remoteDir, ORCAD_LOG_FILENAME)}. ${restored}` - } - } - - await writeRelayFile( - options.conn, - options.host, - orcadActivationPath(options.host, options.remoteHome), - serializeOrcadActivationRecord(withActivatedVersion(record, fullVersion, snapshot, now())), - { signal: options.signal } + return logOrcadActivationOutcome( + `update to ${fullVersion}`, + () => + withOrcadActivationLock( + options, + (lock) => activateInstalledOrcad(options, fullVersion, remoteDir, lock), + held + ), + ['installed-and-activated', 'already-active'] ) - return { outcome: 'installed-and-activated', fullVersion, verdict } } diff --git a/src/main/ssh/orcad-remote-gc.test.ts b/src/main/ssh/orcad-remote-gc.test.ts index 044bd974e98..974677d45f5 100644 --- a/src/main/ssh/orcad-remote-gc.test.ts +++ b/src/main/ssh/orcad-remote-gc.test.ts @@ -23,6 +23,8 @@ vi.mock('./ssh-relay-install-lock', () => ({ import { execCommand } from './ssh-relay-deploy-helpers' import { gcOldOrcadVersions } from './orcad-remote-gc' import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { serializeOrcadActivationTransaction } from './orcad-activation-transaction' +import { createOrcadActivationTransaction } from './orcad-activation-transaction-transitions' import { getRemoteHostPlatform } from './ssh-remote-platform' import type { SshConnection } from './ssh-connection' @@ -39,8 +41,22 @@ function scriptHost(options: { listing: string[] liveness?: Record removed: string[] + /** The activation journal: absent by default, raw contents, or a read with no answer. */ + journal?: string | Error + fenceHeld?: boolean }): void { mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('__ORCAD_RECORD_PRESENT__') && command.includes('transaction.json')) { + if (options.journal instanceof Error) { + throw options.journal + } + return options.journal === undefined + ? '__ORCAD_RECORD_ABSENT__\n' + : `__ORCAD_RECORD_PRESENT__\n${options.journal}` + } + if (command.includes('.orcad-activation-transaction') && command.includes('LOCKED')) { + return options.fenceHeld ? 'LOCKED' : 'OPEN' + } if (command.includes('-mindepth 1 -maxdepth 1')) { return options.listing.join('\n') } @@ -184,7 +200,8 @@ describe('orcad GC', () => { ).rejects.toBe(error) expect(removed).toEqual([]) - expect(mockExec).toHaveBeenCalledTimes(4) + // Journal read and fence probe, then the GC pass up to the unconfirmed probe. + expect(mockExec).toHaveBeenCalledTimes(6) expect(mockExec.mock.calls.at(-1)?.[1]).toContain('.orcad-pid') }) @@ -226,4 +243,57 @@ describe('orcad GC', () => { await gcOldOrcadVersions({ ...options, nodeRuntimePins: ['a'.repeat(64)] }) expect(inventories()).toBe(1) }) + + describe('with an activation transaction in flight', () => { + const listing = ['orcad-0.1.0+01d', 'orcad-0.2.0+9ee0', 'orcad-0.3.0+cc0'] + const record = { ...emptyOrcadActivationRecord(), active: '0.3.0+cc0' } + const run = () => + gcOldOrcadVersions({ + conn, + host, + remoteHome: '/home/u', + currentDirAbsPath: '/home/u/.orca-remote/orcad-0.3.0+cc0', + record + }) + + it('keeps every slot a pending journal names', async () => { + const removed: string[] = [] + const before = { ...record, previous: '0.1.0+01d' } + scriptHost({ + listing, + removed, + journal: serializeOrcadActivationTransaction( + createOrcadActivationTransaction({ + transactionId: '7f1c2a7e-6c1b-4a8e-9f0e-0a1b2c3d4e5f', + candidateVersion: '0.2.0+9ee0', + recordBefore: before, + snapshotDirName: 'pre-0.2.0+9ee0-1', + now: new Date(0) + }) + ) + }) + await run() + expect(removed).toEqual([]) + }) + + it.each([ + ['an unreadable journal', { journal: '{"schemaVersion":99}' }], + ['a journal read with no answer', { journal: new Error('lost') }], + ['a held fence without a journal', { fenceHeld: true }] + ])('collects nothing behind %s', async (_name, state) => { + const removed: string[] = [] + vi.spyOn(console, 'warn').mockImplementation(() => {}) + scriptHost({ listing, removed, ...state }) + await run() + expect(removed).toEqual([]) + expect(mockExec.mock.calls.some(([, command]) => command.includes('-mindepth 1'))).toBe(false) + }) + + it('never names a snapshot or rescue copy as a candidate', async () => { + const removed: string[] = [] + scriptHost({ listing: [...listing, 'orcad-state-snapshots'], removed }) + await run() + expect(removed).not.toContain('orcad-state-snapshots') + }) + }) }) diff --git a/src/main/ssh/orcad-remote-gc.ts b/src/main/ssh/orcad-remote-gc.ts index 2f1692201e4..9cf038b1101 100644 --- a/src/main/ssh/orcad-remote-gc.ts +++ b/src/main/ssh/orcad-remote-gc.ts @@ -10,7 +10,8 @@ * * On top of the ownership rule, orcad pins three directories that are idle-looking but * load-bearing: the active version, the rollback target, and whichever version the LIVE - * terminal daemon was forked from. + * terminal daemon was forked from. Every version an in-flight activation journal names is + * pinned too, and an unreadable journal skips the pass entirely. */ import type { SshConnection } from './ssh-connection' import { execCommand } from './ssh-relay-deploy-helpers' @@ -19,12 +20,16 @@ import { ORCAD_INSTALL_MODEL } from './remote-install-model' import { gcOldRemoteInstallVersions } from './ssh-relay-versioned-install' import { orcadGcPinnedDirNames, type OrcadActivationRecord } from './orcad-activation-record' import { - orcadLivenessBlocksGc, - orcadLivenessProbeCommand, - parseOrcadLiveness + ORCAD_NEVER_LAUNCHED, + orcadLivenessAnswerBlocksGc, + orcadLivenessProbeCommand } from './orcad-remote-launch' -import type { RemoteHostPlatform } from './ssh-remote-platform' import { gcRemoteNodeRuntimeStore } from './remote-node-runtime-store-gc' +import { readOrcadGcTransactionPins } from './orcad-gc-transaction-pins' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { orcadRemoteBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_LIVENESS_MANY_MARKER } from './orcad-windows-host-script' export type OrcadGcOptions = { conn: SshConnection @@ -50,6 +55,11 @@ export type OrcadGcOptions = { } export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise { + const transaction = await readOrcadGcTransactionPins(options) + if (transaction.state === 'keep-all') { + console.warn('[orcad-gc] An activation transaction is unreadable or unjournaled; skipping GC.') + return + } await gcOldRemoteInstallVersions( options.conn, ORCAD_INSTALL_MODEL, @@ -57,7 +67,15 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise options.currentDirAbsPath, options.host, { - pinnedDirNames: orcadGcPinnedDirNames(options.record, options.liveDaemonVersion), + pinnedDirNames: [ + ...orcadGcPinnedDirNames(options.record, options.liveDaemonVersion), + ...transaction.dirNames + ], + // Windows screens every candidate in one node.exe; the per-dir probe below rechecks only + // the few that screened dead, under the GC claim. + ...(isWindowsRemoteHost(options.host) + ? { resolveExtraPinnedDirNames: (candidates) => windowsLiveCandidates(options, candidates) } + : {}), isDirLive: async (dir) => { try { const probe = await execCommand( @@ -68,7 +86,7 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise signal: options.signal } ) - return orcadLivenessBlocksGc(parseOrcadLiveness(probe)) + return orcadLivenessAnswerBlocksGc(probe) } catch (error) { if (isUnconfirmedSshCommandTermination(error)) { throw error @@ -88,3 +106,42 @@ export async function gcOldOrcadVersions(options: OrcadGcOptions): Promise }) } } + +/** Candidates that are not proven dead, as pins; null (keep everything) when the host cannot say. */ +async function windowsLiveCandidates( + options: OrcadGcOptions, + candidates: readonly string[] +): Promise { + const dirs = candidates.map((name) => + joinRemotePath(options.host, options.remoteHome, RELAY_REMOTE_DIR, name) + ) + let output: string + try { + output = await execCommand( + options.conn, + orcadWindowsHostOpCommand( + options.host, + orcadRemoteBaseDir(options.host, options.remoteHome), + 'liveness-many', + dirs + ), + { wrapCommand: false, signal: options.signal } + ) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return null + } + const line = output + .split(/\r?\n/u) + .map((candidate) => candidate.trim()) + .find((candidate) => candidate.startsWith(`${ORCAD_WINDOWS_LIVENESS_MANY_MARKER} `)) + const states = line?.slice(ORCAD_WINDOWS_LIVENESS_MANY_MARKER.length + 1).split(',') ?? [] + if (states.length !== candidates.length) { + return null + } + return candidates.filter( + (_name, index) => states[index] !== 'DEAD' && states[index] !== ORCAD_NEVER_LAUNCHED + ) +} diff --git a/src/main/ssh/orcad-remote-host-support.ts b/src/main/ssh/orcad-remote-host-support.ts index 7ef8548ecdf..4517573022f 100644 --- a/src/main/ssh/orcad-remote-host-support.ts +++ b/src/main/ssh/orcad-remote-host-support.ts @@ -3,9 +3,9 @@ * discovered at runtime. * * The install transaction is host-agnostic — it is the relay's, and the relay runs on - * Windows. The launch, liveness and stop path is not: it uses `nohup`, a redirected stdout, - * `kill -0` and `ps`. Emitting a PowerShell-shaped approximation of that would produce a - * deploy that reports success on a host where nothing is running. + * Windows. Every managed-orcad operation runs on Windows through the host script + * (`orcad-windows-host-script.ts`); the guard below remains only on POSIX command builders + * that the Windows paths never call. */ import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' @@ -13,9 +13,8 @@ export class OrcadRemoteLaunchUnsupportedError extends Error { readonly code = 'orcad_remote_launch_unsupported_host' constructor(hostLabel: string) { super( - `Deploying orcad to a ${hostLabel} host is not implemented. The install transaction is ` + - 'host-agnostic, but the launch and readiness path is POSIX-only: it uses nohup, a ' + - 'redirected stdout and `kill -0` liveness. Use the relay for this host.' + `This orcad command does not run on a ${hostLabel} host. The install transaction is ` + + 'host-agnostic, but this step is a POSIX command with a separate Windows path.' ) this.name = 'OrcadRemoteLaunchUnsupportedError' } @@ -27,8 +26,17 @@ export function assertPosixOrcadHost(host: RemoteHostPlatform): void { } } +/** Stdout of the launched candidate: exactly one `orca_server_ready` line, then nothing. */ +export const ORCAD_READINESS_FILENAME = '.orcad-readiness' +/** Stderr, including the bind-exposure line and every supervision message. */ +export const ORCAD_LOG_FILENAME = 'orcad.log' +// Why a cap: the readiness file is candidate-written stdout, and a runaway writer must not be read whole. +export const ORCAD_READINESS_MAX_BYTES = 256 * 1024 + /** PID of the launched orcad, written into its own version dir at launch. */ export const ORCAD_PID_FILENAME = '.orcad-pid' +/** Windows' `.orcad-pid`: `{"pid":N,"creationTimeMs":M|null}`, since a PID alone is no identity there. */ +export const ORCAD_WINDOWS_PROCESS_FILENAME = '.orcad-process.json' /** * A shell function answering whether a PID is a *running* process. diff --git a/src/main/ssh/orcad-remote-install-termination.test.ts b/src/main/ssh/orcad-remote-install-termination.test.ts index 7951c3755ba..2956cfceb78 100644 --- a/src/main/ssh/orcad-remote-install-termination.test.ts +++ b/src/main/ssh/orcad-remote-install-termination.test.ts @@ -19,6 +19,14 @@ import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transf import { installOrcadBundle } from './orcad-remote-install' import { getRemoteHostPlatform } from './ssh-remote-platform' import { decodeRemotePowerShellScript } from './ssh-remote-powershell' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { + initOrcadHeldFenceTokenFile, + ORCAD_HELD_FENCE_TOKENS_FILE_NAME +} from './orcad-held-fence-tokens' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: All connection operations are mocked. const conn = {} as SshConnection @@ -166,3 +174,56 @@ describe.each([ } ) }) + +// BUG-23: the relaunch proves a lock its own quit left only while the token is still recorded. +describe('the install lock token', () => { + const host = getRemoteHostPlatform('linux-x64') + const install = (): Promise => + installOrcadBundle( + { + conn, + host, + localOrcadDir: '/local/orcad', + target: 'linux-x64-glibc', + nodeRuntimeArchive: async () => '/cache/node-archive' + }, + fullVersion, + '/home/u/.orca-remote/orcad-version' + ) + + function bindStore(): () => string { + const dir = mkdtempSync(join(tmpdir(), 'orcad-install-token-')) + initOrcadHeldFenceTokenFile(join(dir, 'orca-data.json')) + return () => { + try { + return readFileSync(join(dir, ORCAD_HELD_FENCE_TOKENS_FILE_NAME), 'utf-8') + } finally { + rmSync(dir, { recursive: true, force: true }) + } + } + } + + function lockToken(): string { + return vi.mocked(acquireInstallLock).mock.calls.at(-1)?.[3]?.owner?.token ?? '' + } + + it('is written into the lock and dropped once the lock is removed', async () => { + const read = bindStore() + await install() + expect(lockToken()).not.toBe('') + expect(read()).not.toContain(lockToken()) + }) + + it('is kept when a quit cut the lock removal short', async () => { + const read = bindStore() + mockUpload.mockRejectedValueOnce(new Error('aborted by quit')) + mockExec.mockImplementation(async (_conn, command) => { + if (command.includes('.install-lock')) { + throw new Error('Not connected') + } + return '' + }) + await expect(install()).rejects.toThrow('aborted by quit') + expect(read()).toContain(lockToken()) + }) +}) diff --git a/src/main/ssh/orcad-remote-install.ts b/src/main/ssh/orcad-remote-install.ts index 017a8d89d85..44feffaef23 100644 --- a/src/main/ssh/orcad-remote-install.ts +++ b/src/main/ssh/orcad-remote-install.ts @@ -1,6 +1,7 @@ +import { randomUUID } from 'node:crypto' import { orcadNodePtyNativeArtifacts, orcadRipgrepArtifact } from '../../shared/orcad-artifacts' import { orcadAgentBrowserNativeName } from '../../shared/orcad-agent-browser-name' -import type { ServerTarget } from '../../shared/node-runtime-pin' +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' import { execCommand } from './ssh-relay-deploy-helpers' import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' @@ -9,6 +10,10 @@ import type { SshConnection } from './ssh-connection' import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' import { ORCAD_INSTALL_MODEL } from './remote-install-model' import { acquireInstallLock } from './ssh-relay-install-lock' +import { ORCAD_FENCE_OWNER_FILENAME } from './orcad-activation-fence-scope' +import { forgetHeldOrcadFence, rememberHeldOrcadFence } from './orcad-held-fence-tokens' +import { exitedOwnLockProof } from './orcad-exited-own-lock' +import { orcadWindowsBaseDir } from './orcad-remote-windows-node' import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' import { abandonInstall, @@ -22,7 +27,7 @@ export async function installOrcadBundle( conn: SshConnection host: RemoteHostPlatform localOrcadDir: string - target: ServerTarget + target: NodeRuntimeTarget /** The locally verified pinned Node archive, fetched only when the host lacks the runtime. */ nodeRuntimeArchive: () => Promise signal?: AbortSignal @@ -46,7 +51,25 @@ export async function installOrcadBundle( ) { return } - await acquireInstallLock(options.conn, remoteDir, options.host, { signal: options.signal }) + // Its token lets a relaunch prove a lock its own quit left mid-upload (BUG-23). + const token = randomUUID() + rememberHeldOrcadFence(token) + try { + await acquireInstallLock(options.conn, remoteDir, options.host, { + signal: options.signal, + owner: { fileName: ORCAD_FENCE_OWNER_FILENAME, token }, + // Nothing keeps writing once its client exited: uploads are SFTP and chmod is immediate. + exitedOwner: exitedOwnLockProof(options, { + baseDir: orcadWindowsBaseDir(options.host, remoteDir), + guardsStateMutation: false + }) + }) + } catch (error) { + if (!isUnconfirmedSshCommandTermination(error)) { + forgetHeldOrcadFence(token) + } + throw error + } let preserveInstallLock = false try { // Re-probe under the lock: a sibling deploy may have finished while we waited. @@ -86,7 +109,10 @@ export async function installOrcadBundle( throw error } finally { if (!preserveInstallLock) { - await abandonInstall(options.conn, remoteDir, options.host) + // A removal a quit cut short keeps the token, so the relaunch can prove the lock its own. + if (await abandonInstall(options.conn, remoteDir, options.host)) { + forgetHeldOrcadFence(token) + } } } } @@ -94,7 +120,7 @@ export async function installOrcadBundle( function executablePermissionsCommand( host: RemoteHostPlatform, directory: string, - target: ServerTarget + target: NodeRuntimeTarget ): string { const required = [orcadRipgrepArtifact(target), ...orcadNodePtyNativeArtifacts(target)] .filter((artifact) => /\/(?:rg|spawn-helper)$/.test(artifact)) diff --git a/src/main/ssh/orcad-remote-launch-file-modes.test.ts b/src/main/ssh/orcad-remote-launch-file-modes.test.ts new file mode 100644 index 00000000000..836f4f661ac --- /dev/null +++ b/src/main/ssh/orcad-remote-launch-file-modes.test.ts @@ -0,0 +1,105 @@ +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { + ORCAD_LOG_FILENAME, + ORCAD_PID_FILENAME, + ORCAD_READINESS_FILENAME, + orcadLaunchCommand +} from './orcad-remote-launch' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +// The readiness file carries the pairing offer's device token: under a login umask of 022 it must +// still come out owner-only, and so must the slot and `~/.orca-remote` that hold it. + +const posix = getRemoteHostPlatform('linux-x64') +const READY_LINE = '{"type":"orca_server_ready","pairing":{"url":"orca://pair#token"}}' + +let home: string +let baseDir: string +let slotDir: string + +function mode(path: string): number { + return statSync(path).mode & 0o777 +} + +async function launchUnderLoginUmask(): Promise { + const command = orcadLaunchCommand(posix, { + remoteInstallDir: slotDir, + // No runtime marker in the slot, so this legacy path runs the stub entry below. + nodePath: '/bin/sh', + fullVersion: '0.2.0+bb01', + userDataDir: join(home, '.orca'), + bindHost: '127.0.0.1', + port: 7777, + activationRoot: join(baseDir, '.orcad-activation-transaction') + }) + await runProcess({ program: '/bin/sh', args: ['-c', `umask 022; ${command}`], timeoutMs: 10_000 }) + const readiness = join(slotDir, ORCAD_READINESS_FILENAME) + const deadline = Date.now() + 5_000 + while (!readFileSync(readiness, 'utf8').includes(READY_LINE) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)) + } +} + +describe.skipIf(process.platform === 'win32')('orcad launch file modes', () => { + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'orcad-launch-modes-')) + baseDir = join(home, '.orca-remote') + slotDir = join(baseDir, 'orcad-0.2.0+bb01') + mkdirSync(slotDir, { recursive: true, mode: 0o755 }) + chmodSync(baseDir, 0o755) + chmodSync(slotDir, 0o755) + writeFileSync(join(slotDir, 'orcad.js'), `printf '%s\\n' '${READY_LINE}'\n`) + }) + + afterEach(() => { + rmSync(home, { recursive: true, force: true }) + }) + + it('creates the readiness, pid and log files owner-only', async () => { + await launchUnderLoginUmask() + + expect(readFileSync(join(slotDir, ORCAD_READINESS_FILENAME), 'utf8')).toContain(READY_LINE) + expect(mode(join(slotDir, ORCAD_READINESS_FILENAME))).toBe(0o600) + expect(mode(join(slotDir, ORCAD_PID_FILENAME))).toBe(0o600) + expect(mode(join(slotDir, ORCAD_LOG_FILENAME))).toBe(0o600) + }) + + it('tightens files and directories an earlier build left readable', async () => { + for (const name of [ORCAD_READINESS_FILENAME, ORCAD_PID_FILENAME, ORCAD_LOG_FILENAME]) { + writeFileSync(join(slotDir, name), 'old', { mode: 0o644 }) + chmodSync(join(slotDir, name), 0o644) + } + + await launchUnderLoginUmask() + + expect(mode(join(slotDir, ORCAD_READINESS_FILENAME))).toBe(0o600) + expect(mode(join(slotDir, ORCAD_PID_FILENAME))).toBe(0o600) + expect(mode(join(slotDir, ORCAD_LOG_FILENAME))).toBe(0o600) + expect(mode(slotDir)).toBe(0o700) + expect(mode(baseDir)).toBe(0o700) + }) + + it('leaves a parent that is not ~/.orca-remote alone', async () => { + slotDir = join(home, 'custom-slot') + mkdirSync(slotDir, { mode: 0o755 }) + chmodSync(home, 0o755) + writeFileSync(join(slotDir, 'orcad.js'), `printf '%s\\n' '${READY_LINE}'\n`) + + await launchUnderLoginUmask() + + expect(mode(home)).toBe(0o755) + expect(mode(slotDir)).toBe(0o700) + }) +}) diff --git a/src/main/ssh/orcad-remote-launch-windows.ts b/src/main/ssh/orcad-remote-launch-windows.ts new file mode 100644 index 00000000000..ba255e8e321 --- /dev/null +++ b/src/main/ssh/orcad-remote-launch-windows.ts @@ -0,0 +1,122 @@ +/** + * Starting a candidate orcad on a Windows SSH host. + * + * Win32-OpenSSH kills the session's job when the session ends, so orcad must leave that job. + * `orcad.js --windows-breakaway-launch` does it the way the relay does: one CreateProcessW with + * CREATE_BREAKAWAY_FROM_JOB through the slot's staged process-tree addon. Unlike the relay there + * is no WMI fallback: Win32_Process.Create is EDR-scored remote execution and refused to a + * standard user's network logon, so a host that cannot break away refuses the launch. + * + * Two execs, neither through PowerShell: the host script resolves the runtime the slot's marker + * names (and drops a stale stop request), then that node.exe runs the launcher with plain argv. + * The launcher records the PID with its creation time; a PID alone is not an identity on Windows. + */ +import { + ORCAD_WINDOWS_BREAKAWAY_CONTRACT, + parseWindowsBreakawayLaunchReport, + WINDOWS_BREAKAWAY_ENV_FLAG, + WINDOWS_BREAKAWAY_LAUNCH_FLAG, + WINDOWS_BREAKAWAY_PROCESS_FILE_FLAG, + WINDOWS_BREAKAWAY_STDERR_FLAG, + WINDOWS_BREAKAWAY_STDERR_KEEP_PREVIOUS_FLAG, + WINDOWS_BREAKAWAY_STDOUT_FLAG +} from '../../shared/windows-breakaway-launch' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + orcadWindowsNodeCommandLine, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_RUNTIME_MARKER } from './orcad-windows-host-script' +import { + ORCAD_LOG_FILENAME, + ORCAD_READINESS_FILENAME, + ORCAD_WINDOWS_PROCESS_FILENAME +} from './orcad-remote-host-support' +import { orcadManagedLaunchEnv, type OrcadLaunchSpec } from './orcad-remote-launch' + +export class OrcadWindowsLaunchRefusedError extends Error { + readonly code = 'orcad_windows_launch_refused' + constructor(reason: string) { + super( + `The Windows host refused to start orcad outside the SSH session (${reason}). orcad needs ` + + 'a job that allows breakaway and a slot with the process-tree launcher; it never falls ' + + 'back to WMI.' + ) + this.name = 'OrcadWindowsLaunchRefusedError' + } +} + +/** Resolves the slot's node.exe and clears a stop request the previous process never consumed. */ +export function windowsOrcadLaunchRuntimeCommand( + host: RemoteHostPlatform, + slotDir: string +): string { + return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, slotDir), 'slot-runtime', [ + slotDir, + 'clear-stop-request' + ]) +} + +export function readWindowsOrcadSlotRuntime(output: string): string { + const runtime = readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_RUNTIME_MARKER) + if (!runtime) { + throw new Error('The Windows host did not name the runtime this orcad slot needs.') + } + return runtime +} + +export function windowsOrcadLaunchCommand( + host: RemoteHostPlatform, + spec: OrcadLaunchSpec, + slotRuntime: string +): string { + const dir = spec.remoteInstallDir + return orcadWindowsNodeCommandLine(slotRuntime, [ + joinRemotePath(host, dir, 'orcad.js'), + WINDOWS_BREAKAWAY_LAUNCH_FLAG, + // The addon creates both with CREATE_ALWAYS, so a previous run's readiness line is gone + // before the launcher returns. + WINDOWS_BREAKAWAY_STDOUT_FLAG, + joinRemotePath(host, dir, ORCAD_READINESS_FILENAME), + WINDOWS_BREAKAWAY_STDERR_FLAG, + joinRemotePath(host, dir, ORCAD_LOG_FILENAME), + // POSIX appends orcad.log; keep the last run's (a crash cause) across a restart here too. + WINDOWS_BREAKAWAY_STDERR_KEEP_PREVIOUS_FLAG, + WINDOWS_BREAKAWAY_PROCESS_FILE_FLAG, + joinRemotePath(host, dir, ORCAD_WINDOWS_PROCESS_FILENAME), + WINDOWS_BREAKAWAY_ENV_FLAG, + `ORCA_VERSION=${spec.fullVersion}`, + WINDOWS_BREAKAWAY_ENV_FLAG, + `ORCA_USER_DATA=${spec.userDataDir}`, + ...orcadManagedLaunchEnv(spec).flatMap(([name, value]) => [ + WINDOWS_BREAKAWAY_ENV_FLAG, + `${name}=${value}` + ]), + ORCAD_WINDOWS_BREAKAWAY_CONTRACT.argsFlag, + '--json', + '--bind', + spec.bindHost, + '--port', + String(spec.port) + ]) +} + +/** The launched PID, or a refusal (no breakaway route) or failure the deploy must surface. */ +export function readWindowsOrcadLaunchReport(output: string): number { + const report = parseWindowsBreakawayLaunchReport(ORCAD_WINDOWS_BREAKAWAY_CONTRACT, output) + if (report?.method === 'breakaway') { + return report.pid + } + if (report?.method === 'unavailable') { + throw new OrcadWindowsLaunchRefusedError( + report.step ? `${report.reason} at ${report.step}` : report.reason + ) + } + const detail = + report?.method === 'failed' + ? `${report.reason}${report.step ? ` at ${report.step}` : ''} (code ${String(report.code ?? 0)})` + : 'no launch report' + throw new Error(`orcad's Windows launcher did not start the candidate: ${detail}.`) +} diff --git a/src/main/ssh/orcad-remote-launch.test.ts b/src/main/ssh/orcad-remote-launch.test.ts index 0be9ddc76c1..2c79fbe2dc7 100644 --- a/src/main/ssh/orcad-remote-launch.test.ts +++ b/src/main/ssh/orcad-remote-launch.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { ORCAD_MANAGED_ACTIVATION_ROOT_ENV } from '../../shared/orcad-idle-exit' import { ORCAD_READINESS_FILENAME, @@ -25,7 +26,8 @@ const SPEC = { fullVersion: '0.2.0+bb01', userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', - port: 7777 + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' } const READY_LINE = JSON.stringify({ @@ -58,6 +60,14 @@ describe('orcadLaunchCommand', () => { expect(command).toContain(`ORCA_USER_DATA='${SPEC.userDataDir}'`) }) + it('names the activation fence on every launch, which enables idle exit', () => { + const command = orcadLaunchCommand(posix, SPEC) + expect(command).toContain(`${ORCAD_MANAGED_ACTIVATION_ROOT_ENV}='${SPEC.activationRoot}'`) + expect(command.indexOf(ORCAD_MANAGED_ACTIVATION_ROOT_ENV)).toBeLessThan( + command.indexOf('nohup') + ) + }) + it('declares the Windows refusal instead of emitting a command that cannot work', () => { expect(() => orcadLaunchCommand(windows, SPEC)).toThrow(OrcadRemoteLaunchUnsupportedError) }) @@ -117,7 +127,8 @@ describe('stopping a running orcad', () => { ['STILL_RUNNING', 'still-running', false], ['SIGNAL_FAILED', 'signal-failed', false], ['UNKNOWN', 'unknown', false], - ['', 'unknown', false] + ['SIGNALED\nUNKNOWN', 'unconfirmed', false], + ['', 'unconfirmed', false] ])('parses %s and frees the host = %s', (output, expected, frees) => { expect(parseOrcadStopOutcome(output)).toBe(expected) expect(orcadStopFreedTheHost(parseOrcadStopOutcome(output))).toBe(frees) diff --git a/src/main/ssh/orcad-remote-launch.ts b/src/main/ssh/orcad-remote-launch.ts index d6c9640ad33..c8468fffbb7 100644 --- a/src/main/ssh/orcad-remote-launch.ts +++ b/src/main/ssh/orcad-remote-launch.ts @@ -13,20 +13,39 @@ * still owns a running service. */ import { shellEscape } from './ssh-connection-utils' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + isWindowsRemoteHost, + joinRemotePath, + remoteBasename, + remoteDirname, + type RemoteHostPlatform +} from './ssh-remote-platform' +import { RELAY_REMOTE_DIR } from './relay-protocol' import { assertPosixOrcadHost as assertPosixHost, + ORCAD_LOG_FILENAME, ORCAD_PID_FILENAME, + ORCAD_READINESS_FILENAME, + ORCAD_READINESS_MAX_BYTES, posixProcessAliveShellFunction } from './orcad-remote-host-support' import type { ServeReadiness } from '../server/serve-readiness' import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' +import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' +import { windowsOrcadLivenessProbeCommand } from './orcad-remote-liveness-windows' +import { + ORCAD_E2E_IDLE_TIMEOUT_ENV, + ORCAD_MANAGED_ACTIVATION_ROOT_ENV, + readOrcadE2EIdleTimeoutMs +} from '../../shared/orcad-idle-exit' -/** Stdout of the launched candidate: exactly one `orca_server_ready` line, then nothing. */ -export const ORCAD_READINESS_FILENAME = '.orcad-readiness' -/** Stderr, including the bind-exposure line and every supervision message. */ -export const ORCAD_LOG_FILENAME = 'orcad.log' -export { ORCAD_PID_FILENAME, OrcadRemoteLaunchUnsupportedError } from './orcad-remote-host-support' +export { + ORCAD_LOG_FILENAME, + ORCAD_PID_FILENAME, + ORCAD_READINESS_FILENAME, + ORCAD_READINESS_MAX_BYTES, + OrcadRemoteLaunchUnsupportedError +} from './orcad-remote-host-support' export type OrcadLaunchSpec = { remoteInstallDir: string @@ -37,6 +56,22 @@ export type OrcadLaunchSpec = { /** Loopback by default; the client reaches it through an SSH local port-forward. */ bindHost: string port: number + /** The host's activation fence. Every SSH launch is client-managed, so every one may idle out. */ + activationRoot: string +} + +/** Env a managed launch adds; an older orcad ignores both. */ +export function orcadManagedLaunchEnv( + spec: OrcadLaunchSpec, + env: NodeJS.ProcessEnv = process.env +): [string, string][] { + const e2eTimeout = readOrcadE2EIdleTimeoutMs(env) + return [ + [ORCAD_MANAGED_ACTIVATION_ROOT_ENV, spec.activationRoot], + ...(e2eTimeout === null + ? [] + : [[ORCAD_E2E_IDLE_TIMEOUT_ENV, String(e2eTimeout)] satisfies [string, string]]) + ] } /** @@ -55,15 +90,31 @@ export function orcadLaunchCommand(host: RemoteHostPlatform, spec: OrcadLaunchSp const log = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_LOG_FILENAME)) const pidFile = shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_PID_FILENAME)) const entry = shellEscape(joinRemotePath(host, spec.remoteInstallDir, 'orcad.js')) + const baseDir = remoteDirname(spec.remoteInstallDir.replace(/\/+$/u, ''), host) + // Only `~/.orca-remote` itself; never tighten an unrelated parent a custom slot path names. + const privateDirs = [ + ...(remoteBasename(baseDir, host) === RELAY_REMOTE_DIR ? [baseDir] : []), + spec.remoteInstallDir + ] return [ + // Why first: the readiness file carries the pairing offer's device token, so every file this + // launch creates must be owner-only from birth, not only the ones after the exec. + 'umask 077 &&', + // Best effort: directories and files an earlier build left 0755/0644 keep their old modes. + `{ chmod 700 ${privateDirs.map(shellEscape).join(' ')} 2>/dev/null || :; } &&`, `cd ${dir} &&`, `${selectOrcadSlotRuntimeCommand(host, spec.remoteInstallDir, spec.nodePath)} &&`, // Why truncate: a re-launch into a dir that already holds a previous readiness line would // otherwise let the deploy activate on the OLD process's health payload. `: > ${readiness} &&`, - 'umask 077 &&', + // Required, not best effort: a truncating redirect keeps an earlier build's 0644 mode. + `chmod 600 ${readiness} &&`, + `{ chmod 600 ${pidFile} ${log} 2>/dev/null || :; } &&`, + // A stop request the previous process never consumed must not stop this one. + `rm -f ${shellEscape(joinRemotePath(host, spec.remoteInstallDir, ORCAD_STOP_REQUEST_FILENAME))} &&`, `ORCA_VERSION=${shellEscape(spec.fullVersion)}`, `ORCA_USER_DATA=${shellEscape(spec.userDataDir)}`, + ...orcadManagedLaunchEnv(spec).map(([name, value]) => `${name}=${shellEscape(value)}`), // Keep $! equal to the runtime PID rather than a waiting shell's PID. `exec nohup "$orcad_runtime" ${entry}`, `--json --bind ${shellEscape(spec.bindHost)} --port ${String(spec.port)}`, @@ -78,7 +129,8 @@ export function readOrcadReadinessCommand( ): string { assertPosixHost(host) const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) - return `cat ${readiness} 2>/dev/null || true` + // One byte over the cap is enough to tell an oversized payload from a full one. + return `head -c ${ORCAD_READINESS_MAX_BYTES + 1} ${readiness} 2>/dev/null || true` } /** @@ -86,25 +138,32 @@ export function readOrcadReadinessCommand( * * Answers `LIVE`, `DEAD`, or `UNKNOWN`. `UNKNOWN` covers a missing or unparseable PID file * and a `kill -0` that failed for a reason other than "no such process" — a permission - * error means someone else's process holds that PID, which is not evidence of death. + * error means someone else's process holds that PID, which is evidence of neither. A slot with + * neither a PID file nor a readiness file (which a launch creates first) answers + * `NEVER_LAUNCHED`, which only GC reads apart from `UNKNOWN`. */ export function orcadLivenessProbeCommand( host: RemoteHostPlatform, remoteInstallDir: string ): string { - assertPosixHost(host) + if (isWindowsRemoteHost(host)) { + return windowsOrcadLivenessProbeCommand(host, remoteInstallDir) + } const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) + const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + const entry = shellEscape(joinRemotePath(host, remoteInstallDir, 'orcad.js')) return [ - posixProcessAliveShellFunction(), + posixProcessAliveShellFunction({ refuseUnverifiable: true }), + // A PID is no identity once reused: a process whose command line does not run this slot's + // orcad.js is not it. No `ps` answer leaves the plain liveness check to decide. + `orcad_entry=${entry};`, + 'orcad_reused() { args=$(ps -o args= -p "$1" 2>/dev/null) && [ -n "$args" ] && ' + + 'case "$args" in *"$orcad_entry"*) return 1;; *) return 0;; esac; };', `pid=$(cat ${pidFile} 2>/dev/null);`, 'case "$pid" in', - '"" ) echo UNKNOWN;;', + `"" ) if [ -e ${pidFile} ] || [ -e ${readiness} ]; then echo UNKNOWN; else echo ${ORCAD_NEVER_LAUNCHED}; fi;;`, '*[!0-9]* ) echo UNKNOWN;;', - // Why EPERM is LIVE and not DEAD: a permission error means some process holds that PID, - // and deleting a tree because we could not signal its owner is the wrong direction. - '* ) if orcad_alive "$pid"; then echo LIVE;', - 'elif kill -0 "$pid" 2>&1 | grep -qi "not permitted"; then echo LIVE;', - 'else echo DEAD; fi;;', + '* ) if orcad_reused "$pid"; then echo DEAD; elif orcad_alive "$pid"; then echo LIVE; else echo DEAD; fi;;', 'esac' ].join(' ') } @@ -116,6 +175,15 @@ export function parseOrcadLiveness(output: string): OrcadLiveness { return value === 'LIVE' || value === 'DEAD' ? value : 'UNKNOWN' } +export const ORCAD_NEVER_LAUNCHED = 'NEVER_LAUNCHED' + +/** GC's reading of a liveness answer: a slot that never launched holds nothing and is removable. */ +export function orcadLivenessAnswerBlocksGc(output: string): boolean { + return output.trim().split('\n').pop()?.trim() === ORCAD_NEVER_LAUNCHED + ? false + : orcadLivenessBlocksGc(parseOrcadLiveness(output)) +} + /** True when GC must leave this directory alone. Inconclusive counts as in use. */ export function orcadLivenessBlocksGc(liveness: OrcadLiveness): boolean { return liveness !== 'DEAD' @@ -134,19 +202,23 @@ export type OrcadReadinessParse = * not `malformed` — reporting a parse failure for a race would fail deploys that were fine. */ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { + if (Buffer.byteLength(raw, 'utf8') > ORCAD_READINESS_MAX_BYTES) { + return { state: 'malformed', reason: 'readiness payload exceeds the 256 KiB limit' } + } const lines = raw.split('\n') - let sawCandidate = false - for (const line of lines) { + for (const [index, line] of lines.entries()) { const trimmed = line.trim() if (!trimmed.startsWith('{')) { continue } - sawCandidate = true let parsed: unknown try { parsed = JSON.parse(trimmed) } catch { - continue + // Only the unterminated last line can still be mid-write; a finished bad line will stay bad. + return index === lines.length - 1 + ? { state: 'pending' } + : { state: 'malformed', reason: 'readiness line is not valid JSON' } } if (typeof parsed !== 'object' || parsed === null) { continue @@ -160,7 +232,7 @@ export function parseOrcadReadinessOutput(raw: string): OrcadReadinessParse { } return { state: 'ready', readiness: toServeReadiness(payload as Record) } } - return sawCandidate ? { state: 'pending' } : { state: 'pending' } + return { state: 'pending' } } function toServeReadiness(payload: Record): ServeReadiness { diff --git a/src/main/ssh/orcad-remote-lifecycle-windows.test.ts b/src/main/ssh/orcad-remote-lifecycle-windows.test.ts new file mode 100644 index 00000000000..23f6624b831 --- /dev/null +++ b/src/main/ssh/orcad-remote-lifecycle-windows.test.ts @@ -0,0 +1,255 @@ +import type * as TerminalBarrier from './orcad-rollback-terminal-barrier' +/** + * The deploy and rollback drivers end to end against a Windows host whose every host op is answered by a + * fake: no POSIX command, no `-EncodedCommand` and no PowerShell hop on the orcad path. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as RecordFile from './orcad-remote-record-file' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-relay-install-lock', () => ({ + acquireInstallLock: vi.fn().mockResolvedValue(undefined), + RELAY_INSTALL_LOCK_NAME: '.install-lock' +})) +vi.mock('./ssh-relay-install-transfers', () => ({ + uploadRelayDirectory: vi.fn().mockResolvedValue(undefined), + writeRelayFile: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) +// The managed stop's Windows commands are covered by orcad-remote-windows-decommission.test.ts. +vi.mock('./orcad-rollback-terminal-barrier', async (importOriginal) => ({ + ...(await importOriginal()), + readOrcadRollbackBarrierTarget: async (_options: unknown, version: string) => ({ + state: 'ready', + context: { + version, + runtimeId: 'r1', + instance: { pid: 4242, startedAtMs: 1, nonce: 'n', lockPath: 'C:/l' } + } + }), + stopIncumbentBehindTerminalBarrier: async () => ({ state: 'retired' }) +})) +vi.mock('./orcad-remote-node-runtime', () => ({ + ensureRemoteOrcadNodeRuntime: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + readLocalFullVersion: () => '0.2.0+bb0100000000', + isRemoteInstallComplete: vi.fn().mockResolvedValue(false), + finalizeInstall: vi.fn().mockResolvedValue(undefined), + abandonInstall: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { deployOrcad, type OrcadDeployOptions } from './orcad-remote-deploy' +import { rollbackOrcad } from './orcad-remote-rollback' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { NODE_RUNTIME_PIN } from '../../shared/node-runtime-pin' + +const mockExec = vi.mocked(execCommand) +const host = getRemoteHostPlatform('win32-x64') +const VERSION = '0.2.0+bb0100000000' +const SLOT_NODE = 'C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe' + +const encoded = (marker: string, value: string): string => + `${marker} ${Buffer.from(value).toString('base64')}\r\n` + +const TARGET = '0.1.0+aa01' + +function readyLine(version = VERSION): string { + return `${JSON.stringify({ + type: 'orca_server_ready', + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash: 'abc123def4567890', + buildVersion: version, + nodeVersion: NODE_RUNTIME_PIN.version, + nodeAbi: '137', + platform: 'win32', + arch: 'x64', + pid: 4242, + stopRequests: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: version, + entryPath: 'C:/x/daemon-entry.js', + protocolVersion: 3, + selfTest: { ok: true, coverage: 'handshake', verdict: 'healthy', durationMs: 5 } + } + } + })}\n` +} + +function scriptWindowsHost(log: string[], activeRecord: string | null = null): void { + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + log.push(text) + const op = /\.js (?:--fence \S+ \S+ )?([a-z-]+)(?: |$)/u.exec(text)?.[1] ?? '' + switch (op) { + case 'record-read': + return activeRecord && text.includes('orcad-active.json') + ? encoded('__ORCAD_RECORD_PRESENT__', activeRecord) + : '__ORCAD_RECORD_ABSENT__\r\n' + case 'build-hash': + return '__ORCAD_BUILD_HASH__ abc123def4567890\r\n' + case 'owner-admission': + return 'CLEAR' + case 'slot-runtime': + return encoded('__ORCAD_RUNTIME__', SLOT_NODE) + case 'readiness-wait': + return encoded('__ORCAD_READINESS__', readyLine(text.includes(TARGET) ? TARGET : VERSION)) + case 'stop': + return 'STOPPED' + case 'snapshot-probe': + return 'PRESENT' + case 'snapshot-restore': + return 'RESTORED' + case 'state-newest-mtime': + return 'UNKNOWN' + case 'snapshot-capture': + return 'CAPTURED' + case 'remove-file': + case 'remove-tree': + return '' + case 'fence-check': + return 'OK' + case 'fence-release': + return 'RELEASED' + default: + break + } + if (text.includes('--orcad-profile-state-preflight')) { + return JSON.stringify({ + type: 'orca_profile_state_ready', + nonce: text.match(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/)?.[0], + runtime: 'node', + runtimeVersion: NODE_RUNTIME_PIN.version, + sqliteVersion: '3.51.0', + artifactVersion: VERSION, + revision: 1 + }) + } + if (text.includes('--windows-breakaway-launch')) { + return 'ORCA_ORCAD_LAUNCH {"method":"breakaway","pid":4242,"inJob":false}\r\n' + } + // The relay's shared install fence is the one pre-existing PowerShell site left on this path. + if (text.startsWith('powershell.exe ')) { + return 'OPEN' + } + throw new Error(`unexpected Windows command: ${text}`) + }) +} + +function options(): OrcadDeployOptions { + return { + conn: Object.assign(Object.create(null), { writeFile: vi.fn().mockResolvedValue(undefined) }), + host, + remoteHome: 'C:/Users/u', + localOrcadDir: '/local/out/orcad', + target: 'win32-x64', + nodePath: 'C:/host/node.exe', + userDataDir: 'C:/Users/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, + readinessTimeoutMs: 1_000, + sleep: async () => {}, + now: () => new Date('2026-10-02T00:00:00.000Z') + } +} + +beforeEach(() => { + mockExec.mockReset() + vi.mocked(writeAtomicOrcadRemoteRecord).mockClear() +}) + +describe('deployOrcad on a Windows host', () => { + it('activates a first install through node.exe host ops only', async () => { + const log: string[] = [] + scriptWindowsHost(log) + const result = await deployOrcad(options()) + expect(result).toMatchObject({ outcome: 'installed-and-activated', fullVersion: VERSION }) + const orcadOps = log.filter((command) => !command.startsWith('powershell.exe ')) + expect(orcadOps.length).toBeGreaterThan(0) + for (const command of orcadOps) { + expect(command).not.toMatch(/EncodedCommand|nohup|kill |head -c|tar |\bsh -c\b/u) + } + const ops = orcadOps.map( + (command) => /\.js (?:--fence \S+ \S+ )?([a-z-]+)/u.exec(command)?.[1] ?? command + ) + expect(ops).toContain('owner-admission') + expect(ops).toContain('snapshot-capture') + expect(ops.indexOf('slot-runtime')).toBeLessThan( + ops.findIndex((entry) => entry.includes('--windows-breakaway-launch')) + ) + expect(ops).toContain('readiness-wait') + const record = vi + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find(([, path]) => path.endsWith('orcad-active.json')) + expect(JSON.parse(record?.[2] ?? '{}')).toMatchObject({ active: VERSION }) + }) + + it('rolls back by managed stop, directory snapshot and node.exe launch', async () => { + const log: string[] = [] + const record = { + ...emptyOrcadActivationRecord(), + active: VERSION, + previous: TARGET, + activatedAt: '2026-10-01T00:00:00.000Z', + snapshot: { + dirName: `pre-${VERSION}-1000`, + takenBeforeVersion: VERSION, + readableByVersion: TARGET, + takenAt: '2026-10-01T00:00:00.000Z' + } + } + scriptWindowsHost(log, JSON.stringify(record)) + const { localOrcadDir: _dir, target: _target, force: _force, ...base } = options() + const result = await rollbackOrcad({ + ...base, + record, + targetBuildHash: 'abc123def4567890', + targetDaemonProtocol: { protocolVersion: 3, previousProtocolVersions: [1, 2] } + }) + expect(result).toMatchObject({ outcome: 'rolled-back', target: TARGET }) + const ops = log.map( + (command) => /\.js (?:--fence \S+ \S+ )?([a-z-]+)/u.exec(command)?.[1] ?? command + ) + // The target starts only after its state is back. + expect(ops).toEqual([ + 'record-read', + 'snapshot-probe', + 'state-newest-mtime', + 'build-hash', + 'snapshot-capture', + 'snapshot-restore', + 'slot-runtime', + // A command the host script does not run is fence-checked by one op just before it. + 'fence-check', + '--windows-breakaway-launch', + 'readiness-wait', + 'record-read', + 'fence-release' + ]) + for (const command of log) { + expect(command).not.toMatch(/EncodedCommand|kill |tar |nohup/u) + } + }) +}) diff --git a/src/main/ssh/orcad-remote-liveness-windows.ts b/src/main/ssh/orcad-remote-liveness-windows.ts new file mode 100644 index 00000000000..b7b42d5d19d --- /dev/null +++ b/src/main/ssh/orcad-remote-liveness-windows.ts @@ -0,0 +1,21 @@ +/** + * Is the orcad a Windows slot launched still running? LIVE / DEAD / UNKNOWN, as on POSIX. + * + * A PID is not an identity on Windows, so the launcher's record pairs it with the process + * creation time and liveness needs both: the PID must be running (libuv's kill(pid, 0), which + * opens the process and reads its exit code) and the slot's process-tree addon must report the + * same creation time. A running PID with another creation time is a reused PID, so the recorded + * process is gone. Anything the host cannot answer is UNKNOWN, never DEAD. The rules live in the + * host script's `liveness` op. + */ +import { orcadWindowsBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export function windowsOrcadLivenessProbeCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, remoteInstallDir), 'liveness', [ + remoteInstallDir + ]) +} diff --git a/src/main/ssh/orcad-remote-log-tail.test.ts b/src/main/ssh/orcad-remote-log-tail.test.ts new file mode 100644 index 00000000000..61b8fa3f4ac --- /dev/null +++ b/src/main/ssh/orcad-remote-log-tail.test.ts @@ -0,0 +1,111 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { + ORCAD_LOG_TAIL_LINES, + ORCAD_LOG_TAIL_MAX_BYTES, + orcadLogTailCommand, + parseOrcadLogTail, + withOrcadLogTail +} from './orcad-remote-log-tail' +import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const execCommand = vi.hoisted(() => vi.fn()) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand })) + +const linux = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const SLOT = '/home/me/.orca-remote/orcad-1.0.0' +const WINDOWS_SLOT = 'C:\\Users\\me\\.orca-remote\\orcad-1.0.0' +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked; the connection is never used. +const conn = {} as never + +let dir = '' +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'orcad-log-tail-')) + execCommand.mockReset() +}) +afterEach(() => rmSync(dir, { recursive: true, force: true })) + +function numberedLines(count: number): string { + return Array.from({ length: count }, (_, index) => `line ${index + 1}`).join('\n') +} + +describe('orcad.log tail', () => { + it('keeps only the last lines and bytes, redacting secrets', () => { + const tail = parseOrcadLogTail(linux, `${numberedLines(100)}\ntoken=abc123secret\n`) + const lines = (tail ?? '').split('\n') + expect(lines).toHaveLength(ORCAD_LOG_TAIL_LINES) + expect(lines[0]).toBe('line 62') + expect(tail).not.toContain('abc123secret') + expect(tail).toContain('[redacted') + + const huge = parseOrcadLogTail(linux, 'x'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 4)) + expect(Buffer.byteLength(huge ?? '', 'utf8')).toBeLessThanOrEqual(ORCAD_LOG_TAIL_MAX_BYTES) + expect(parseOrcadLogTail(linux, ' \n')).toBeNull() + }) + + it('reads the log on Windows through the node.exe host script, never an encoded command', () => { + const command = orcadLogTailCommand(windows, WINDOWS_SLOT) + expect(command).toContain('node.exe') + expect(command).toContain(' log-tail ') + expect(command).not.toMatch(/EncodedCommand/iu) + }) + + it.skipIf(process.platform === 'win32')( + 'runs the POSIX command against a real file', + async () => { + const slot = join(dir, 'slot') + mkdirSync(slot) + writeFileSync(join(slot, 'orcad.log'), `${numberedLines(60)}\n`) + const run = await runProcess({ + program: '/bin/sh', + args: ['-c', orcadLogTailCommand(linux, slot)], + timeoutMs: 5_000 + }) + expect(parseOrcadLogTail(linux, run.stdout)?.split('\n').at(-1)).toBe('line 60') + const missing = await runProcess({ + program: '/bin/sh', + args: ['-c', orcadLogTailCommand(linux, join(dir, 'absent'))], + timeoutMs: 5_000 + }) + expect(parseOrcadLogTail(linux, missing.stdout)).toBeNull() + } + ) + + it('reads the end of the log through the real Windows host script op', async () => { + const script = join(dir, 'host-script.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const log = join(dir, 'orcad.log') + writeFileSync(log, `${'y'.repeat(ORCAD_LOG_TAIL_MAX_BYTES * 2)}\nfatal: bind failed\n`) + const run = await runProcess({ + program: process.execPath, + args: [script, 'log-tail', log, String(ORCAD_LOG_TAIL_MAX_BYTES)], + timeoutMs: 15_000 + }) + const tail = parseOrcadLogTail(windows, run.stdout) + expect(tail?.endsWith('fatal: bind failed')).toBe(true) + const absent = await runProcess({ + program: process.execPath, + args: [script, 'log-tail', join(dir, 'absent.log'), '100'], + timeoutMs: 15_000 + }) + expect(parseOrcadLogTail(windows, absent.stdout)).toBeNull() + }) + + it('appends the tail to a failure message, and leaves it alone when the host cannot answer', async () => { + execCommand.mockResolvedValueOnce('starting\nError: EADDRINUSE\n') + await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe( + 'Launch failed.\nLast lines of orcad.log:\nstarting\nError: EADDRINUSE' + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + execCommand.mockRejectedValueOnce(new Error('Connection lost')) + await expect(withOrcadLogTail({ conn, host: linux }, SLOT, 'Launch failed.')).resolves.toBe( + 'Launch failed.' + ) + warn.mockRestore() + }) +}) diff --git a/src/main/ssh/orcad-remote-log-tail.ts b/src/main/ssh/orcad-remote-log-tail.ts new file mode 100644 index 00000000000..d588e0d35db --- /dev/null +++ b/src/main/ssh/orcad-remote-log-tail.ts @@ -0,0 +1,71 @@ +/** + * The end of a slot's orcad.log, appended to a failed deploy, launch or rollback so the error + * says why orcad stopped rather than only where its log is. Errors and the main log only; never + * telemetry. Best-effort: a host that can't answer leaves the error as it was. + */ +import { redactString } from '../observability/redactor' +import { shellEscape } from './ssh-connection-utils' +import { ORCAD_LOG_FILENAME } from './orcad-remote-host-support' +import { execCommand } from './ssh-relay-deploy-helpers' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_LOG_TAIL_MARKER } from './orcad-windows-host-script' +import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control' + +export const ORCAD_LOG_TAIL_LINES = 40 +export const ORCAD_LOG_TAIL_MAX_BYTES = 8 * 1024 +// Short, because it only decorates an error the caller already has. +const ORCAD_LOG_TAIL_TIMEOUT_MS = 10_000 + +export function orcadLogTailCommand(host: RemoteHostPlatform, slotDir: string): string { + const log = joinRemotePath(host, slotDir, ORCAD_LOG_FILENAME) + if (isWindowsRemoteHost(host)) { + return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, slotDir), 'log-tail', [ + log, + String(ORCAD_LOG_TAIL_MAX_BYTES) + ]) + } + return `tail -c ${ORCAD_LOG_TAIL_MAX_BYTES} ${shellEscape(log)} 2>/dev/null || true` +} + +/** The last lines of what the host printed, redacted; null when the log is empty or absent. */ +export function parseOrcadLogTail(host: RemoteHostPlatform, output: string): string | null { + const raw = isWindowsRemoteHost(host) + ? (readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_LOG_TAIL_MARKER) ?? '') + : output + const bounded = Buffer.from(raw, 'utf8').subarray(-ORCAD_LOG_TAIL_MAX_BYTES).toString('utf8') + const lines = bounded.replace(/\s+$/u, '').split(/\r?\n/u).slice(-ORCAD_LOG_TAIL_LINES) + const tail = redactString(lines.join('\n')) + return tail.trim() ? tail : null +} + +export async function readOrcadLogTail( + target: OrcadRemoteExecTarget, + slotDir: string +): Promise { + try { + // No abort signal: the failure being reported may be the cancellation itself. + const output = await execCommand(target.conn, orcadLogTailCommand(target.host, slotDir), { + wrapCommand: target.host.commandDialect !== 'powershell', + timeoutMs: ORCAD_LOG_TAIL_TIMEOUT_MS + }) + return parseOrcadLogTail(target.host, output) + } catch (error) { + console.warn('[ssh] Could not read the orcad.log tail:', error) + return null + } +} + +/** `message` followed by the log tail, or `message` alone when the host had none to give. */ +export async function withOrcadLogTail( + target: OrcadRemoteExecTarget, + slotDir: string, + message: string +): Promise { + const tail = await readOrcadLogTail(target, slotDir) + return tail ? `${message}\nLast lines of orcad.log:\n${tail}` : message +} diff --git a/src/main/ssh/orcad-remote-preflight.ts b/src/main/ssh/orcad-remote-preflight.ts index 40a17bcdde0..85e41339422 100644 --- a/src/main/ssh/orcad-remote-preflight.ts +++ b/src/main/ssh/orcad-remote-preflight.ts @@ -7,10 +7,17 @@ import { parseOrcadProfilePreflight } from '../../shared/orcad-profile-preflight' import { assertPosixOrcadHost } from './orcad-remote-host-support' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + orcadWindowsNodeCommandLine, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_RUNTIME_MARKER } from './orcad-windows-host-script' import { orcadNodeSlotRuntimeCommand } from './orcad-remote-runtime' import { execCommand } from './ssh-relay-deploy-helpers' import { shellEscape } from './ssh-connection-utils' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' import type { SshConnection } from './ssh-connection' export function orcadProfilePreflightCommand( @@ -40,10 +47,47 @@ export async function preflightInstalledOrcad(options: { signal?: AbortSignal }): Promise { const nonce = randomUUID() - const output = await execCommand( - options.conn, - orcadProfilePreflightCommand(options.host, options.remoteInstallDir, nonce), - { signal: options.signal, timeoutMs: ORCAD_PROFILE_PREFLIGHT_TIMEOUT_MS } - ) + const command = isWindowsRemoteHost(options.host) + ? await windowsOrcadProfilePreflightCommand(options, nonce) + : orcadProfilePreflightCommand(options.host, options.remoteInstallDir, nonce) + const output = await execCommand(options.conn, command, { + signal: options.signal, + timeoutMs: ORCAD_PROFILE_PREFLIGHT_TIMEOUT_MS, + wrapCommand: !isWindowsRemoteHost(options.host) + }) parseOrcadProfilePreflight(output, nonce, ORCAD_NODE_RUNTIME_IDENTITY, options.fullVersion) } + +/** + * Windows: resolve the slot's node.exe, then run its `orcad.js` with plain argv. No + * ORCA_BACKGROUND_LAUNCH here: orcad opens no window, and argv cannot set environment. + */ +async function windowsOrcadProfilePreflightCommand( + options: { + conn: SshConnection + host: RemoteHostPlatform + remoteInstallDir: string + signal?: AbortSignal + }, + nonce: string +): Promise { + const { host, remoteInstallDir } = options + const runtime = readOrcadWindowsEncodedAnswer( + await execCommand( + options.conn, + orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, remoteInstallDir), 'slot-runtime', [ + remoteInstallDir + ]), + { signal: options.signal, wrapCommand: false } + ), + ORCAD_WINDOWS_RUNTIME_MARKER + ) + if (!runtime) { + throw new Error('The Windows host did not name the runtime this orcad slot needs.') + } + return orcadWindowsNodeCommandLine(runtime, [ + joinRemotePath(host, remoteInstallDir, 'orcad.js'), + ORCAD_PROFILE_PREFLIGHT_FLAG, + nonce + ]) +} diff --git a/src/main/ssh/orcad-remote-primitives.test.ts b/src/main/ssh/orcad-remote-primitives.test.ts new file mode 100644 index 00000000000..fc2af62b9ff --- /dev/null +++ b/src/main/ssh/orcad-remote-primitives.test.ts @@ -0,0 +1,281 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: (error: unknown) => + error instanceof Error && error.message === 'channel lost' +})) + +vi.mock('./ssh-remote-platform-detection', () => ({ detectRemoteHostPlatform: vi.fn() })) +vi.mock('./orcad-windows-host-preparation', () => ({ prepareWindowsOrcadHost: vi.fn() })) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' +import { prepareWindowsOrcadHost } from './orcad-windows-host-preparation' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { emptyOrcadActivationRecord } from './orcad-activation-record' +import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { + launchOrcadSlotAndAwaitReadiness, + OrcadActiveReadinessError, + probeActiveOrcadReadiness +} from './orcad-active-readiness' +import { parseOrcadReadinessOutput } from './orcad-remote-launch' + +const mockExec = vi.mocked(execCommand) +const linux = getRemoteHostPlatform('linux-x64') +const windows = getRemoteHostPlatform('win32-x64') +const conn: SshConnection = Object.create(null) +const target = { conn, host: linux } +const BUILD_HASH = 'abc123def4567890' + +function readyLine( + buildHash = BUILD_HASH, + daemon: { coverage?: 'pty-spawn' | 'handshake'; platform?: string } = {} +): string { + const selfTest = { ok: true, verdict: 'healthy', durationMs: 5 } + return JSON.stringify({ + type: 'orca_server_ready', + runtimeId: 'r1', + boundEndpoint: 'ws://127.0.0.1:7777', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { available: false, reason: 'disabled_by_operator', guidance: 'n/a' }, + health: { + buildHash, + buildVersion: '0.2.0+bb01', + nodeVersion: '24.21.0', + nodeAbi: '137', + platform: daemon.platform ?? 'linux', + arch: 'x64', + pid: 1, + terminalDaemon: { + state: 'live', + ownsFreshSessions: true, + pid: 2, + buildVersion: '0.2.0+bb01', + entryPath: '/x/daemon-entry.js', + protocolVersion: 38, + selfTest: + 'coverage' in daemon + ? { ...selfTest, ...(daemon.coverage ? { coverage: daemon.coverage } : {}) } + : { ...selfTest, coverage: 'pty-spawn' } + } + } + }) +} + +beforeEach(() => { + mockExec.mockReset() +}) + +describe('orcad host record files', () => { + it('tells an absent record from one whose read gave no answer', async () => { + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({ + state: 'absent' + }) + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_PRESENT__\n{"a":1}') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).resolves.toEqual({ + state: 'present', + raw: '{"a":1}' + }) + mockExec.mockResolvedValueOnce('') + await expect(readBoundedOrcadRemoteRecord(target, '/r.json', 64)).rejects.toThrow( + 'no verifiable answer' + ) + }) + + it('keeps the partial file when the write may still be running on the host', async () => { + mockExec.mockRejectedValueOnce(new Error('channel lost')) + await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow() + expect(mockExec).toHaveBeenCalledOnce() + mockExec.mockRejectedValueOnce(new Error('exit 1')).mockResolvedValueOnce('') + await expect(writeAtomicOrcadRemoteRecord(target, '/r.json', '{}')).rejects.toThrow() + expect(String(mockExec.mock.calls.at(-1)?.[1])).toContain('rm -f') + }) + + it('refuses a Windows read that names no pinned node.exe to run it', async () => { + await expect( + readBoundedOrcadRemoteRecord({ conn, host: windows }, 'C:/r.json', 64) + ).rejects.toThrow('pinned node.exe') + expect(mockExec).not.toHaveBeenCalled() + }) +}) + +describe('activation record store', () => { + const options = { conn, host: linux, remoteHome: '/home/u' } + const newer = JSON.stringify({ ...emptyOrcadActivationRecord(), schemaVersion: 2 }) + + it('reads a lost read as an error, never as an empty record', async () => { + mockExec.mockRejectedValueOnce(new Error('channel lost')) + await expect(readOrcadActivationRecord(options)).rejects.toThrow('channel lost') + }) + + it('reads a newer schema as unreadable and never overwrites it', async () => { + mockExec.mockResolvedValue(`__ORCAD_RECORD_PRESENT__\n${newer}`) + await expect(readOrcadActivationRecord(options)).rejects.toThrow('schemaVersion 2') + await expect(writeOrcadActivationRecord(options, emptyOrcadActivationRecord())).rejects.toThrow( + 'Refusing to overwrite' + ) + expect(mockExec.mock.calls.some(([, command]) => String(command).includes('mv -f'))).toBe(false) + }) + + it('writes atomically when the host has no record yet', async () => { + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\n').mockResolvedValueOnce('') + await writeOrcadActivationRecord(options, emptyOrcadActivationRecord()) + expect(String(mockExec.mock.calls[1]?.[1])).toMatch(/printf %s .* && mv -f /s) + }) +}) + +describe('installed build identity and readiness', () => { + const slot = { ...target, remoteInstallDir: '/home/u/.orca-remote/orcad-0.2.0+bb01' } + const expectation = { buildHash: BUILD_HASH, fullVersion: '0.2.0+bb01' } + + it('reads the 16-hex build hash the slot reports', async () => { + mockExec.mockResolvedValueOnce(`noise\n__ORCAD_BUILD_HASH__ ${BUILD_HASH.toUpperCase()}\n`) + await expect(readRemoteOrcadBuildHash(target, '/slot')).resolves.toBe(BUILD_HASH) + mockExec.mockResolvedValueOnce('') + await expect(readRemoteOrcadBuildHash(target, '/slot')).rejects.toThrow() + }) + + it.each([ + ['DEAD', 'exited'], + ['UNKNOWN', 'unverifiable'], + ['', 'unverifiable'] + ])('reports a %j liveness probe as %s', async (liveness, verdict) => { + mockExec.mockResolvedValueOnce(liveness) + await expect(probeActiveOrcadReadiness(slot, expectation)).rejects.toMatchObject({ + verdict + }) + }) + + it('accepts only the expected build once the process is live', async () => { + mockExec.mockResolvedValueOnce('LIVE').mockResolvedValueOnce(`${readyLine()}\n`) + await expect(probeActiveOrcadReadiness(slot, expectation)).resolves.toMatchObject({ + runtimeId: 'r1' + }) + mockExec + .mockResolvedValueOnce('LIVE') + .mockResolvedValueOnce(`${readyLine('ffffffffffffffff')}\n`) + const rejected = probeActiveOrcadReadiness(slot, expectation) + await expect(rejected).rejects.toBeInstanceOf(OrcadActiveReadinessError) + await expect(rejected).rejects.toMatchObject({ verdict: 'rejected' }) + }) + + it.each([ + ['a spawn-probed PTY', { coverage: 'pty-spawn' as const }], + [ + 'a handshake on a host whose daemon never spawn-probes', + { coverage: 'handshake' as const, platform: 'win32' } + ], + ['an older build that does not report coverage', { coverage: undefined }] + ])('accepts %s', async (_name, daemon) => { + mockExec + .mockResolvedValueOnce('LIVE') + .mockResolvedValueOnce(`${readyLine(BUILD_HASH, daemon)}\n`) + await expect(probeActiveOrcadReadiness(slot, expectation)).resolves.toMatchObject({ + runtimeId: 'r1' + }) + }) + + it('rejects handshake-only coverage on a host whose daemon should spawn a PTY', async () => { + mockExec + .mockResolvedValueOnce('LIVE') + .mockResolvedValueOnce(`${readyLine(BUILD_HASH, { coverage: 'handshake' })}\n`) + await expect(probeActiveOrcadReadiness(slot, expectation)).rejects.toMatchObject({ + verdict: 'rejected', + message: expect.stringContaining("coverage 'handshake'") + }) + }) + + it('applies the same coverage rule to a slot it launches', async () => { + mockExec + .mockResolvedValueOnce('4242') + .mockResolvedValueOnce(`${readyLine(BUILD_HASH, { coverage: 'handshake' })}\n`) + await expect( + launchOrcadSlotAndAwaitReadiness( + { ...target, readinessTimeoutMs: 1_000, sleep: async () => {} }, + { + remoteInstallDir: slot.remoteInstallDir, + nodePath: '/usr/bin/node', + fullVersion: '0.2.0+bb01', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' + }, + expectation + ) + ).rejects.toMatchObject({ verdict: 'rejected' }) + }) + + it('reads readiness once even when the client was descheduled past its deadline', async () => { + mockExec.mockResolvedValueOnce('4242').mockResolvedValueOnce(`${readyLine(BUILD_HASH)}\n`) + await expect( + launchOrcadSlotAndAwaitReadiness( + { ...target, readinessTimeoutMs: 0, sleep: async () => {} }, + { + remoteInstallDir: slot.remoteInstallDir, + nodePath: '/usr/bin/node', + fullVersion: '0.2.0+bb01', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + activationRoot: '/home/u/.orca-remote/.orcad-activation-transaction' + }, + expectation + ) + ).resolves.toMatchObject({ runtimeId: 'r1' }) + }) +}) + +describe('readiness parsing bounds', () => { + it('waits on a half-written last line but rejects a finished invalid one', () => { + expect(parseOrcadReadinessOutput('{"type":"orca_ser')).toEqual({ state: 'pending' }) + expect(parseOrcadReadinessOutput('{"type":"orca_ser\n')).toMatchObject({ + state: 'malformed' + }) + }) + + it('rejects a payload over the size cap', () => { + expect(parseOrcadReadinessOutput('x'.repeat(256 * 1024 + 1))).toMatchObject({ + state: 'malformed' + }) + }) +}) + +describe('orcad remote context', () => { + it('prepares a Windows host (runtime, host script) before reading the activation record', async () => { + vi.mocked(detectRemoteHostPlatform).mockResolvedValueOnce(windows) + const order: string[] = [] + vi.mocked(prepareWindowsOrcadHost).mockImplementationOnce(async () => { + order.push('prepare') + }) + mockExec.mockImplementation(async (_conn, command: string) => { + if (command.includes('record-read')) { + order.push('record') + return '__ORCAD_RECORD_ABSENT__\r\n' + } + return 'C:\\Users\\u\r\n' + }) + const sshTarget = { id: 't', label: 't', host: 'h', port: 22, username: 'u' } + const context = await resolveOrcadRemoteContext(sshTarget, conn) + expect(context).toMatchObject({ serverTarget: 'win32-x64', remoteHome: 'C:/Users/u' }) + expect(vi.mocked(prepareWindowsOrcadHost).mock.calls[0]?.[0]).toMatchObject({ + remoteHome: 'C:/Users/u', + serverTarget: 'win32-x64' + }) + expect(order).toEqual(['prepare', 'record']) + }) +}) diff --git a/src/main/ssh/orcad-remote-process-control-windows.ts b/src/main/ssh/orcad-remote-process-control-windows.ts new file mode 100644 index 00000000000..5898a65fe01 --- /dev/null +++ b/src/main/ssh/orcad-remote-process-control-windows.ts @@ -0,0 +1,22 @@ +/** + * Stopping a Windows orcad: the slot's stop-request file, never a signal. + * + * `kill -TERM` and `process.kill(pid, 'SIGTERM')` are TerminateProcess on Windows, which skips + * the durable shutdown and leaves the instance lock behind, so a build whose readiness does not + * advertise `health.stopRequests` is refused rather than terminated. The host script's `stop` op + * answers with the POSIX command's tokens, plus UNSUPPORTED. + */ +import { orcadWindowsBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +export function windowsStopOrcadCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + options: { waitSeconds: number; justLaunched: boolean } +): string { + return orcadWindowsHostOpCommand(host, orcadWindowsBaseDir(host, remoteInstallDir), 'stop', [ + remoteInstallDir, + String(options.waitSeconds), + options.justLaunched ? '1' : '0' + ]) +} diff --git a/src/main/ssh/orcad-remote-process-control.ts b/src/main/ssh/orcad-remote-process-control.ts index 33fd877ba41..3650d5a6a71 100644 --- a/src/main/ssh/orcad-remote-process-control.ts +++ b/src/main/ssh/orcad-remote-process-control.ts @@ -5,20 +5,24 @@ * current owner; SIGKILL would skip flushing state and releasing the instance lock. */ import { shellEscape } from './ssh-connection-utils' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' import { ORCAD_READINESS_FILENAME } from './orcad-remote-launch' -import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' import { - assertPosixOrcadHost as assertPosixHost, - ORCAD_PID_FILENAME, - posixProcessAliveShellFunction -} from './orcad-remote-host-support' + ORCAD_STOP_REQUEST_FILENAME, + ORCAD_STOP_REQUESTS_CAPABILITY +} from '../../shared/orcad-stop-request' +import { selectOrcadSlotRuntimeCommand } from './orcad-remote-runtime' +import { ORCAD_PID_FILENAME, posixProcessAliveShellFunction } from './orcad-remote-host-support' +import { windowsStopOrcadCommand } from './orcad-remote-process-control-windows' /** - * Signal the orcad recorded in a version dir and wait for it to go. + * Ask the orcad recorded in a version dir to stop, and wait for it to go. * + * A build whose readiness advertises `health.stopRequests` is asked through the slot-local + * request file, which only that orcad watches; older builds keep receiving SIGTERM. Both need + * the readiness PID to corroborate the launcher's PID first, so a reused PID is never stopped. * `justLaunched` is only for this client's fixed exec launcher, including pre-readiness exits. - * Incumbents need their own readiness PID to corroborate the launcher's PID before any signal. + * Windows has no graceful signal, so it only ever writes the request file. */ export function stopOrcadCommand( host: RemoteHostPlatform, @@ -28,29 +32,44 @@ export function stopOrcadCommand( | { justLaunched?: false; nodePath: string } ) ): string { - assertPosixHost(host) + if (isWindowsRemoteHost(host)) { + return windowsStopOrcadCommand(host, remoteInstallDir, { + waitSeconds: options.waitSeconds, + justLaunched: options.justLaunched === true + }) + } const pidFile = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_PID_FILENAME)) const readiness = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + const requestFile = shellEscape( + joinRemotePath(host, remoteInstallDir, ORCAD_STOP_REQUEST_FILENAME) + ) const readRuntimePid = [ `const r = JSON.parse(require('node:fs').readFileSync(process.argv[1], 'utf8'));`, `const pid = r?.type === 'orca_server_ready' ? r.health?.pid : null;`, `if (!Number.isSafeInteger(pid) || pid <= 1) process.exit(1);`, - `process.stdout.write(String(pid));` + `const mode = r.health?.stopRequests === ${ORCAD_STOP_REQUESTS_CAPABILITY} ? 'request' : 'signal';`, + `process.stdout.write(String(pid) + ':' + mode);` ].join(' ') return [ posixProcessAliveShellFunction({ refuseUnverifiable: true }), `pid=$(cat ${pidFile} 2>/dev/null);`, 'case "$pid" in "" | *[!0-9]* ) echo NO_PID; exit 0;; esac;', + 'stop_mode=signal;', // Older launchers recorded a waiting shell, whose exit does not prove runtime exit. ...(options.justLaunched ? [] : [ - `runtime_pid=$(${selectOrcadSlotRuntimeCommand(host, remoteInstallDir, options.nodePath)}; ` + + `runtime_answer=$(${selectOrcadSlotRuntimeCommand(host, remoteInstallDir, options.nodePath)}; ` + `"$orcad_runtime" -e ${shellEscape(readRuntimePid)} ${readiness} 2>/dev/null) || { echo UNKNOWN; exit 0; };`, - '[ "$pid" = "$runtime_pid" ] || { echo UNKNOWN; exit 0; };' + '[ "$pid" = "${runtime_answer%%:*}" ] || { echo UNKNOWN; exit 0; };', + 'stop_mode=${runtime_answer#*:};' ]), 'orcad_alive "$pid" || { echo ALREADY_EXITED; exit 0; };', - 'kill -TERM "$pid" 2>/dev/null || { echo SIGNAL_FAILED; exit 0; };', + 'if [ "$stop_mode" = request ]; then', + `( umask 077; : > ${requestFile} ) 2>/dev/null || { echo SIGNAL_FAILED; exit 0; };`, + 'else kill -TERM "$pid" 2>/dev/null || { echo SIGNAL_FAILED; exit 0; }; fi;', + // Past here the stop may be under way, so a lost or unknown answer must keep the fence. + 'echo SIGNALED;', `i=0; while [ "$i" -lt ${options.waitSeconds} ]; do`, 'orcad_alive "$pid" || { echo STOPPED; exit 0; };', 'sleep 1; i=$((i + 1)); done;', @@ -64,10 +83,23 @@ export type OrcadStopOutcome = | 'no-pid' | 'still-running' | 'signal-failed' + /** Windows only: the build cannot be asked to stop, and terminating it would skip shutdown. */ + | 'unsupported' + /** Explicitly unknown before any stop was sent: nothing happened. */ | 'unknown' + /** Unknown or unparseable once a stop may have been sent: the host may still be changing. */ + | 'unconfirmed' export function parseOrcadStopOutcome(output: string): OrcadStopOutcome { - switch (output.trim().split('\n').pop()?.trim() ?? '') { + const lines = output + .trim() + .split(/\r?\n/u) + .map((line) => line.trim()) + const last = lines.at(-1) ?? '' + if (last === 'UNKNOWN' && !lines.includes('SIGNALED')) { + return 'unknown' + } + switch (last) { case 'STOPPED': return 'stopped' case 'ALREADY_EXITED': @@ -78,8 +110,10 @@ export function parseOrcadStopOutcome(output: string): OrcadStopOutcome { return 'still-running' case 'SIGNAL_FAILED': return 'signal-failed' + case 'UNSUPPORTED': + return 'unsupported' default: - return 'unknown' + return 'unconfirmed' } } diff --git a/src/main/ssh/orcad-remote-readiness-wait.test.ts b/src/main/ssh/orcad-remote-readiness-wait.test.ts new file mode 100644 index 00000000000..8e1043a59dc --- /dev/null +++ b/src/main/ssh/orcad-remote-readiness-wait.test.ts @@ -0,0 +1,40 @@ +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { orcadReadinessWaitCommand } from './orcad-remote-readiness-wait' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +// BUG-17: under host load each poll's reads outlasted its sleep, so a counted loop ran past the +// client's 30 s exec timeout and the launch was failed while the candidate was still starting. +describe.skipIf(process.platform === 'win32')('the host-side readiness wait', () => { + it('ends by its wall-clock deadline however slow each poll is', async () => { + const root = mkdtempSync(join(tmpdir(), 'orcad-readiness-wait-')) + roots.push(root) + const bin = join(root, 'bin') + mkdirSync(bin) + // A host where every `wc` takes half a second. + writeFileSync(join(bin, 'wc'), '#!/bin/sh\nsleep 0.5\nexec /usr/bin/wc "$@"\n') + chmodSync(join(bin, 'wc'), 0o755) + const slot = join(root, 'slot') + mkdirSync(slot) + const command = orcadReadinessWaitCommand(getRemoteHostPlatform('linux-x64'), slot, 2) + const startedAt = Date.now() + await runProcess({ + program: '/bin/sh', + args: ['-c', command], + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ''}` }, + timeoutMs: 30_000 + }) + // A counted loop would take about 8 x (0.25 s + 1 s) here. + expect(Date.now() - startedAt).toBeLessThan(4_500) + }) +}) diff --git a/src/main/ssh/orcad-remote-readiness-wait.ts b/src/main/ssh/orcad-remote-readiness-wait.ts new file mode 100644 index 00000000000..310d017272b --- /dev/null +++ b/src/main/ssh/orcad-remote-readiness-wait.ts @@ -0,0 +1,94 @@ +/** + * Waiting for a launched orcad's readiness line on the host, not across SSH. + * + * The client used to re-read the readiness file every 500 ms, one exec each, for up to three + * minutes. A burst of short-lived processes under sshd is itself an EDR signal on Windows + * (docs/reference/windows-edr-posture.md). One exec now waits host-side for a complete line, an + * oversized file, or its own bounded deadline. + */ +import { shellEscape } from './ssh-connection-utils' +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { + ORCAD_READINESS_FILENAME, + ORCAD_READINESS_MAX_BYTES, + parseOrcadReadinessOutput, + readOrcadReadinessCommand, + type OrcadReadinessParse +} from './orcad-remote-launch' +import { + orcadWindowsBaseDir, + orcadWindowsHostOpCommand, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_READINESS_MARKER } from './orcad-windows-host-script' + +/** Under the 30 s exec timeout, so one wait never reads as an unanswered host. */ +export const ORCAD_READINESS_WAIT_MAX_SECONDS = 20 + +/** Settles on a newline (a finished line) or more than the cap; both are final for the parser. */ +function posixReadinessWaitCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + waitSeconds: number +): string { + const file = shellEscape(joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME)) + const read = `head -c ${ORCAD_READINESS_MAX_BYTES + 1} ${file} 2>/dev/null` + return [ + // Fractional sleep is not POSIX; probe it once so polling stays fine-grained where it can. + 'if sleep 0.25 2>/dev/null; then orcad_readiness_wait_step=0.25;', + 'else orcad_readiness_wait_step=1; fi;', + // Why a wall-clock deadline, not a step count: on a loaded host each step's reads take far + // longer than its sleep, a counted loop overran the client's exec timeout, and the launch + // was failed while the candidate was still starting (BUG-17). + `orcad_readiness_wait_end=$(($(date +%s) + ${waitSeconds}));`, + 'while [ "$(date +%s)" -lt "$orcad_readiness_wait_end" ]; do', + `[ "$(${read} | wc -l)" -gt 0 ] && break;`, + `[ "$(${read} | wc -c)" -gt ${ORCAD_READINESS_MAX_BYTES} ] && break;`, + 'sleep "$orcad_readiness_wait_step"; done;', + `${read} || true` + ].join(' ') +} + +export function orcadReadinessWaitCommand( + host: RemoteHostPlatform, + remoteInstallDir: string, + waitSeconds: number +): string { + const seconds = Math.max(0, Math.min(ORCAD_READINESS_WAIT_MAX_SECONDS, Math.ceil(waitSeconds))) + if (!isWindowsRemoteHost(host)) { + return posixReadinessWaitCommand(host, remoteInstallDir, seconds) + } + return orcadWindowsHostOpCommand( + host, + orcadWindowsBaseDir(host, remoteInstallDir), + 'readiness-wait', + [ + joinRemotePath(host, remoteInstallDir, ORCAD_READINESS_FILENAME), + String(ORCAD_READINESS_MAX_BYTES), + String(seconds) + ] + ) +} + +/** What the readiness file held when the host stopped waiting; `pending` if still unfinished. */ +export function parseOrcadReadinessWaitOutput( + host: RemoteHostPlatform, + output: string +): OrcadReadinessParse { + return parseOrcadReadinessOutput( + isWindowsRemoteHost(host) + ? (readOrcadWindowsEncodedAnswer(output, ORCAD_WINDOWS_READINESS_MARKER) ?? '') + : output + ) +} + +/** Reads the readiness line as it stands; parse with `parseOrcadReadinessWaitOutput`. */ +export function readOrcadReadinessNowCommand( + host: RemoteHostPlatform, + remoteInstallDir: string +): string { + // Why: Windows has no `head`; its host script's wait op with no wait reads the same bytes. + return isWindowsRemoteHost(host) + ? orcadReadinessWaitCommand(host, remoteInstallDir, 0) + : readOrcadReadinessCommand(host, remoteInstallDir) +} diff --git a/src/main/ssh/orcad-remote-record-file.ts b/src/main/ssh/orcad-remote-record-file.ts new file mode 100644 index 00000000000..c1cacb3a2a6 --- /dev/null +++ b/src/main/ssh/orcad-remote-record-file.ts @@ -0,0 +1,120 @@ +/** + * Small JSON records Orca keeps on an orcad host (activation, transactions, stop receipts). + * + * Reads are bounded and never swallow a failure: a record that could not be read is not an + * absent one, and treating it as absent is how a client deploys over a live install. + */ +import { randomUUID } from 'node:crypto' +import { shellEscape } from './ssh-connection-utils' +import { removeRemoteFileCommand } from './ssh-remote-commands' +import { execOrcadRemote, type OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { isWindowsRemoteHost, joinRemotePath } from './ssh-remote-platform' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { + orcadWindowsHostOpCommand, + readOrcadWindowsEncodedAnswer +} from './orcad-remote-windows-node' +import { + ORCAD_RECORD_ABSENT_MARKER as ABSENT_MARKER, + ORCAD_RECORD_PRESENT_MARKER as PRESENT_MARKER +} from './orcad-windows-host-script' + +/** `present` carries raw bytes; schema checks belong to the caller that owns the format. */ +export type OrcadRemoteRecordRead = { state: 'absent' } | { state: 'present'; raw: string } + +/** `~/.orca-remote`, where the host script and the pinned runtime live. */ +function windowsBaseDir(target: OrcadRemoteExecTarget): string { + if (!target.remoteHome) { + throw new Error( + 'orcad host records on Windows need the remote home to find the pinned node.exe' + ) + } + return joinRemotePath(target.host, target.remoteHome, RELAY_REMOTE_DIR) +} + +export async function readBoundedOrcadRemoteRecord( + target: OrcadRemoteExecTarget, + path: string, + maxBytes: number +): Promise { + if (isWindowsRemoteHost(target.host)) { + return readWindowsRecord(target, path, maxBytes) + } + const file = shellEscape(path) + // Why markers: an empty stdout must never be mistaken for "no record" when the read failed. + const output = await execOrcadRemote( + target, + `if [ ! -e ${file} ] && [ ! -L ${file} ]; then printf '%s\\n' ${ABSENT_MARKER}; exit 0; fi; ` + + `[ -f ${file} ] || exit 65; size=$(wc -c < ${file}) || exit 65; ` + + `[ "$size" -le ${maxBytes} ] || exit 65; ` + + `printf '%s\\n' ${PRESENT_MARKER}; cat ${file}` + ) + const newline = output.indexOf('\n') + const marker = (newline === -1 ? output : output.slice(0, newline)).trim() + if (marker === ABSENT_MARKER) { + return { state: 'absent' } + } + if (marker !== PRESENT_MARKER) { + throw new Error('orcad host record read returned no verifiable answer') + } + return { state: 'present', raw: newline === -1 ? '' : output.slice(newline + 1) } +} + +async function readWindowsRecord( + target: OrcadRemoteExecTarget, + path: string, + maxBytes: number +): Promise { + const output = await execOrcadRemote( + target, + orcadWindowsHostOpCommand(target.host, windowsBaseDir(target), 'record-read', [ + path, + String(maxBytes) + ]) + ) + if (output.split(/\r?\n/u).some((line) => line.trim() === ABSENT_MARKER)) { + return { state: 'absent' } + } + const raw = readOrcadWindowsEncodedAnswer(output, PRESENT_MARKER) + if (raw === null) { + throw new Error('orcad host record read returned no verifiable answer') + } + return { state: 'present', raw } +} + +export async function writeAtomicOrcadRemoteRecord( + target: OrcadRemoteExecTarget, + path: string, + contents: string +): Promise { + const partialPath = `${path}.partial.${process.pid}.${randomUUID()}` + const baseDir = isWindowsRemoteHost(target.host) ? windowsBaseDir(target) : null + try { + if (baseDir) { + await target.conn.writeFile(partialPath, contents, { + hostPlatform: target.host, + signal: target.signal + }) + await execOrcadRemote( + target, + orcadWindowsHostOpCommand(target.host, baseDir, 'record-publish', [partialPath, path]) + ) + return + } + await execOrcadRemote( + target, + `umask 077; printf %s ${shellEscape(contents)} > ${shellEscape(partialPath)} && ` + + `mv -f ${shellEscape(partialPath)} ${shellEscape(path)}` + ) + } catch (error) { + // Why keep the partial on an unconfirmed termination: the write may still be running there. + if (!isUnconfirmedSshCommandTermination(error)) { + const discard = baseDir + ? orcadWindowsHostOpCommand(target.host, baseDir, 'remove-file', [partialPath]) + : removeRemoteFileCommand(target.host, partialPath) + await execOrcadRemote(target, discard).catch(() => {}) + } + throw error + } +} diff --git a/src/main/ssh/orcad-remote-rollback.test.ts b/src/main/ssh/orcad-remote-rollback.test.ts index 4d2f9150a75..59baa86881a 100644 --- a/src/main/ssh/orcad-remote-rollback.test.ts +++ b/src/main/ssh/orcad-remote-rollback.test.ts @@ -1,4 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as RecordFile from './orcad-remote-record-file' +import type * as InstallLock from './ssh-relay-install-lock' +import type * as TerminalBarrier from './orcad-rollback-terminal-barrier' vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn(), @@ -9,12 +12,52 @@ vi.mock('./ssh-relay-install-transfers', () => ({ writeRelayFile: vi.fn().mockResolvedValue(undefined), uploadRelayDirectory: vi.fn().mockResolvedValue(undefined) })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) + +const barrier = vi.hoisted(() => { + const state: { log: string[] | null; state: 'retired' | 'unproven' } = { + log: null, + state: 'retired' + } + return state +}) +// The managed stop and its terminal barrier are proven in orcad-activation-crash-recovery.test.ts. +vi.mock('./orcad-rollback-terminal-barrier', async (importOriginal) => ({ + ...(await importOriginal()), + readOrcadRollbackBarrierTarget: async (_options: unknown, version: string) => ({ + state: 'ready', + context: { + version, + runtimeId: 'r1', + instance: { pid: 1, startedAtMs: 1, nonce: 'n', lockPath: '/l' } + } + }), + stopIncumbentBehindTerminalBarrier: async ( + _options: unknown, + _id: string, + context: { version: string } + ) => { + barrier.log?.push(`stop:${context.version}`) + return barrier.state === 'retired' + ? { state: 'retired' } + : { state: 'unproven', reason: 'A terminal started after the census.' } + } +})) import { execCommand } from './ssh-relay-deploy-helpers' -import { writeRelayFile } from './ssh-relay-install-transfers' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' import { rollbackOrcad, type OrcadRollbackOptions } from './orcad-remote-rollback' import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record' import { getRemoteHostPlatform } from './ssh-remote-platform' +import { isReadinessRead } from './orcad-activation-host-test-harness' +import { isSnapshotCaptureCommand } from './orcad-snapshot-capture-command' import type { SshConnection } from './ssh-connection' const mockExec = vi.mocked(execCommand) @@ -68,35 +111,65 @@ function readyLine(version: string): string { }) } -function scriptHost( - log: string[], - overrides: { restore?: string; readinessAtMs?: number } = {} -): void { +type HostOverrides = { + restores?: string[] + readinessAtMs?: number + targetReady?: boolean + snapshot?: string + comparison?: string +} + +function scriptHost(log: string[], overrides: HostOverrides = {}): void { + barrier.log = log + barrier.state = 'retired' + const restores = [...(overrides.restores ?? [])] mockExec.mockImplementation(async (_conn, command: string) => { const text = String(command) - if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { - return 'PRESENT' + if (text.startsWith('tail -c')) { + return text.includes(TARGET) ? 'orcad: listen EADDRINUSE 127.0.0.1\n' : '' } - if (text.includes('find ') && text.includes('stat')) { + if (text.includes('__ORCAD_RECORD_PRESENT__')) { + return text.includes('transaction.json') + ? '__ORCAD_RECORD_ABSENT__\n' + : `__ORCAD_RECORD_PRESENT__\n${JSON.stringify(record())}` + } + if (text.includes('__ORCAD_BUILD_HASH__')) { + return `__ORCAD_BUILD_HASH__ ${BUILD_HASH}\n` + } + if (text.includes('echo PRESENT')) { + return overrides.snapshot ?? 'PRESENT' + } + if (text.includes('stat -c %Y')) { return 'UNKNOWN' } if (text.includes('kill -TERM')) { log.push(`stop:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) return 'STOPPED' } + if (text.includes('verdict=UNCHANGED')) { + log.push('compare') + return overrides.comparison ?? 'CHANGED' + } + if (isSnapshotCaptureCommand(text)) { + log.push('rescue') + return 'CAPTURED' + } if (text.includes('tar -C') && text.includes('-xf')) { - log.push('restore') - return overrides.restore ?? 'RESTORED' + log.push(text.includes('rollback-rescue-') ? 'restore-rescue' : 'restore') + return restores.shift() ?? 'RESTORED' } if (text.includes('nohup')) { log.push(`launch:${text.includes(ACTIVE) ? ACTIVE : TARGET}`) return '9999' } - if (text.startsWith('cat ') && text.includes('.orcad-readiness')) { + if (isReadinessRead(text) && text.includes('.orcad-readiness')) { if (overrides.readinessAtMs !== undefined && Date.now() < overrides.readinessAtMs) { return '' } - return readyLine(TARGET) + if (text.includes(ACTIVE)) { + return readyLine(ACTIVE) + } + return overrides.targetReady === false ? '' : readyLine(TARGET) } return '' }) @@ -112,8 +185,9 @@ function options(overrides: Partial = {}): OrcadRollbackOp userDataDir: '/home/u/.orca', bindHost: '127.0.0.1', port: 7777, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, targetBuildHash: BUILD_HASH, + targetDaemonProtocol: { protocolVersion: 3, previousProtocolVersions: [1, 2] }, readinessTimeoutMs: 50, sleep: async () => {}, now: () => new Date('2026-02-02T00:00:00.000Z'), @@ -133,7 +207,7 @@ describe('rollbackOrcad', () => { expect(result).toMatchObject({ outcome: 'rolled-back', target: TARGET }) // Restoring under a running orcad would replace the store beneath a process holding it; // starting first would let the older build migrate the newer build's state. - expect(log).toEqual([`stop:${ACTIVE}`, 'restore', `launch:${TARGET}`]) + expect(log).toEqual([`stop:${ACTIVE}`, 'rescue', 'restore', `launch:${TARGET}`]) }) it('allows rollback startup time after a slow bundled preflight', async () => { @@ -152,7 +226,7 @@ describe('rollbackOrcad', () => { ) expect(result.outcome).toBe('rolled-back') expect(elapsedMs).toBe(100_000) - expect(log).toEqual([`stop:${ACTIVE}`, 'restore', `launch:${TARGET}`]) + expect(log).toEqual([`stop:${ACTIVE}`, 'rescue', 'restore', `launch:${TARGET}`]) } finally { clock.mockRestore() } @@ -162,57 +236,95 @@ describe('rollbackOrcad', () => { const log: string[] = [] scriptHost(log) const result = await rollbackOrcad( - options({ census: { liveSessions: 3, startedSinceActivation: 2 } }) + options({ census: { liveSessions: 3, startedSinceActivation: 2, daemonProtocolVersion: 3 } }) ) expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_orphans_live_terminals' }) expect(log).toEqual([]) - expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + expect(vi.mocked(writeAtomicOrcadRemoteRecord)).not.toHaveBeenCalled() }) it('refuses when the snapshot is gone from the host', async () => { const log: string[] = [] - mockExec.mockImplementation(async (_conn, command: string) => - String(command).includes('state.tar') ? 'ABSENT' : '' - ) + scriptHost(log, { snapshot: 'ABSENT' }) const result = await rollbackOrcad(options()) expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_snapshot_missing' }) expect(log).toEqual([]) }) - it('does not start the old build when the restore failed', async () => { + it('does not read a lost snapshot probe as a missing snapshot', async () => { const log: string[] = [] - scriptHost(log, { restore: 'FAILED' }) + scriptHost(log, { snapshot: '' }) const result = await rollbackOrcad(options()) - expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_rollback_restore_failed' }) - expect(log).toEqual([`stop:${ACTIVE}`, 'restore']) - expect(result.outcome === 'failed' && result.reason).toContain('Do NOT start the older build') + expect(result).toMatchObject({ + outcome: 'refused', + code: 'orcad_rollback_snapshot_unverifiable' + }) + expect(log).toEqual([]) }) - it('leaves the record naming the newer version when the target fails to come up', async () => { + it('puts the rescued state and the newer build back when the restore failed', async () => { const log: string[] = [] - scriptHost(log) - mockExec.mockImplementation(async (_conn, command: string) => { - const text = String(command) - if (text.includes('state.tar') && text.includes('test -f') && !text.includes('tar -C')) { - return 'PRESENT' - } - if (text.includes('kill -TERM')) { - return 'STOPPED' - } - if (text.includes('tar -C') && text.includes('-xf')) { - return 'RESTORED' - } - // The target never publishes readiness. - return '' - }) + scriptHost(log, { restores: ['FAILED'] }) + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_rollback_restore_failed' }) + expect(log).toEqual([ + `stop:${ACTIVE}`, + 'rescue', + 'restore', + 'restore-rescue', + `launch:${ACTIVE}` + ]) + expect(result.outcome === 'failed' && result.reason).toContain('is serving again') + }) + + it('keeps the newer state when a failed target may have changed it, until an operator accepts', async () => { + const log: string[] = [] + scriptHost(log, { targetReady: false }) const result = await rollbackOrcad(options()) expect(result).toMatchObject({ outcome: 'failed', code: 'orcad_activation_no_readiness' }) + expect(result.outcome === 'failed' && result.reason).toContain('Recover to restore') + expect(result.outcome === 'failed' && result.reason).toContain( + 'Last lines of orcad.log:\norcad: listen EADDRINUSE' + ) + expect(log).toEqual([ + `stop:${ACTIVE}`, + 'rescue', + 'restore', + `launch:${TARGET}`, + `stop:${TARGET}`, + 'compare' + ]) // Until the target is proven serving, `active` must still name the version an operator // would have to bring back. - expect(vi.mocked(writeRelayFile)).not.toHaveBeenCalled() + expect( + vi + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.some((call) => String(call[1]).endsWith('orcad-active.json')) + ).toBe(false) + }) + + it('puts the newer build back unasked when a failed target left the restored state untouched', async () => { + const log: string[] = [] + scriptHost(log, { targetReady: false, comparison: 'UNCHANGED' }) + const result = await rollbackOrcad(options()) + expect(result.outcome === 'failed' && result.reason).toContain('is serving again') + expect(log.slice(-3)).toEqual(['compare', 'restore-rescue', `launch:${ACTIVE}`]) + const compare = mockExec.mock.calls + .map((call) => String(call[1])) + .find((command) => command.includes('verdict=UNCHANGED')) + expect(compare).not.toContain('rollback-rescue-') + }) + + it('keeps the newer state and restarts the newer build when work appeared after the census', async () => { + const log: string[] = [] + scriptHost(log) + barrier.state = 'unproven' + const result = await rollbackOrcad(options()) + expect(result).toMatchObject({ outcome: 'refused', code: 'orcad_rollback_terminals_at_stop' }) + expect(log).toEqual([`stop:${ACTIVE}`, `launch:${ACTIVE}`]) }) it('records the rollback only after the target answers healthy', async () => { @@ -220,9 +332,9 @@ describe('rollbackOrcad', () => { scriptHost(log) await rollbackOrcad(options()) const written = vi - .mocked(writeRelayFile) - .mock.calls.find((call) => String(call[2]).endsWith('orcad-active.json')) - expect(JSON.parse(String(written?.[3]))).toMatchObject({ + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find((call) => String(call[1]).endsWith('orcad-active.json')) + expect(JSON.parse(String(written?.[2]))).toMatchObject({ active: TARGET, previous: null, snapshot: null diff --git a/src/main/ssh/orcad-remote-rollback.ts b/src/main/ssh/orcad-remote-rollback.ts index 412021c2f22..b9100b72657 100644 --- a/src/main/ssh/orcad-remote-rollback.ts +++ b/src/main/ssh/orcad-remote-rollback.ts @@ -8,46 +8,27 @@ * build cannot be shown to read the result. Rollback therefore restores the pre-activation * snapshot, and refuses when restoring it would orphan work (`assessOrcadRollback`). * - * The order below is the whole safety argument: stop, then restore, then start. Restoring - * under a running orcad would replace the store beneath a process holding it open, and - * starting before restoring would let the old build migrate the new build's state — the - * failure this is meant to avoid, arrived at from the other side. + * The order is the whole safety argument: stop, rescue, restore, then start. Restoring under + * a running orcad would replace the store beneath a process holding it open, and starting + * before restoring would let the old build migrate the new build's state. The rescue copy of + * the newer state, and the journal under the activation fence, make each step undoable. */ +import { logOrcadActivationOutcome } from './orcad-activation-outcome-log' import type { SshConnection } from './ssh-connection' +import type { OrcadActivationRecord } from './orcad-activation-record' +import type { OrcadTerminalCensus } from './orcad-update-plan' +import type { OrcadActivationVerdict } from './orcad-activation-gate' +import type { OrcadDaemonProtocolFacts } from './orcad-daemon-protocol-crossing' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { sameOrcadActivationRecord } from './orcad-activation-transaction' +import type { RemoteHostPlatform } from './ssh-remote-platform' +import { + resolveOrcadActivationReadinessTimeout, + withOrcadActivationLock +} from './orcad-activation-lock' +import { orcadActivationFenceRefusal } from './orcad-activation-fence-hold' import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' -import { execCommand } from './ssh-relay-deploy-helpers' -import { ORCAD_INSTALL_MODEL } from './remote-install-model' -import { computeRemoteInstallDir } from './ssh-relay-versioned-install' -import { writeRelayFile } from './ssh-relay-install-transfers' -import { RELAY_REMOTE_DIR } from './relay-protocol' -import { - ORCAD_STATE_SNAPSHOT_DIR, - serializeOrcadActivationRecord, - withRolledBackVersion, - type OrcadActivationRecord -} from './orcad-activation-record' -import { assessOrcadRollback, type OrcadTerminalCensus } from './orcad-update-plan' -import { evaluateOrcadActivation, type OrcadActivationVerdict } from './orcad-activation-gate' -import { - ORCAD_LOG_FILENAME, - orcadLaunchCommand, - parseOrcadReadinessOutput, - readOrcadReadinessCommand -} from './orcad-remote-launch' -import { - newestStateMtimeCommand, - parseNewestStateMtimeSeconds, - parseOrcadSnapshotRestore, - probeOrcadStateSnapshotCommand, - restoreOrcadStateSnapshotCommand -} from './orcad-state-snapshot' -import { - orcadStopFreedTheHost, - parseOrcadStopOutcome, - stopOrcadCommand -} from './orcad-remote-process-control' -import { orcadActivationPath } from './orcad-activation-record-store' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import { rollbackOrcadLocked } from './orcad-rollback-transition' export type OrcadRollbackOptions = { conn: SshConnection @@ -61,6 +42,8 @@ export type OrcadRollbackOptions = { census: OrcadTerminalCensus /** Expected build hash of the rollback target, from the client's copy of those bytes. */ targetBuildHash: string + /** The rollback target's daemon protocol facts, from the same copy. */ + targetDaemonProtocol: OrcadDaemonProtocolFacts readinessTimeoutMs?: number now?: () => Date sleep?: (ms: number) => Promise @@ -72,170 +55,38 @@ export type OrcadRollbackResult = | { outcome: 'refused'; code: string; reason: string } | { outcome: 'failed'; code: string; reason: string } -const READINESS_POLL_MS = 500 -const STOP_WAIT_SECONDS = 20 - -function exec(options: OrcadRollbackOptions, command: string): Promise { - return execCommand(options.conn, command, { - wrapCommand: options.host.commandDialect !== 'powershell', - signal: options.signal - }) -} - -function snapshotDirPath(options: OrcadRollbackOptions, dirName: string): string { - return joinRemotePath( - options.host, - options.remoteHome, - RELAY_REMOTE_DIR, - ORCAD_STATE_SNAPSHOT_DIR, - dirName - ) -} - -/** Has the store been written since activation? `null` when it cannot be established. */ -async function readStateWritesSinceActivation( - options: OrcadRollbackOptions -): Promise { - if (!options.record.activatedAt) { - return null - } - const activatedAtSeconds = Math.floor(Date.parse(options.record.activatedAt) / 1000) - if (!Number.isFinite(activatedAtSeconds)) { - return null - } - const newest = parseNewestStateMtimeSeconds( - await exec(options, newestStateMtimeCommand(options.host, options.userDataDir)).catch(() => '') - ) - return newest === null ? null : newest >= activatedAtSeconds -} - -export async function rollbackOrcad(options: OrcadRollbackOptions): Promise { - const now = options.now ?? ((): Date => new Date()) - const snapshotPresent = options.record.snapshot - ? ( - await exec( - options, - probeOrcadStateSnapshotCommand( - options.host, - snapshotDirPath(options, options.record.snapshot.dirName) - ) - ).catch(() => 'ABSENT') - ).trim() === 'PRESENT' - : false - - const safety = assessOrcadRollback({ - record: options.record, - snapshotPresent, - census: options.census, - stateWritesSinceActivation: await readStateWritesSinceActivation(options) - }) - if (safety.safety === 'unsafe') { - return { outcome: 'refused', code: safety.code, reason: safety.reason } - } - - if (options.record.active) { - const outgoingDir = computeRemoteInstallDir( - ORCAD_INSTALL_MODEL, - options.remoteHome, - options.record.active +export async function rollbackOrcad(input: OrcadRollbackOptions): Promise { + const options = { + ...input, + readinessTimeoutMs: resolveOrcadActivationReadinessTimeout( + input.readinessTimeoutMs, + ORCAD_STARTUP_READINESS_TIMEOUT_MS ) - const stopped = parseOrcadStopOutcome( - await exec( + } + return logOrcadActivationOutcome( + `rollback to ${options.record.previous ?? 'none'}`, + () => + withOrcadActivationLock( options, - stopOrcadCommand(options.host, outgoingDir, { - waitSeconds: STOP_WAIT_SECONDS, - nodePath: options.nodePath - }) - ) - ) - if (!orcadStopFreedTheHost(stopped)) { - return { - outcome: 'failed', - code: 'orcad_rollback_stop_incomplete', - reason: - `Could not verify that orcad ${options.record.active} exited (${stopped}). ` + - 'Nothing was restored. Orca requires matching runtime readiness before signaling ' + - 'an incumbent and confirmed exit before replacing its state.' - } - } - } - - // Why between stop and start: the store must be replaced while no orcad holds it, and - // before the older build gets a chance to migrate the newer build's state. - const restored = parseOrcadSnapshotRestore( - await exec( - options, - restoreOrcadStateSnapshotCommand( - options.host, - options.userDataDir, - // Guarded by `assessOrcadRollback`: `unsafe` covers a missing snapshot. - snapshotDirPath(options, options.record.snapshot?.dirName ?? '') - ) - ).catch(() => 'FAILED') + async (lock) => { + if ( + !sameOrcadActivationRecord(await readOrcadActivationRecord(options), options.record) + ) { + return { + outcome: 'refused', + code: 'orcad_rollback_record_changed', + reason: + 'The host activation record changed while this rollback was waiting. Refresh the ' + + 'host state and review the new rollback target before trying again.' + } + } + return rollbackOrcadLocked(options, lock) + }, + async () => { + const { code, reason } = await orcadActivationFenceRefusal(options, 'rollback') + return { outcome: 'refused', code, reason } + } + ), + ['rolled-back'] ) - if (restored !== 'restored') { - return { - outcome: 'failed', - code: 'orcad_rollback_restore_failed', - reason: - `The pre-activation snapshot could not be restored (${restored}). orcad is stopped and ` + - 'the data root may be partially replaced. Do NOT start the older build against it; ' + - `re-deploy ${options.record.active ?? 'the newer version'}, which can read what is there.` - } - } - - const targetDir = computeRemoteInstallDir(ORCAD_INSTALL_MODEL, options.remoteHome, safety.target) - await exec( - options, - orcadLaunchCommand(options.host, { - remoteInstallDir: targetDir, - nodePath: options.nodePath, - fullVersion: safety.target, - userDataDir: options.userDataDir, - bindHost: options.bindHost, - port: options.port - }) - ) - const deadline = Date.now() + (options.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS) - const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) - let parsed = parseOrcadReadinessOutput('') - while (Date.now() < deadline && parsed.state === 'pending') { - options.signal?.throwIfAborted() - parsed = parseOrcadReadinessOutput( - await exec(options, readOrcadReadinessCommand(options.host, targetDir)) - ) - if (parsed.state === 'pending') { - await sleep(READINESS_POLL_MS) - } - } - const verdict = evaluateOrcadActivation(parsed.state === 'ready' ? parsed.readiness : null, { - buildHash: options.targetBuildHash, - fullVersion: safety.target - }) - if (verdict.decision === 'reject') { - return { - outcome: 'failed', - code: verdict.code, - reason: - `The rollback target ${safety.target} did not come up healthy: ${verdict.reason} The ` + - `store has been restored to its pre-activation state. Its stderr is at ` + - `${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` - } - } - - // Why the record is written last: until the target is proven serving, `active` still names - // the version an operator would need to bring back, and `previous` still names this target. - await writeRelayFile( - options.conn, - options.host, - orcadActivationPath(options.host, options.remoteHome), - serializeOrcadActivationRecord(withRolledBackVersion(options.record, now())), - { signal: options.signal } - ) - return { - outcome: 'rolled-back', - target: safety.target, - discarded: safety.safety === 'lossy' ? safety.discards : [], - verdict - } } diff --git a/src/main/ssh/orcad-remote-runtime-control.test.ts b/src/main/ssh/orcad-remote-runtime-control.test.ts new file mode 100644 index 00000000000..d412e539cb1 --- /dev/null +++ b/src/main/ssh/orcad-remote-runtime-control.test.ts @@ -0,0 +1,57 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: (error: unknown) => + error instanceof Error && error.message === 'unconfirmed' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { OrcadLaunchSpec } from './orcad-remote-launch' + +const host = getRemoteHostPlatform('linux-arm64') +const spec: OrcadLaunchSpec = { + remoteInstallDir: '/root/.orca-remote/orcad-0.1.0+07d0995735e0', + nodePath: '/usr/bin/node', + fullVersion: '0.1.0+07d0995735e0', + userDataDir: '/root/.orca', + bindHost: '127.0.0.1', + port: 6768, + activationRoot: '/root/.orca-remote/.orcad-activation-transaction' +} +const READY = `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1' })}\n` +const mockExec = vi.mocked(execCommand) + +function launch() { + return launchOrcadAndAwaitReadiness( + { conn: Object.create(null), host, readinessTimeoutMs: 60_000, sleep: async () => {} }, + spec + ) +} + +describe('waiting for a launched candidate', () => { + beforeEach(() => { + mockExec.mockReset() + vi.spyOn(console, 'warn').mockImplementation(() => {}) + }) + + // BUG-17: one failed read of the readiness file failed the launch, and the activation then + // stopped a candidate that went ready a moment later. + it('retries a failed readiness read instead of failing the launch', async () => { + mockExec + .mockResolvedValueOnce('1786\n') + .mockRejectedValueOnce(new Error('(SSH) Channel open failure: open failed')) + .mockResolvedValueOnce(READY) + await expect(launch()).resolves.toMatchObject({ + state: 'ready', + readiness: { runtimeId: 'r1' } + }) + }) + + it('still fails at once on an unconfirmed termination, which may have run remotely', async () => { + mockExec.mockResolvedValueOnce('1786\n').mockRejectedValueOnce(new Error('unconfirmed')) + await expect(launch()).rejects.toThrow('unconfirmed') + }) +}) diff --git a/src/main/ssh/orcad-remote-runtime-control.ts b/src/main/ssh/orcad-remote-runtime-control.ts new file mode 100644 index 00000000000..590834b487b --- /dev/null +++ b/src/main/ssh/orcad-remote-runtime-control.ts @@ -0,0 +1,158 @@ +/** Launch a slot and wait for its readiness: the one loop deploy, rollback and recovery share. */ +import { ORCAD_STARTUP_READINESS_TIMEOUT_MS } from '../../shared/orcad-profile-preflight' +import { + orcadLaunchCommand, + parseOrcadReadinessOutput, + type OrcadLaunchSpec, + type OrcadReadinessParse +} from './orcad-remote-launch' +import { + readWindowsOrcadLaunchReport, + readWindowsOrcadSlotRuntime, + windowsOrcadLaunchCommand, + windowsOrcadLaunchRuntimeCommand +} from './orcad-remote-launch-windows' +import { + ORCAD_READINESS_WAIT_MAX_SECONDS, + orcadReadinessWaitCommand, + parseOrcadReadinessWaitOutput +} from './orcad-remote-readiness-wait' +import type { SshConnection } from './ssh-connection' +import { errorMessage } from '../../shared/error-message' +import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { + currentOrcadFence, + isOrcadFenceLost, + OrcadFenceLostError, + posixOrcadFenceGuard +} from './orcad-activation-fence-scope' +import { orcadRemoteBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { ORCAD_WINDOWS_FENCE_ARG } from './orcad-windows-host-fence-ops' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' + +// Only between host-side waits, so a host that answers early cannot turn this into a tight loop. +const READINESS_RETRY_PAUSE_MS = 1_000 + +export type OrcadRemoteExecTarget = { + conn: SshConnection + host: RemoteHostPlatform + signal?: AbortSignal + /** Locates the runtime store for host-record scripts; required on Windows, unused elsewhere. */ + remoteHome?: string +} + +/** Under a held activation fence, the step runs only while the host still names this run its owner. */ +export async function execOrcadRemote( + target: OrcadRemoteExecTarget, + command: string, + signal = target.signal +): Promise { + const fence = currentOrcadFence() + const run = (line: string): Promise => + execCommand(target.conn, line, { + wrapCommand: target.host.commandDialect !== 'powershell', + signal + }) + try { + if (!fence) { + return await run(command) + } + if (!isWindowsRemoteHost(target.host)) { + return await run(`${posixOrcadFenceGuard(fence)} ${command}`) + } + // A host op checks inside the host script; anything else is checked by one op just before it. + if (!command.includes(ORCAD_WINDOWS_FENCE_ARG) && target.remoteHome) { + const baseDir = orcadRemoteBaseDir(target.host, target.remoteHome) + await run(orcadWindowsHostOpCommand(target.host, baseDir, 'fence-check', [])) + } + return await run(command) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + throw isOrcadFenceLost(error) ? new OrcadFenceLostError() : error + } +} + +/** A confirmed failure reads as `fallback`; an unconfirmed one or a lost fence propagates. */ +export function execOrcadRemoteOr( + target: OrcadRemoteExecTarget, + command: string, + fallback = '' +): Promise { + return execOrcadRemote(target, command).catch((error: unknown) => { + if (isUnconfirmedSshCommandTermination(error) || error instanceof OrcadFenceLostError) { + throw error + } + return fallback + }) +} + +/** Recovery paths must finish even when the request that started them was cancelled. */ +export function withoutAbortSignal( + options: T +): Omit { + const { signal: _signal, ...rest } = options + return rest +} + +export async function launchOrcadAndAwaitReadiness( + target: OrcadRemoteExecTarget & { + readinessTimeoutMs?: number + sleep?: (ms: number) => Promise + }, + spec: OrcadLaunchSpec +): Promise { + if (isWindowsRemoteHost(target.host)) { + const slotRuntime = readWindowsOrcadSlotRuntime( + await execOrcadRemote( + target, + windowsOrcadLaunchRuntimeCommand(target.host, spec.remoteInstallDir) + ) + ) + readWindowsOrcadLaunchReport( + await execOrcadRemote(target, windowsOrcadLaunchCommand(target.host, spec, slotRuntime)) + ) + } else { + await execOrcadRemote(target, orcadLaunchCommand(target.host, spec)) + } + const deadline = Date.now() + (target.readinessTimeoutMs ?? ORCAD_STARTUP_READINESS_TIMEOUT_MS) + const sleep = target.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))) + let last = parseOrcadReadinessOutput('') + let lastWaitError: unknown + // At least one read: a client descheduled past a short deadline must not fail a ready launch. + for (let first = true; first || Date.now() < deadline; first = false) { + target.signal?.throwIfAborted() + const waitSeconds = Math.min( + ORCAD_READINESS_WAIT_MAX_SECONDS, + Math.ceil((deadline - Date.now()) / 1000) + ) + let output: string + try { + output = await execOrcadRemote( + target, + orcadReadinessWaitCommand(target.host, spec.remoteInstallDir, waitSeconds) + ) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error) || error instanceof OrcadFenceLostError) { + throw error + } + // Why retry: a failed read (a refused channel, a timed-out wait) says nothing about the + // launched process. Failing the launch here makes the caller stop a healthy candidate. + lastWaitError = error + console.warn(`[orcad] readiness wait failed; retrying: ${errorMessage(error)}`) + await sleep(READINESS_RETRY_PAUSE_MS) + continue + } + lastWaitError = undefined + last = parseOrcadReadinessWaitOutput(target.host, output) + if (last.state !== 'pending') { + return last + } + await sleep(READINESS_RETRY_PAUSE_MS) + } + if (lastWaitError !== undefined) { + throw lastWaitError + } + return last +} diff --git a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts index 4839fd7b2c2..3e9ff558707 100644 --- a/src/main/ssh/orcad-remote-shell-commands.integration.test.ts +++ b/src/main/ssh/orcad-remote-shell-commands.integration.test.ts @@ -26,6 +26,7 @@ import { openProfileStateDatabase } from '../persistence/profile-state/profile-s import { orcadLaunchCommand, + orcadLivenessAnswerBlocksGc, orcadLivenessProbeCommand, ORCAD_PID_FILENAME, ORCAD_READINESS_FILENAME, @@ -37,6 +38,11 @@ import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' +import { shellEscape } from './ssh-connection-utils' +import { + orcadReadinessWaitCommand, + parseOrcadReadinessWaitOutput +} from './orcad-remote-readiness-wait' import { captureOrcadStateSnapshotCommand, compareOrcadStateSnapshotCommand, @@ -49,11 +55,14 @@ import { restoreOrcadStateSnapshotCommand } from './orcad-state-snapshot' import { getRemoteHostPlatform } from './ssh-remote-platform' +import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' const host = getRemoteHostPlatform('linux-x64') let root = '' let dataDir = '' let snapshotDir = '' +// The ~/.orca-remote stand-in that holds the state-mutation lock. +const baseDir = (): string => join(root, '.orca-remote') let versionDir = '' const launchedPids = new Set() @@ -86,20 +95,39 @@ afterEach(() => { rmSync(root, { recursive: true, force: true }) }) -async function launchTestRuntime(legacyWrapper = false): Promise<{ +async function launchTestRuntime( + legacyWrapper = false, + stopRequests = false +): Promise<{ runtimePid: number recordedPid: number terminatedFile: string }> { const terminatedFile = join(versionDir, 'terminated') + const requestFile = join(versionDir, ORCAD_STOP_REQUEST_FILENAME) writeFileSync( join(versionDir, 'orcad.js'), [ + `const fs = require('node:fs');`, `process.on('SIGTERM', () => {`, - ` require('node:fs').writeFileSync(${JSON.stringify(terminatedFile)}, 'terminated');`, + ` fs.writeFileSync(${JSON.stringify(terminatedFile)}, 'terminated');`, ` process.exit(0);`, `});`, - `console.log(JSON.stringify({type: 'orca_server_ready', health: {pid: process.pid}}));`, + ...(stopRequests + ? [ + // Like orcad's listener: consume the slot request, then stop. + `setInterval(() => {`, + ` if (fs.existsSync(${JSON.stringify(requestFile)})) {`, + ` fs.unlinkSync(${JSON.stringify(requestFile)});`, + ` fs.writeFileSync(${JSON.stringify(terminatedFile)}, 'requested');`, + ` process.exit(0);`, + ` }`, + `}, 20);` + ] + : []), + `console.log(JSON.stringify({type: 'orca_server_ready', health: {pid: process.pid${ + stopRequests ? ', stopRequests: 1' : '' + }}}));`, `setTimeout(() => process.exit(1), 10_000);` ].join('\n') ) @@ -109,7 +137,8 @@ async function launchTestRuntime(legacyWrapper = false): Promise<{ fullVersion: '0.2.0+bb01', userDataDir: dataDir, bindHost: '127.0.0.1', - port: 0 + port: 0, + activationRoot: join(dataDir, '.orcad-activation-transaction') }) if (legacyWrapper) { // The trailing command retains the old macOS waiting-shell behavior on every POSIX shell. @@ -151,7 +180,7 @@ function stopTestRuntime(justLaunched = false): ReturnType { it('detects candidate SQLite migration without modifying current state or the snapshot', () => { - sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) const archive = readFileSync(join(snapshotDir, 'state.tar')) const compare = (): boolean => orcadSnapshotIsUnchanged(sh(compareOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) @@ -180,7 +209,7 @@ describe('state snapshot commands, run for real', () => { expect( orcadSnapshotIsUnchanged(sh(compareOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) ).toBe(false) - sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) writeFileSync(join(snapshotDir, 'state.tar'), 'not an archive') expect( orcadSnapshotIsUnchanged(sh(compareOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) @@ -198,7 +227,9 @@ describe('state snapshot commands, run for real', () => { symlinkSync(external, profile) expect( - parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + parseOrcadSnapshotCapture( + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) + ) ).toBe('failed') mkdirSync(snapshotDir, { recursive: true }) @@ -212,7 +243,9 @@ describe('state snapshot commands, run for real', () => { it('captures, then restores state the newer build overwrote', () => { expect( - parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + parseOrcadSnapshotCapture( + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) + ) ).toBe('captured') expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('PRESENT') @@ -221,7 +254,9 @@ describe('state snapshot commands, run for real', () => { writeFileSync(join(dataDir, 'profiles', 'p1', 'new-build-only.json'), '{}') expect( - parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) + ) ).toBe('restored') expect(readFileSync(join(dataDir, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"before"}') // Removed before extraction, so the older build never sees a file it cannot interpret. @@ -241,7 +276,9 @@ describe('state snapshot commands, run for real', () => { // the generated command reads the now-quiescent files. expect(existsSync(`${databasePath}-wal`)).toBe(true) expect( - parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + parseOrcadSnapshotCapture( + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) + ) ).toBe('captured') } finally { opened.db.close() @@ -257,7 +294,9 @@ describe('state snapshot commands, run for real', () => { changed.db.close() } expect( - parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir))) + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) + ) ).toBe('restored') const restored = openProfileStateDatabase(databasePath, 'p1') @@ -272,11 +311,11 @@ describe('state snapshot commands, run for real', () => { }) it('leaves the live daemon runtime dir untouched through capture and restore', () => { - sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + sh(captureOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) // The daemon is running across the rollback and rewrites its token; a restore that // reached /daemon would break the fence that keeps its terminals adoptable. writeFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'token-after-restart') - sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir)) + sh(restoreOrcadStateSnapshotCommand(host, dataDir, snapshotDir, baseDir())) expect(readFileSync(join(dataDir, 'daemon', 'daemon.sock.token'), 'utf8')).toBe( 'token-after-restart' ) @@ -286,7 +325,9 @@ describe('state snapshot commands, run for real', () => { const emptyRoot = join(root, 'fresh') mkdirSync(emptyRoot) expect( - parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, emptyRoot, snapshotDir))) + parseOrcadSnapshotCapture( + sh(captureOrcadStateSnapshotCommand(host, emptyRoot, snapshotDir, baseDir())) + ) ).toBe('empty') expect(sh(probeOrcadStateSnapshotCommand(host, snapshotDir)).trim()).toBe('ABSENT') }) @@ -294,7 +335,7 @@ describe('state snapshot commands, run for real', () => { it('reports MISSING rather than claiming a restore it did not perform', () => { expect( parseOrcadSnapshotRestore( - sh(restoreOrcadStateSnapshotCommand(host, dataDir, join(root, 'nope'))) + sh(restoreOrcadStateSnapshotCommand(host, dataDir, join(root, 'nope'), baseDir())) ) ).toBe('missing') }) @@ -310,7 +351,9 @@ describe('state snapshot commands, run for real', () => { mkdirSync(join(nasty, 'profiles'), { recursive: true }) writeFileSync(join(nasty, 'orca-profile-index.json'), '{"v":"quoted"}') expect( - parseOrcadSnapshotCapture(sh(captureOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + parseOrcadSnapshotCapture( + sh(captureOrcadStateSnapshotCommand(host, nasty, snapshotDir, baseDir())) + ) ).toBe('captured') expect( orcadSnapshotIsUnchanged(sh(compareOrcadStateSnapshotCommand(host, nasty, snapshotDir))) @@ -320,7 +363,9 @@ describe('state snapshot commands, run for real', () => { orcadSnapshotIsUnchanged(sh(compareOrcadStateSnapshotCommand(host, nasty, snapshotDir))) ).toBe(false) expect( - parseOrcadSnapshotRestore(sh(restoreOrcadStateSnapshotCommand(host, nasty, snapshotDir))) + parseOrcadSnapshotRestore( + sh(restoreOrcadStateSnapshotCommand(host, nasty, snapshotDir, baseDir())) + ) ).toBe('restored') expect(readFileSync(join(nasty, 'orca-profile-index.json'), 'utf8')).toBe('{"v":"quoted"}') }) @@ -338,6 +383,21 @@ describe('liveness and stop commands, run for real', () => { expect(stopTestRuntime()).toBe('already-exited') }) + it('stops a build that consumes stop requests by request file, not by signal', async () => { + const { terminatedFile } = await launchTestRuntime(false, true) + expect(stopTestRuntime()).toBe('stopped') + expect(readFileSync(terminatedFile, 'utf8')).toBe('requested') + expect(existsSync(join(versionDir, ORCAD_STOP_REQUEST_FILENAME))).toBe(false) + }) + + it('clears a stop request the previous process never consumed before launching', async () => { + writeFileSync(join(versionDir, ORCAD_STOP_REQUEST_FILENAME), '') + const { runtimePid } = await launchTestRuntime(false, true) + await new Promise((resolve) => setTimeout(resolve, 100)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('LIVE') + expect(runtimePid).toBeGreaterThan(1) + }) + it('refuses a legacy wrapper PID both before and after its shell exits', async () => { const { runtimePid, recordedPid, terminatedFile } = await launchTestRuntime(true) expect(recordedPid).not.toBe(runtimePid) @@ -400,13 +460,21 @@ describe('liveness and stop commands, run for real', () => { })}` ) ) - expect(outcome).toBe('unknown') + // After a delivered SIGTERM the host may still be changing, so it is not "nothing happened". + expect(outcome).toBe(phase === 'before' ? 'unknown' : 'unconfirmed') expect(orcadStopFreedTheHost(outcome)).toBe(false) expect(() => process.kill(runtimePid, 0)).not.toThrow() expect(existsSync(terminatedFile)).toBe(false) }) it('reports UNKNOWN with no pid file, and DEAD for a pid that has exited', () => { + // A slot installed but never launched: only GC reads it apart from UNKNOWN. + rmSync(join(versionDir, ORCAD_READINESS_FILENAME), { force: true }) + const neverLaunched = sh(orcadLivenessProbeCommand(host, versionDir)) + expect(parseOrcadLiveness(neverLaunched)).toBe('UNKNOWN') + expect(orcadLivenessAnswerBlocksGc(neverLaunched)).toBe(false) + writeFileSync(join(versionDir, ORCAD_READINESS_FILENAME), '') + expect(orcadLivenessAnswerBlocksGc(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe(true) expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') writeFileSync(join(versionDir, ORCAD_PID_FILENAME), 'not-a-pid') expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('UNKNOWN') @@ -416,8 +484,35 @@ describe('liveness and stop commands, run for real', () => { expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') }) + it('reports a permission-denied liveness probe as UNKNOWN in any locale', () => { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), '4242') + const deniedKill = + 'kill() { if [ "$LC_ALL" = C ]; then echo "kill: Operation not permitted" >&2; ' + + 'else echo "kill: Vorgang nicht zulässig" >&2; fi; return 1; };' + expect( + parseOrcadLiveness( + sh(`LC_ALL=de_DE.UTF-8; ${deniedKill} ${orcadLivenessProbeCommand(host, versionDir)}`) + ) + ).toBe('UNKNOWN') + }) + + it('reports a reused PID running something other than this slot as DEAD', () => { + const stranger = spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' }) + try { + writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(stranger.pid)) + expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('DEAD') + } finally { + stranger.kill('SIGKILL') + } + }) + it('reports LIVE for a running process and stops it with SIGTERM', async () => { - const child = spawn('/bin/sh', ['-c', 'sleep 30'], { stdio: 'ignore' }) + // Stands in for orcad: its command line runs this slot's orcad.js. + const child = spawn( + process.execPath, + ['-e', 'setTimeout(() => {}, 30000)', join(versionDir, 'orcad.js')], + { stdio: 'ignore' } + ) try { writeFileSync(join(versionDir, ORCAD_PID_FILENAME), String(child.pid)) expect(parseOrcadLiveness(sh(orcadLivenessProbeCommand(host, versionDir)))).toBe('LIVE') @@ -477,3 +572,43 @@ describe('liveness and stop commands, run for real', () => { ).toBe('no-pid') }) }) + +describe('host-side readiness wait, run for real', () => { + const readiness = () => join(versionDir, ORCAD_READINESS_FILENAME) + const line = `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1' })}\n` + + it('returns a finished line without waiting out its bound', () => { + writeFileSync(readiness(), line) + const started = Date.now() + const result = parseOrcadReadinessWaitOutput( + host, + sh(orcadReadinessWaitCommand(host, versionDir, 10)) + ) + expect(Date.now() - started).toBeLessThan(5_000) + expect(result).toMatchObject({ state: 'ready', readiness: { runtimeId: 'r1' } }) + }) + + it('waits for a line still being written, and answers pending when its bound ends', () => { + writeFileSync(readiness(), line.slice(0, 10)) + // The writer finishes after the wait starts; the shell must pick it up mid-wait. + sh( + `(sleep 1; printf '%s\\n' ${shellEscape(line.trimEnd().slice(10))} >> ${shellEscape(readiness())}) >/dev/null 2>&1 &` + ) + expect( + parseOrcadReadinessWaitOutput(host, sh(orcadReadinessWaitCommand(host, versionDir, 10))) + ).toMatchObject({ state: 'ready' }) + writeFileSync(readiness(), '{"type":"orca_ser') + expect( + parseOrcadReadinessWaitOutput(host, sh(orcadReadinessWaitCommand(host, versionDir, 1))) + ).toEqual({ state: 'pending' }) + }) + + it('reads a missing file as pending', () => { + expect( + parseOrcadReadinessWaitOutput( + host, + sh(orcadReadinessWaitCommand(host, `${versionDir}-none`, 0)) + ) + ).toEqual({ state: 'pending' }) + }) +}) diff --git a/src/main/ssh/orcad-remote-stop.ts b/src/main/ssh/orcad-remote-stop.ts new file mode 100644 index 00000000000..345926b0d72 --- /dev/null +++ b/src/main/ssh/orcad-remote-stop.ts @@ -0,0 +1,138 @@ +/** + * Decommissioning a managed orcad: stop its active instance and record that nothing serves. + * + * Runs under the activation fence and journal, so an interrupted decommission recovers like an + * activation does. It refuses while the terminal census says terminals are live or cannot be + * counted, and it never signals: the instance-bound request reaches only the orcad it names. + * Only proven exit deactivates the record; anything less withdraws the request or keeps the + * fence. Windows runs the same steps: the request is a staged file and exit proof is orcad's own. + */ +import { randomUUID } from 'node:crypto' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { sameOrcadActivationRecord } from './orcad-activation-transaction' +import { writeOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { withOrcadActivationLock } from './orcad-activation-lock' +import { orcadActivationFenceRefusal } from './orcad-activation-fence-hold' +import { + createOrcadDecommissionTransaction, + withOrcadDecommissionProcessExited, + withOrcadDecommissionStopDispatched +} from './orcad-decommission-transaction' +import { readRemoteOrcadManagedStopTarget } from './orcad-managed-remote-stop' +import { settleOrcadDecommissionStop } from './orcad-decommission-stop' +import type { OrcadSlotOptions } from './orcad-recovery-slot' +import type { OrcadDecommissionResult } from '../../shared/orcad-decommission' +import type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' + +export type OrcadDecommissionOptions = OrcadSlotOptions & { + /** The record the caller reviewed; a changed host record refuses rather than guessing. */ + record: OrcadActivationRecord + /** Taken by the caller just before; any live or uncounted terminal refuses. */ + census: OrcadTerminalCensus + now?: () => Date +} + +type Refusal = Extract + +function refuse(verdict: Refusal['verdict'], code: string, reason: string): Refusal { + return { outcome: 'refused', verdict, code, reason } +} + +function censusRefusal(census: OrcadTerminalCensus): Refusal | null { + if (census.liveSessions === null) { + return refuse( + 'unverifiable', + 'orcad_decommission_census_unavailable', + 'The terminal daemon did not answer a session count, so decommissioning could end ' + + 'running work. Retry when the host answers.' + ) + } + if (census.liveSessions > 0) { + return refuse( + 'live', + 'orcad_decommission_terminals_running', + `${census.liveSessions} terminal${census.liveSessions === 1 ? ' is' : 's are'} still ` + + 'running on this host. Close them before decommissioning the server.' + ) + } + return null +} + +export async function decommissionRemoteOrcad( + options: OrcadDecommissionOptions +): Promise { + const now = options.now ?? ((): Date => new Date()) + return withOrcadActivationLock( + options, + async (lock) => { + const record = await readOrcadActivationRecord(options) + if (!sameOrcadActivationRecord(record, options.record)) { + return refuse( + 'unverifiable', + 'orcad_decommission_record_changed', + 'The host activation record changed since it was reviewed. Refresh and try again.' + ) + } + const activeVersion = record.active + if (!activeVersion) { + return refuse( + 'unverifiable', + 'orcad_decommission_nothing_active', + 'No orcad version is active on this host, so there is nothing to decommission.' + ) + } + const census = censusRefusal(options.census) + if (census) { + return census + } + const target = await readRemoteOrcadManagedStopTarget(options, activeVersion) + if (target.state === 'refused') { + return refuse(target.verdict, target.code, target.reason) + } + + let transaction = createOrcadDecommissionTransaction({ + transactionId: randomUUID(), + recordBefore: { ...record, active: activeVersion }, + now: now() + }) + await writeOrcadActivationTransaction(options, transaction) + lock.retainOnError() + const request = { + schemaVersion: 1 as const, + transactionId: transaction.transactionId, + ...target.context, + // Best effort: an idle daemon goes with orcad, a busy one keeps its terminals. + retireIdleDaemon: true as const + } + // Durable before it can reach the host, so recovery can settle exactly this request. + transaction = withOrcadDecommissionStopDispatched(transaction, request, now()) + await writeOrcadActivationTransaction(options, transaction) + + const settlement = await settleOrcadDecommissionStop(options, request) + if (settlement.state === 'withdrawn') { + // orcad never acted on it and keeps serving; the record never changed. + return refuse(settlement.verdict, 'orcad_decommission_stop_withdrawn', settlement.reason) + } + if (settlement.state === 'unsettled') { + lock.retain() + return refuse(settlement.verdict, 'orcad_decommission_stop_unsettled', settlement.reason) + } + transaction = withOrcadDecommissionProcessExited(transaction, now()) + await writeOrcadActivationTransaction(options, transaction) + await writeOrcadActivationRecord(options, transaction.recordAfter) + return { + outcome: 'decommissioned', + version: activeVersion, + retirement: settlement.retirement + } + }, + async () => { + const refusal = await orcadActivationFenceRefusal(options, 'stop') + return refuse('unverifiable', refusal.code, refusal.reason) + } + ) +} diff --git a/src/main/ssh/orcad-remote-windows-commands.test.ts b/src/main/ssh/orcad-remote-windows-commands.test.ts new file mode 100644 index 00000000000..013000dc955 --- /dev/null +++ b/src/main/ssh/orcad-remote-windows-commands.test.ts @@ -0,0 +1,363 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import type { SshConnection } from './ssh-connection' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { orcadLivenessProbeCommand, type OrcadLaunchSpec } from './orcad-remote-launch' +import { + OrcadWindowsLaunchRefusedError, + readWindowsOrcadLaunchReport, + windowsOrcadLaunchCommand, + windowsOrcadLaunchRuntimeCommand +} from './orcad-remote-launch-windows' +import { parseOrcadStopOutcome, stopOrcadCommand } from './orcad-remote-process-control' +import { orcadReadinessWaitCommand } from './orcad-remote-readiness-wait' +import { remoteOrcadBuildHashCommand, readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { + readBoundedOrcadRemoteRecord, + writeAtomicOrcadRemoteRecord +} from './orcad-remote-record-file' +import { launchOrcadAndAwaitReadiness } from './orcad-remote-runtime-control' +import { probeActiveOrcadReadiness } from './orcad-active-readiness' +import { completeRemoteOrcadManagedStop } from './orcad-managed-remote-stop' +import { + installOrcadWindowsHostScript, + orcadWindowsNodeCommandLine, + orcadWindowsPinnedNodePath, + OrcadWindowsCommandLineError +} from './orcad-remote-windows-node' +import { + ORCAD_WINDOWS_HOST_SCRIPT, + ORCAD_WINDOWS_HOST_SCRIPT_FILENAME +} from './orcad-windows-host-script' +import type { OrcadSlotOptions } from './orcad-recovery-slot' +import { NODE_RUNTIME_ASSETS } from '../../shared/node-runtime-pin' + +const mockExec = vi.mocked(execCommand) +const host = getRemoteHostPlatform('win32-x64') +const base = 'C:/Users/u/.orca-remote' +const slot = `${base}/orcad-0.2.0+bb01` +const sha = NODE_RUNTIME_ASSETS['win32-x64'].executableSha256 +const NODE = `C:\\Users\\u\\.orca-remote\\runtimes\\node-${sha}\\node.exe` +const SCRIPT = `${base}/${ORCAD_WINDOWS_HOST_SCRIPT_FILENAME}` +const SLOT_NODE = 'C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe' +const spec: OrcadLaunchSpec = { + remoteInstallDir: slot, + nodePath: 'C:/host/node.exe', + fullVersion: '0.2.0+bb01', + userDataDir: 'C:/Users/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + activationRoot: `${base}/.orcad-activation-transaction` +} + +function windowsConn(): { conn: SshConnection; writes: [string, string][] } { + const writes: [string, string][] = [] + const conn: SshConnection = Object.assign(Object.create(null), { + writeFile: async (path: string, contents: string) => { + writes.push([path, contents]) + } + }) + return { conn, writes } +} + +function encoded(marker: string, value: string): string { + return `${marker} ${Buffer.from(value).toString('base64')}\r\n` +} + +beforeEach(() => { + mockExec.mockReset() +}) + +describe('Windows orcad commands run node.exe directly', () => { + const commands = (): [string, string][] => [ + ['launch runtime', windowsOrcadLaunchRuntimeCommand(host, slot)], + ['launch', windowsOrcadLaunchCommand(host, spec, SLOT_NODE)], + ['readiness wait', orcadReadinessWaitCommand(host, slot, 20)], + ['liveness', orcadLivenessProbeCommand(host, slot)], + ['stop', stopOrcadCommand(host, slot, { waitSeconds: 20, nodePath: spec.nodePath })], + ['build hash', remoteOrcadBuildHashCommand(host, slot)] + ] + + it.each(commands())('%s: no PowerShell hop, no encoding, no WMI or signal', (_name, command) => { + expect(command).toMatch(/^C:\\Users\\u\\\.orca-remote\\runtimes\\node-[0-9a-f]+\\node\.exe /u) + expect(command).not.toMatch( + /EncodedCommand|powershell|pwsh|cmd\.exe|ExecutionPolicy|Cim|Wmi|taskkill|SIGTERM|kill -/iu + ) + // Nothing either DefaultShell expands, and nothing a script would need quoting for. + expect(command).not.toMatch(/[%$`']/u) + }) + + it('host ops use the client pin; only the launch uses the slot runtime', () => { + expect(orcadWindowsPinnedNodePath(host, base)).toBe(`${base}/runtimes/node-${sha}/node.exe`) + expect(orcadLivenessProbeCommand(host, slot)).toBe(`${NODE} ${SCRIPT} liveness ${slot}`) + expect(stopOrcadCommand(host, slot, { waitSeconds: 20, justLaunched: true })).toBe( + `${NODE} ${SCRIPT} stop ${slot} "20" "1"` + ) + }) + + it('golden: the two launch lines', () => { + expect(windowsOrcadLaunchRuntimeCommand(host, slot)).toBe( + `${NODE} ${SCRIPT} slot-runtime ${slot} clear-stop-request` + ) + expect(windowsOrcadLaunchCommand(host, spec, SLOT_NODE)).toMatchInlineSnapshot( + `"C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.js --windows-breakaway-launch --stdout-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-readiness --stderr-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/orcad.log --stderr-keep-previous --process-file C:/Users/u/.orca-remote/orcad-0.2.0+bb01/.orcad-process.json --env ORCA_VERSION=0.2.0+bb01 --env ORCA_USER_DATA=C:/Users/u/.orca --env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args --json --bind "127.0.0.1" --port "7777""` + ) + }) + + it('passes the managed idle-exit fence through the breakaway environment', () => { + const command = windowsOrcadLaunchCommand( + host, + { ...spec, activationRoot: 'C:/Users/u/.orca-remote/.orcad-activation-transaction' }, + SLOT_NODE + ) + expect(command).toContain( + '--env ORCA_ORCAD_MANAGED_ACTIVATION_ROOT=C:/Users/u/.orca-remote/.orcad-activation-transaction --orcad-args' + ) + expect(command).not.toMatch(/[%$`']/u) + }) + + it('never polls across SSH: runtime, launch, then one host-side wait', async () => { + mockExec + .mockResolvedValueOnce(encoded('__ORCAD_RUNTIME__', SLOT_NODE)) + .mockResolvedValueOnce( + 'ORCA_ORCAD_LAUNCH {"method":"breakaway","pid":4242,"inJob":false}\r\n' + ) + .mockResolvedValueOnce( + encoded( + '__ORCAD_READINESS__', + `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1' })}\n` + ) + ) + const sleep = vi.fn(async () => {}) + const result = await launchOrcadAndAwaitReadiness( + { conn: Object.create(null), host, readinessTimeoutMs: 60_000, sleep }, + spec + ) + expect(result).toMatchObject({ state: 'ready', readiness: { runtimeId: 'r1' } }) + expect(mockExec).toHaveBeenCalledTimes(3) + expect(String(mockExec.mock.calls[1]?.[1]).startsWith(`${SLOT_NODE} `)).toBe(true) + expect(sleep).not.toHaveBeenCalled() + }) +}) + +describe('the active-slot readiness probe on Windows', () => { + it('reads readiness through the host script instead of the POSIX head command', async () => { + mockExec + .mockResolvedValueOnce('LIVE\r\n') + .mockResolvedValueOnce( + encoded( + '__ORCAD_READINESS__', + `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1' })}\n` + ) + ) + const probe = probeActiveOrcadReadiness( + { conn: Object.create(null), host, remoteInstallDir: slot }, + { buildHash: 'bb01', fullVersion: '0.2.0+bb01' } + ) + + // The identity gate may still refuse this payload; the point is that the host is asked. + await probe.catch((error: unknown) => { + expect(error).not.toMatchObject({ name: 'OrcadRemoteLaunchUnsupportedError' }) + }) + expect(mockExec.mock.calls[1]?.[1]).toBe( + `${NODE} ${SCRIPT} readiness-wait ${slot}/.orcad-readiness "262144" "0"` + ) + }) +}) + +describe('Windows command lines for both DefaultShells', () => { + it('double-quotes arguments with spaces or leading digits, and passes the rest bare', () => { + expect( + orcadWindowsNodeCommandLine('C:/rt/node.exe', ['C:/Users/Ann Lee/x', '20', 'stop']) + ).toBe('C:\\rt\\node.exe "C:/Users/Ann Lee/x" "20" stop') + }) + + it('falls back to one unencoded powershell -Command when node.exe itself needs quoting', () => { + expect( + orcadWindowsNodeCommandLine("C:/Users/Ann O'Lee/rt/node.exe", [ + "C:/Users/Ann O'Lee/x", + 'stop' + ]) + ).toBe( + `powershell.exe -NoProfile -NonInteractive -Command "& 'C:\\Users\\Ann O''Lee\\rt\\node.exe' 'C:/Users/Ann O''Lee/x' 'stop'"` + ) + }) + + it.each(['C:/Users/100%/x', 'C:/Users/$me/x', 'C:/Users/a`b/x'])( + 'refuses %s rather than letting a shell expand it', + (value) => { + expect(() => orcadWindowsNodeCommandLine('C:/rt/node.exe', [value])).toThrow( + OrcadWindowsCommandLineError + ) + } + ) +}) + +describe('no -EncodedCommand in the W1 builders', () => { + it.each([ + 'orcad-remote-windows-node.ts', + 'orcad-windows-host-script.ts', + 'orcad-remote-launch-windows.ts', + 'orcad-remote-liveness-windows.ts', + 'orcad-remote-process-control-windows.ts', + 'orcad-remote-readiness-wait.ts', + 'orcad-remote-record-file.ts', + 'orcad-remote-build-hash.ts', + 'orcad-managed-remote-stop.ts', + 'orcad-remote-runtime-control.ts' + ])('%s', (file) => { + const code = readFileSync(join(__dirname, file), 'utf8') + .replace(/\/\*[\s\S]*?\*\//gu, '') + .replace(/^\s*\/\/.*$/gmu, '') + expect(code).not.toMatch(/EncodedCommand|powerShellCommand/u) + }) +}) + +describe('Windows launch report', () => { + it('reads a PID, a refusal, or a failure', () => { + expect( + readWindowsOrcadLaunchReport('ORCA_ORCAD_LAUNCH {"method":"breakaway","pid":9,"inJob":false}') + ).toBe(9) + expect(() => + readWindowsOrcadLaunchReport( + 'ORCA_ORCAD_LAUNCH {"method":"unavailable","reason":"breakaway-denied","step":"create-process","code":5}' + ) + ).toThrow(OrcadWindowsLaunchRefusedError) + expect(() => + readWindowsOrcadLaunchReport( + 'ORCA_ORCAD_LAUNCH {"method":"failed","reason":"failed","step":"open-stdout","code":32}' + ) + ).toThrow('open-stdout (code 32)') + // The relay's report is not orcad's. + expect(() => + readWindowsOrcadLaunchReport('ORCA_RELAY_LAUNCH {"method":"breakaway","pid":9,"inJob":false}') + ).toThrow('no launch report') + }) + + it('reads UNSUPPORTED as a stop refusal', () => { + expect(parseOrcadStopOutcome('UNSUPPORTED')).toBe('unsupported') + }) +}) + +describe('Windows build hash and host script', () => { + it('reads the same marker as POSIX', async () => { + mockExec.mockResolvedValueOnce('__ORCAD_BUILD_HASH__ ABC123DEF4567890\r\n') + await expect(readRemoteOrcadBuildHash({ conn: Object.create(null), host }, slot)).resolves.toBe( + 'abc123def4567890' + ) + }) + + it('stages a missing host script through a partial file that installs itself, once per connection', async () => { + const { conn, writes } = windowsConn() + mockExec + .mockRejectedValueOnce(new Error('Cannot find module')) + .mockResolvedValueOnce('ORCAD_HOST_SCRIPT_PRESENT\r\n') + await installOrcadWindowsHostScript({ conn, host }, base) + await installOrcadWindowsHostScript({ conn, host }, base) + expect(writes).toHaveLength(1) + const [partial, contents] = writes[0] ?? [] + expect(contents).toBe(ORCAD_WINDOWS_HOST_SCRIPT) + expect(String(mockExec.mock.calls[0]?.[1])).toBe(`${NODE} ${SCRIPT} script-present`) + expect(String(mockExec.mock.calls[1]?.[1])).toBe(`${NODE} ${partial} script-install ${SCRIPT}`) + expect(mockExec).toHaveBeenCalledTimes(2) + }) + + it('never rewrites a host script that is already present', async () => { + const { conn, writes } = windowsConn() + mockExec.mockResolvedValueOnce('ORCAD_HOST_SCRIPT_PRESENT\r\n') + await installOrcadWindowsHostScript({ conn, host }, base) + expect(writes).toEqual([]) + }) +}) + +describe('Windows host records', () => { + it('decodes a present record and reads absent as absent', async () => { + const { conn } = windowsConn() + const target = { conn, host, remoteHome: 'C:/Users/u' } + mockExec.mockResolvedValueOnce(encoded('__ORCAD_RECORD_PRESENT__', '{"owner":"Zoë"}')) + await expect(readBoundedOrcadRemoteRecord(target, 'C:/r.json', 64)).resolves.toEqual({ + state: 'present', + raw: '{"owner":"Zoë"}' + }) + expect(String(mockExec.mock.calls[0]?.[1])).toBe(`${NODE} ${SCRIPT} record-read C:/r.json "64"`) + mockExec.mockResolvedValueOnce('__ORCAD_RECORD_ABSENT__\r\n') + await expect(readBoundedOrcadRemoteRecord(target, 'C:/r.json', 64)).resolves.toEqual({ + state: 'absent' + }) + mockExec.mockResolvedValueOnce('') + await expect(readBoundedOrcadRemoteRecord(target, 'C:/r.json', 64)).rejects.toThrow( + 'no verifiable answer' + ) + }) + + it('refuses a Windows record without the remote home that locates node.exe', async () => { + await expect( + readBoundedOrcadRemoteRecord({ conn: Object.create(null), host }, 'C:/r.json', 64) + ).rejects.toThrow('remote home') + }) + + it('stages the contents as a file and never puts them on a command line', async () => { + const { conn, writes } = windowsConn() + mockExec.mockResolvedValueOnce('') + const contents = '{"secret-ish":"record body"}' + await writeAtomicOrcadRemoteRecord( + { conn, host, remoteHome: 'C:/Users/u' }, + 'C:/r.json', + contents + ) + expect(writes).toHaveLength(1) + expect(writes[0]?.[0]).toMatch(/^C:\/r\.json\.partial\./u) + expect(writes[0]?.[1]).toBe(contents) + const command = String(mockExec.mock.calls[0]?.[1]) + expect(command).not.toContain('record body') + expect(command.startsWith(`${NODE} ${SCRIPT} record-publish `)).toBe(true) + }) +}) + +describe('Windows managed stop', () => { + it('hands orcad a staged request file, not JSON on argv', async () => { + const { conn, writes } = windowsConn() + const request = { + schemaVersion: 1 as const, + transactionId: '0b9f6a3e-9e2c-4c8e-8f58-4c0f6b1d2e3a', + version: '0.2.0+bb01', + runtimeId: 'r1', + instance: { pid: 9, startedAtMs: 5, nonce: 'n', lockPath: 'C:/Users/u/.orca/orcad.lock' } + } + mockExec + .mockResolvedValueOnce(encoded('__ORCAD_RUNTIME__', SLOT_NODE)) + .mockResolvedValueOnce( + `${JSON.stringify({ ...request, kind: 'orcad_managed_stop_completion', verdict: 'live', receiptPersisted: false })}\r\n` + ) + .mockResolvedValueOnce('') + const options: OrcadSlotOptions = { + conn, + host, + remoteHome: 'C:/Users/u', + nodePath: 'C:/host/node.exe', + userDataDir: 'C:/Users/u/.orca', + bindHost: '127.0.0.1', + port: 7777 + } + await expect(completeRemoteOrcadManagedStop(options, request)).resolves.toMatchObject({ + verdict: 'live' + }) + const [stagedPath, stagedBody] = writes[0] ?? ['', ''] + expect(JSON.parse(stagedBody)).toEqual(request) + expect(String(mockExec.mock.calls[0]?.[1])).toBe(`${NODE} ${SCRIPT} slot-runtime ${slot}`) + const run = String(mockExec.mock.calls[1]?.[1]) + expect(run).not.toContain(request.transactionId) + expect(run).toBe( + `${SLOT_NODE} ${slot}/orcad.js --complete-managed-stop --request-file ${stagedPath}` + ) + expect(String(mockExec.mock.calls[2]?.[1])).toBe(`${NODE} ${SCRIPT} remove-file ${stagedPath}`) + }) +}) diff --git a/src/main/ssh/orcad-remote-windows-decommission.test.ts b/src/main/ssh/orcad-remote-windows-decommission.test.ts new file mode 100644 index 00000000000..c923d4012d8 --- /dev/null +++ b/src/main/ssh/orcad-remote-windows-decommission.test.ts @@ -0,0 +1,175 @@ +/** + * Decommission and GC against a Windows host whose host ops are answered by a fake: the stop is + * a staged managed request run by the slot's node.exe, and GC screens every candidate in one + * node.exe instead of one per version dir. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as RecordFile from './orcad-remote-record-file' +import type * as InstallLock from './ssh-relay-install-lock' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ + execCommand: vi.fn(), + isUnconfirmedSshCommandTermination: () => false +})) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn().mockResolvedValue(undefined) +})) +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal>()), + gcOldRemoteInstallVersions: vi.fn().mockResolvedValue(undefined) +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' +import { gcOldRemoteInstallVersions } from './ssh-relay-versioned-install' +import { decommissionRemoteOrcad } from './orcad-remote-stop' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { emptyOrcadActivationRecord, withActivatedVersion } from './orcad-activation-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const mockExec = vi.mocked(execCommand) +const host = getRemoteHostPlatform('win32-x64') +const VERSION = '0.2.0+bb01' +const SLOT_NODE = 'C:\\Users\\u\\.orca-remote\\runtimes\\node-ab\\node.exe' +const record = withActivatedVersion(emptyOrcadActivationRecord(), VERSION, null, new Date(0)) +const encoded = (marker: string, value: string): string => + `${marker} ${Buffer.from(value).toString('base64')}\r\n` +const lock = { + pid: 4242, + startedAtMs: 1_700_000_000_123, + identity: 'u', + version: VERSION, + acquiredAt: '2026-10-01T00:00:00.000Z', + nonce: 'nonce-1' +} + +function conn() { + return Object.assign(Object.create(null), { writeFile: vi.fn().mockResolvedValue(undefined) }) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('decommissioning a Windows orcad', () => { + it('stops it by a staged managed request and records that nothing serves', async () => { + const log: string[] = [] + mockExec.mockImplementation(async (_conn, command: string) => { + const text = String(command) + log.push(text) + const op = /\.js (?:--fence \S+ \S+ )?([a-z-]+)(?: |$)/u.exec(text)?.[1] ?? '' + if (op === 'record-read') { + if (text.includes('orcad-active.json')) { + return encoded('__ORCAD_RECORD_PRESENT__', JSON.stringify(record)) + } + return text.includes('orcad.lock') + ? encoded('__ORCAD_RECORD_PRESENT__', JSON.stringify(lock)) + : '__ORCAD_RECORD_ABSENT__\r\n' + } + if (op === 'readiness-wait') { + return encoded( + '__ORCAD_READINESS__', + `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1', health: { pid: 4242, stopRequests: 1 } })}\n` + ) + } + if (op === 'slot-runtime') { + return encoded('__ORCAD_RUNTIME__', SLOT_NODE) + } + if (op === 'fence-check') { + return 'OK' + } + if (op === 'fence-release') { + return 'RELEASED' + } + if (op === 'remove-file' || op === 'remove-tree') { + return '' + } + if (text.includes('--complete-managed-stop')) { + const staged = /--request-file (\S+)/u.exec(text)?.[1] ?? '' + const writes = vi.mocked(options.conn.writeFile).mock.calls + const request = JSON.parse(String(writes.find(([path]) => path === staged)?.[1])) + return `${JSON.stringify({ ...request, kind: 'orcad_managed_stop_completion', verdict: 'exited', receiptPersisted: true, retirement: 'retired' })}\r\n` + } + throw new Error(`unexpected Windows command: ${text}`) + }) + const options = { + conn: conn(), + host, + remoteHome: 'C:/Users/u', + nodePath: 'C:/host/node.exe', + userDataDir: 'C:/Users/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + record, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, + now: () => new Date('2026-10-02T00:00:00.000Z') + } + expect(await decommissionRemoteOrcad(options)).toEqual({ + outcome: 'decommissioned', + version: VERSION, + retirement: 'retired' + }) + const written = vi + .mocked(writeAtomicOrcadRemoteRecord) + .mock.calls.find(([, path]) => path.endsWith('orcad-active.json')) + expect(JSON.parse(written?.[2] ?? '{}')).toMatchObject({ active: null, previous: VERSION }) + for (const command of log) { + expect(command).not.toMatch(/EncodedCommand|kill |SIGTERM|taskkill|Stop-Process/u) + expect(command).not.toContain('nonce-1') + } + expect(log.some((command) => command.startsWith(`${SLOT_NODE} `))).toBe(true) + }) +}) + +describe('Windows orcad GC', () => { + const gcOptions = () => ({ + conn: conn(), + host, + remoteHome: 'C:/Users/u', + currentDirAbsPath: 'C:/Users/u/.orca-remote/orcad-0.3.0+cc01', + record + }) + + async function screen(answer: string | Error): Promise { + mockExec.mockImplementation(async (_conn, command: string) => { + if (String(command).includes(' record-read ')) { + return '__ORCAD_RECORD_ABSENT__\r\n' + } + // The activation fence probe is the relay's shared lock check. + if (String(command).startsWith('powershell.exe ')) { + return 'OPEN' + } + if (answer instanceof Error) { + throw answer + } + return answer + }) + await gcOldOrcadVersions(gcOptions()) + const passOptions = vi.mocked(gcOldRemoteInstallVersions).mock.calls[0]?.[5] + return ( + (await passOptions?.resolveExtraPinnedDirNames?.(['orcad-a', 'orcad-b', 'orcad-c'])) ?? null + ) + } + + it('screens every candidate in one node.exe and pins all but the proven dead', async () => { + expect(await screen('__ORCAD_LIVENESS__ LIVE,DEAD,UNKNOWN\r\n')).toEqual(['orcad-a', 'orcad-c']) + const screens = mockExec.mock.calls.filter(([, command]) => + String(command).includes('liveness-many') + ) + expect(screens).toHaveLength(1) + expect(String(screens[0]?.[1])).toContain( + 'C:/Users/u/.orca-remote/orcad-a C:/Users/u/.orca-remote/orcad-b C:/Users/u/.orca-remote/orcad-c' + ) + }) + + it('deletes nothing when the host cannot answer for every candidate', async () => { + expect(await screen('__ORCAD_LIVENESS__ DEAD,DEAD\r\n')).toBeNull() + vi.clearAllMocks() + expect(await screen(new Error('exit 1'))).toBeNull() + }) +}) diff --git a/src/main/ssh/orcad-remote-windows-node.ts b/src/main/ssh/orcad-remote-windows-node.ts new file mode 100644 index 00000000000..aad7b545e38 --- /dev/null +++ b/src/main/ssh/orcad-remote-windows-node.ts @@ -0,0 +1,193 @@ +/** + * Running node.exe on a Windows orcad host straight from the SSH exec, with plain argv. + * + * sshd hands the command to its DefaultShell, cmd.exe on a stock install or PowerShell when an + * admin sets it, and Orca does not probe which. So the line is built from the subset both parse + * the same way: an unquoted executable path, then arguments that are bare or double-quoted and + * contain nothing either shell expands (`%`, `$`, backtick, `"`). When the executable path + * itself needs quoting (a profile with a space), the line falls back to one unencoded + * `powershell.exe -Command`, which both shells pass through intact. Anything else is refused + * rather than encoded (docs/reference/windows-edr-posture.md). + */ +import { currentOrcadFence } from './orcad-activation-fence-scope' +import { ORCAD_WINDOWS_FENCE_ARG } from './orcad-windows-host-fence-ops' +import { + ORCAD_NODE_RUNTIME_DIR_PREFIX, + ORCAD_NODE_RUNTIME_WINDOWS_EXECUTABLE, + ORCAD_RUNTIMES_DIRNAME +} from '../../shared/orcad-artifacts' +import { pinnedNodeRuntimeAsset, type NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import { randomUUID } from 'node:crypto' +import type { SshConnection } from './ssh-connection' +import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { RELAY_REMOTE_DIR } from './relay-protocol' +import { joinRemotePath, remoteDirname, type RemoteHostPlatform } from './ssh-remote-platform' +import { powerShellLiteral } from './ssh-remote-powershell' +import { + ORCAD_WINDOWS_HOST_SCRIPT, + ORCAD_WINDOWS_HOST_SCRIPT_FILENAME, + ORCAD_WINDOWS_HOST_SCRIPT_PRESENT, + type OrcadWindowsHostOp +} from './orcad-windows-host-script' + +const BARE_EXECUTABLE = /^[A-Za-z]:\\[A-Za-z0-9._+~\\-]*$/u +// Leading digits are quoted: PowerShell would read `1e5` as a number and pass `100000`. +const BARE_ARGUMENT = /^[A-Za-z-][A-Za-z0-9._:+=/\\-]*$/u +// Expanded by cmd.exe (`%`) or PowerShell (`$`, backtick, typographic quotes) inside "..." +const UNQUOTABLE = /["%$`\r\n“”„‘’‚‛]/u + +export class OrcadWindowsCommandLineError extends Error { + readonly code = 'orcad_windows_command_line_unsafe' + constructor(value: string) { + super( + `Orca cannot pass ${JSON.stringify(value)} to node.exe on this Windows host: it contains ` + + 'a character cmd.exe or PowerShell would expand.' + ) + this.name = 'OrcadWindowsCommandLineError' + } +} + +function windowsPath(value: string): string { + return value.replace(/\//gu, '\\') +} + +function quotedArgument(value: string): string { + if (BARE_ARGUMENT.test(value)) { + return value + } + // A trailing backslash would escape the closing quote for CommandLineToArgvW. + if (value === '' || UNQUOTABLE.test(value) || value.endsWith('\\')) { + throw new OrcadWindowsCommandLineError(value) + } + return `"${value}"` +} + +/** One node.exe invocation that parses identically under cmd.exe and PowerShell. */ +export function orcadWindowsNodeCommandLine(executable: string, args: readonly string[]): string { + const program = windowsPath(executable) + const argv = args.map(quotedArgument) + if (BARE_EXECUTABLE.test(program)) { + return [program, ...argv].join(' ') + } + // The inner line is single-quoted PowerShell inside one double-quoted argument both shells keep. + for (const value of [program, ...args]) { + if (UNQUOTABLE.test(value)) { + throw new OrcadWindowsCommandLineError(value) + } + } + const inner = ['&', ...[program, ...args].map(powerShellLiteral)].join(' ') + return `powershell.exe -NoProfile -NonInteractive -Command "${inner}"` +} + +/** `~/.orca-remote` from the remote home. */ +export function orcadRemoteBaseDir(host: RemoteHostPlatform, remoteHome: string): string { + return joinRemotePath(host, remoteHome, RELAY_REMOTE_DIR) +} + +/** `~/.orca-remote`, the parent of every slot and of the runtime store. */ +export function orcadWindowsBaseDir(host: RemoteHostPlatform, slotDir: string): string { + return remoteDirname(slotDir.replace(/\/+$/u, ''), host) +} + +/** This client's pinned node.exe in the runtime store; deterministic from the host's arch. */ +export function orcadWindowsPinnedNodePath(host: RemoteHostPlatform, baseDir: string): string { + const target: NodeRuntimeTarget = host.arch === 'arm64' ? 'win32-arm64' : 'win32-x64' + return joinRemotePath( + host, + baseDir, + ORCAD_RUNTIMES_DIRNAME, + `${ORCAD_NODE_RUNTIME_DIR_PREFIX}${pinnedNodeRuntimeAsset(target).executableSha256}`, + ORCAD_NODE_RUNTIME_WINDOWS_EXECUTABLE + ) +} + +export function orcadWindowsHostScriptPath(host: RemoteHostPlatform, baseDir: string): string { + return joinRemotePath(host, baseDir, ORCAD_WINDOWS_HOST_SCRIPT_FILENAME) +} + +/** `node.exe `, run by this client's pinned runtime. */ +export function orcadWindowsHostOpCommand( + host: RemoteHostPlatform, + baseDir: string, + op: OrcadWindowsHostOp, + args: readonly string[] +): string { + // Under a held fence, the host script checks it still owns it before running the op. + const fence = currentOrcadFence() + return orcadWindowsNodeCommandLine(orcadWindowsPinnedNodePath(host, baseDir), [ + orcadWindowsHostScriptPath(host, baseDir), + ...(fence ? [ORCAD_WINDOWS_FENCE_ARG, fence.lockDir, fence.token] : []), + op, + ...args + ]) +} + +// Content-addressed paths this connection already saw on the host. +const stagedHostScripts = new WeakMap>() + +/** + * Stages the host script before any host op. The name is content-addressed, so it is written + * only when missing, and through a partial file and a rename: a concurrent node.exe never reads + * a truncated script. + */ +export async function installOrcadWindowsHostScript( + target: { conn: SshConnection; host: RemoteHostPlatform; signal?: AbortSignal }, + baseDir: string +): Promise { + const scriptPath = orcadWindowsHostScriptPath(target.host, baseDir) + const staged = stagedHostScripts.get(target.conn) ?? new Set() + stagedHostScripts.set(target.conn, staged) + if (staged.has(scriptPath)) { + return + } + // Both checks run a host-script op on the pinned node.exe, staged first: no inline code. + const runOp = async ( + script: string, + op: OrcadWindowsHostOp, + args: string[] + ): Promise => { + const command = orcadWindowsNodeCommandLine(orcadWindowsPinnedNodePath(target.host, baseDir), [ + script, + op, + ...args + ]) + const answer = await execCommand(target.conn, command, { + wrapCommand: false, + signal: target.signal + }).catch((error: unknown) => { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + // A missing script makes node.exe exit nonzero, which reads as "not present". + return '' + }) + return answer.trim().split(/\r?\n/u).at(-1) === ORCAD_WINDOWS_HOST_SCRIPT_PRESENT + } + if (!(await runOp(scriptPath, 'script-present', []))) { + const partial = `${scriptPath}.${randomUUID()}.partial` + await target.conn.writeFile(partial, ORCAD_WINDOWS_HOST_SCRIPT, { + hostPlatform: target.host, + signal: target.signal + }) + if (!(await runOp(partial, 'script-install', [scriptPath]))) { + throw new Error(`Could not stage the orcad host script at ${scriptPath}.`) + } + } + staged.add(scriptPath) +} + +/** The decoded payload after `marker`, or null when the host printed no such line. */ +export function readOrcadWindowsEncodedAnswer(output: string, marker: string): string | null { + const line = output + .split(/\r?\n/u) + .map((candidate) => candidate.trim()) + .findLast((candidate) => candidate === marker || candidate.startsWith(`${marker} `)) + if (line === undefined) { + return null + } + const encoded = line.slice(marker.length).trim() + if (!/^[A-Za-z0-9+/]*={0,2}$/u.test(encoded)) { + return null + } + return Buffer.from(encoded, 'base64').toString('utf8') +} diff --git a/src/main/ssh/orcad-remote-windows-scripts.test.ts b/src/main/ssh/orcad-remote-windows-scripts.test.ts new file mode 100644 index 00000000000..69d3dfc77b0 --- /dev/null +++ b/src/main/ssh/orcad-remote-windows-scripts.test.ts @@ -0,0 +1,347 @@ +/** + * Runs the host script Windows orcad hosts execute, under this machine's node. + * + * They use only `fs`, `path` and `process.kill(pid, 0)`, whose ESRCH/EPERM split libuv gives on + * every platform. The process-tree addon is faked by a preload that loads `*.node` as a table of + * creation times, so the identity rules run for real without Windows. + */ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { ORCAD_WINDOWS_PROCESS_TREE_FILENAME } from '../../shared/orcad-artifacts' +import { ORCAD_STOP_REQUEST_FILENAME } from '../../shared/orcad-stop-request' +import { ORCAD_WINDOWS_PROCESS_FILENAME } from './orcad-remote-host-support' +import { ORCAD_READINESS_FILENAME } from './orcad-remote-launch' +import { parseOrcadReadinessWaitOutput } from './orcad-remote-readiness-wait' +import { readOrcadWindowsEncodedAnswer } from './orcad-remote-windows-node' +import { + ORCAD_RECORD_ABSENT_MARKER, + ORCAD_RECORD_PRESENT_MARKER, + ORCAD_WINDOWS_HOST_SCRIPT, + ORCAD_WINDOWS_READINESS_MARKER, + ORCAD_WINDOWS_RUNTIME_MARKER, + type OrcadWindowsHostOp +} from './orcad-windows-host-script' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const FAKE_ADDON_PRELOAD = [ + 'const Module=require("module"),fs=require("fs");', + 'Module._extensions[".node"]=(m,f)=>{m.exports={getProcessCreationTime:(pid)=>{', + 'try{return JSON.parse(fs.readFileSync(f+".json","utf8"))[pid]}catch{return undefined}}}};' +].join('') +// Above any real PID on macOS and Linux defaults, so kill(pid, 0) is ESRCH. +const DEAD_PID = 4_194_303 + +let dir = '' +let preload = '' +let script = '' +const children: { kill: () => boolean }[] = [] + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'orcad-win-scripts-')) + preload = join(dir, 'fake-addon-preload.js') + writeFileSync(preload, FAKE_ADDON_PRELOAD) + script = join(dir, 'orcad-host-script.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) +}) + +afterEach(() => { + for (const child of children.splice(0)) { + child.kill() + } + rmSync(dir, { recursive: true, force: true }) +}) + +async function runOp(op: OrcadWindowsHostOp, args: string[], withAddon = true) { + return runProcess({ + program: process.execPath, + args: [...(withAddon ? ['--require', preload] : []), script, op, ...args], + timeoutMs: 15_000 + }) +} + +function stageAddon(creationTimes: Record): void { + writeFileSync(join(dir, ORCAD_WINDOWS_PROCESS_TREE_FILENAME), '') + writeFileSync( + join(dir, `${ORCAD_WINDOWS_PROCESS_TREE_FILENAME}.json`), + JSON.stringify(creationTimes) + ) +} + +function recordProcess(pid: number, creationTimeMs: number | null): void { + writeFileSync(join(dir, ORCAD_WINDOWS_PROCESS_FILENAME), JSON.stringify({ pid, creationTimeMs })) +} + +function readyLine(health: Record): string { + return `${JSON.stringify({ type: 'orca_server_ready', runtimeId: 'r1', health })}\n` +} + +describe('Windows liveness script', () => { + const liveness = async (withAddon = true) => (await runOp('liveness', [dir], withAddon)).stdout + + it('is LIVE only when the PID runs and its creation time matches the record', async () => { + stageAddon({ [process.pid]: 1000 }) + recordProcess(process.pid, 1000) + expect(await liveness()).toBe('LIVE') + }) + + it('is DEAD for an exited PID and for a running PID with another creation time', async () => { + stageAddon({ [process.pid]: 2000 }) + recordProcess(DEAD_PID, 1000) + expect(await liveness()).toBe('DEAD') + recordProcess(process.pid, 1000) + expect(await liveness()).toBe('DEAD') + }) + + it('answers a whole GC pass of dirs from one process, in order', async () => { + const live = join(dir, 'live') + const dead = join(dir, 'dead') + const unknown = join(dir, 'unknown') + const neverLaunched = join(dir, 'never-launched') + mkdirSync(unknown) + writeFileSync(join(unknown, ORCAD_READINESS_FILENAME), '') + mkdirSync(neverLaunched) + for (const [slot, pid] of [ + [live, process.pid], + [dead, DEAD_PID] + ] as const) { + mkdirSync(slot) + writeFileSync(join(slot, ORCAD_WINDOWS_PROCESS_TREE_FILENAME), '') + writeFileSync( + join(slot, `${ORCAD_WINDOWS_PROCESS_TREE_FILENAME}.json`), + JSON.stringify({ [process.pid]: 1000 }) + ) + writeFileSync( + join(slot, ORCAD_WINDOWS_PROCESS_FILENAME), + JSON.stringify({ pid, creationTimeMs: 1000 }) + ) + } + const { stdout } = await runOp('liveness-many', [live, dead, unknown, neverLaunched]) + expect(stdout.trim()).toBe('__ORCAD_LIVENESS__ LIVE,DEAD,UNKNOWN,NEVER_LAUNCHED') + }) + + it('is UNKNOWN when nothing proves identity: no record, no time, no addon', async () => { + // A slot that never launched has neither a record nor a readiness file. + expect(await liveness()).toBe('NEVER_LAUNCHED') + writeFileSync(join(dir, ORCAD_READINESS_FILENAME), '') + expect(await liveness()).toBe('UNKNOWN') + recordProcess(process.pid, null) + stageAddon({ [process.pid]: 1000 }) + expect(await liveness()).toBe('UNKNOWN') + recordProcess(process.pid, 1000) + expect(await liveness(false)).toBe('UNKNOWN') + stageAddon({}) + expect(await liveness()).toBe('UNKNOWN') + }) +}) + +describe('Windows stop script', () => { + const requestFile = () => join(dir, ORCAD_STOP_REQUEST_FILENAME) + const stop = async (justLaunched: boolean, waitSeconds = 5) => + (await runOp('stop', [dir, String(waitSeconds), justLaunched ? '1' : '0'])).stdout + .trim() + .split(/\r?\n/u) + .at(-1) + + /** Stands in for orcad's stop-request listener: exits once the request file appears. */ + function fakeOrcad(): number { + const child = spawnProcess({ + program: process.execPath, + args: [ + '-e', + 'const fs=require("fs");setInterval(()=>{if(fs.existsSync(process.argv[1]))process.exit(0)},50)', + requestFile() + ] + }) + children.push(child) + if (!child.pid) { + throw new Error('fake orcad did not start') + } + return child.pid + } + + it('asks a capable build through the request file and waits for it to exit', async () => { + const pid = fakeOrcad() + stageAddon({ [pid]: 1000 }) + recordProcess(pid, 1000) + writeFileSync(join(dir, ORCAD_READINESS_FILENAME), readyLine({ pid, stopRequests: 1 })) + expect(await stop(false)).toBe('STOPPED') + }) + + it('writes the request for a just-launched candidate that has not published readiness', async () => { + const pid = fakeOrcad() + stageAddon({ [pid]: 1000 }) + recordProcess(pid, 1000) + expect(await stop(true)).toBe('STOPPED') + }) + + it('refuses a build that cannot be asked, without writing anything', async () => { + stageAddon({ [process.pid]: 1000 }) + recordProcess(process.pid, 1000) + writeFileSync(join(dir, ORCAD_READINESS_FILENAME), readyLine({ pid: process.pid })) + expect(await stop(false)).toBe('UNSUPPORTED') + expect(existsSync(requestFile())).toBe(false) + }) + + it('never addresses a PID the readiness record does not corroborate', async () => { + stageAddon({ [process.pid]: 1000 }) + recordProcess(process.pid, 1000) + writeFileSync(join(dir, ORCAD_READINESS_FILENAME), readyLine({ pid: 1, stopRequests: 1 })) + expect(await stop(false)).toBe('UNKNOWN') + expect(existsSync(requestFile())).toBe(false) + // A settled stop needs readiness unless this client just launched the slot. + writeFileSync(join(dir, ORCAD_READINESS_FILENAME), '') + expect(await stop(false)).toBe('UNKNOWN') + }) + + it('reports no record, an exited process, and one that outlives the wait', async () => { + expect(await stop(true)).toBe('NO_PID') + stageAddon({ [process.pid]: 1000 }) + recordProcess(DEAD_PID, 1000) + expect(await stop(true)).toBe('ALREADY_EXITED') + recordProcess(process.pid, 1000) + expect(await stop(true, 0)).toBe('STILL_RUNNING') + expect(existsSync(requestFile())).toBe(true) + }) +}) + +describe('Windows readiness wait script', () => { + const host = getRemoteHostPlatform('win32-x64') + const file = () => join(dir, ORCAD_READINESS_FILENAME) + const wait = async (seconds: number) => { + const { stdout } = await runOp( + 'readiness-wait', + [file(), String(256 * 1024), String(seconds)], + false + ) + return parseOrcadReadinessWaitOutput(host, stdout) + } + + it('answers as soon as a line is complete, byte-exact through base64', async () => { + const line = readyLine({ pid: 7, stopRequests: 1, dataDir: 'C:/Users/Zoë/.orca' }) + setTimeout(() => writeFileSync(file(), line), 300) + const started = Date.now() + const result = await wait(10) + expect(Date.now() - started).toBeLessThan(8_000) + expect(result).toMatchObject({ state: 'ready', readiness: { runtimeId: 'r1' } }) + expect(result.state === 'ready' && result.readiness.health).toMatchObject({ + dataDir: 'C:/Users/Zoë/.orca' + }) + }) + + it('is still pending when its bounded wait ends on a partial line', async () => { + writeFileSync(file(), '{"type":"orca_ser') + expect(await wait(1)).toEqual({ state: 'pending' }) + }) + + it('reads a missing file as nothing yet', async () => { + expect(await wait(0)).toEqual({ state: 'pending' }) + }) +}) + +describe('Windows record scripts', () => { + const read = (path: string, max = 1024) => runOp('record-read', [path, String(max)], false) + + it('tells absent from present, and returns the bytes exactly', async () => { + const path = join(dir, 'orcad-active.json') + expect((await read(path)).stdout.trim()).toBe(ORCAD_RECORD_ABSENT_MARKER) + writeFileSync(path, '{"active":"0.2.0+bb01","owner":"Zoë"}') + const present = await read(path) + expect(readOrcadWindowsEncodedAnswer(present.stdout, ORCAD_RECORD_PRESENT_MARKER)).toBe( + '{"active":"0.2.0+bb01","owner":"Zoë"}' + ) + }) + + it('refuses an oversized record or a directory rather than reading it as absent', async () => { + const path = join(dir, 'big.json') + writeFileSync(path, 'x'.repeat(2048)) + expect((await read(path)).code).toBe(65) + mkdirSync(join(dir, 'a-dir')) + expect((await read(join(dir, 'a-dir'))).code).toBe(65) + }) + + it('publishes a staged record over the existing one', async () => { + const path = join(dir, 'transaction.json') + const staged = `${path}.partial.1.abc` + writeFileSync(path, 'old') + writeFileSync(staged, 'new') + const result = await runOp('record-publish', [staged, path], false) + expect(result.code).toBe(0) + expect(readFileSync(path, 'utf8')).toBe('new') + expect(existsSync(staged)).toBe(false) + }) + + it('fails without a stage rather than publishing nothing', async () => { + const result = await runOp( + 'record-publish', + [join(dir, 'missing'), join(dir, 'transaction.json')], + false + ) + expect(result.code).not.toBe(0) + }) +}) + +describe('Windows host script staging', () => { + it('answers present from a whole script, and installs itself from its partial upload', async () => { + expect((await runOp('script-present', [], false)).stdout.trim()).toBe( + 'ORCAD_HOST_SCRIPT_PRESENT' + ) + const partial = join(dir, 'staged.partial') + const target = join(dir, 'installed.js') + writeFileSync(partial, ORCAD_WINDOWS_HOST_SCRIPT) + const installed = await runProcess({ + program: process.execPath, + args: [partial, 'script-install', target], + timeoutMs: 15_000 + }) + expect(installed.stdout.trim()).toBe('ORCAD_HOST_SCRIPT_PRESENT') + expect(existsSync(partial)).toBe(false) + expect(readFileSync(target, 'utf8')).toBe(ORCAD_WINDOWS_HOST_SCRIPT) + }) +}) + +describe('Windows slot runtime and file removal', () => { + const slot = () => join(dir, 'orcad-0.2.0+bb01') + const sha = 'a'.repeat(64) + + it('names the runtime the slot marker points at, and clears a stale stop request on launch', async () => { + mkdirSync(slot()) + writeFileSync(join(slot(), '.runtime-node'), `${sha}\n`) + writeFileSync(join(slot(), ORCAD_STOP_REQUEST_FILENAME), '') + expect((await runOp('slot-runtime', [slot()], false)).code).toBe(78) + mkdirSync(join(dir, 'runtimes', `node-${sha}`), { recursive: true }) + writeFileSync(join(dir, 'runtimes', `node-${sha}`, 'node.exe'), '') + const plain = await runOp('slot-runtime', [slot()], false) + expect(readOrcadWindowsEncodedAnswer(plain.stdout, ORCAD_WINDOWS_RUNTIME_MARKER)).toBe( + join(dir, 'runtimes', `node-${sha}`, 'node.exe') + ) + expect(existsSync(join(slot(), ORCAD_STOP_REQUEST_FILENAME))).toBe(true) + await runOp('slot-runtime', [slot(), 'clear-stop-request'], false) + expect(existsSync(join(slot(), ORCAD_STOP_REQUEST_FILENAME))).toBe(false) + }) + + it('refuses a marker that is not a bare sha256', async () => { + mkdirSync(slot()) + writeFileSync(join(slot(), '.runtime-node'), '../../escape') + expect((await runOp('slot-runtime', [slot()], false)).code).toBe(78) + }) + + it('removes a file and treats an absent one as removed', async () => { + const file = join(dir, 'staged.json') + writeFileSync(file, '{}') + expect((await runOp('remove-file', [file], false)).code).toBe(0) + expect(existsSync(file)).toBe(false) + expect((await runOp('remove-file', [file], false)).code).toBe(0) + }) + + it('rejects an unknown op', async () => { + const result = await runProcess({ program: process.execPath, args: [script, 'nope'] }) + expect(result.code).toBe(64) + }) +}) + +it('decodes nothing from output that lacks the marker', () => { + expect(readOrcadWindowsEncodedAnswer('noise\r\n', ORCAD_WINDOWS_READINESS_MARKER)).toBeNull() +}) diff --git a/src/main/ssh/orcad-retained-fence-takeover.test.ts b/src/main/ssh/orcad-retained-fence-takeover.test.ts new file mode 100644 index 00000000000..b70b3e4f3c6 --- /dev/null +++ b/src/main/ssh/orcad-retained-fence-takeover.test.ts @@ -0,0 +1,41 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { runProcess } from '../../shared/child-process/run-process' +import { + orphanInstallLockCommand, + tryCreateInstallLockCommand, + tryStealInstallLockCommand +} from './ssh-relay-install-lock-commands' +import { INSTALL_LOCK_STALE_SECONDS } from './ssh-relay-install-lock' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const host = getRemoteHostPlatform(process.platform === 'darwin' ? 'darwin-arm64' : 'linux-x64') +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +async function sh(command: string): Promise { + const result = await runProcess({ program: '/bin/sh', args: ['-c', command], timeoutMs: 10_000 }) + return result.stdout.trim() +} + +// BUG-17: a recovery that took a fence over, failed and retained it left a fresh lock behind, so +// every later recovery read "still fresh" and the host could never be recovered. +describe.skipIf(process.platform === 'win32')('a retained activation fence', () => { + it('stays fresh to stale takeover until its holder marks it ownerless', async () => { + const root = mkdtempSync(join(tmpdir(), 'orcad-fence-')) + roots.push(root) + const lockDir = join(root, '.install-lock') + expect(await sh(tryCreateInstallLockCommand(host, lockDir))).toBe('OK') + const steal = tryStealInstallLockCommand(host, lockDir, INSTALL_LOCK_STALE_SECONDS) + + expect(await sh(steal)).toBe('BUSY') + await sh(orphanInstallLockCommand(host, lockDir)) + expect(await sh(steal)).toMatch(/OK$/u) + }) +}) diff --git a/src/main/ssh/orcad-retained-source-lists.test.ts b/src/main/ssh/orcad-retained-source-lists.test.ts new file mode 100644 index 00000000000..3c9f2903e99 --- /dev/null +++ b/src/main/ssh/orcad-retained-source-lists.test.ts @@ -0,0 +1,140 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { FolderWorkspace } from '../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../shared/project-group-types' +import type { Repo } from '../../shared/repo-types' +import type { SshTarget } from '../../shared/ssh-types' +import { isAdmissibleDirectSshAuthority } from '../../shared/ssh-retained-payload-admission' +import type { Store } from '../persistence' +import { orcadMigrationCutoverFixture } from './orcad-migration-cutover-fixture' +import { writeOrcadMigrationSourceCutover } from './orcad-migration-cutover-journal' + +const userData = vi.hoisted(() => ({ dir: '' })) +vi.mock('../../shared/app-environment', () => ({ + getAppEnvironment: () => ({ getPath: () => userData.dir }) +})) +vi.mock('./ssh-provider-authority', () => ({ isCurrentSshProviderAuthority: () => true })) +const provider = {} +vi.mock('../providers/ssh-git-dispatch', () => ({ getSshGitProvider: () => provider })) + +const { isFrozenOrcadSourceSessionPartition, visibleProjectGroups } = + await import('./orcad-retained-source') +const { listReposForExecutionHost } = await import('../ipc/repos/host-repo-catalog-snapshot') + +const FENCED: SshTarget = { + id: 'ssh-box', + label: 'Box', + host: 'box.example.com', + port: 22, + username: 'me', + generation: 2, + orcadFence: { environmentId: 'env-1' } +} + +function group(id: string, connectionId: string | null): ProjectGroup { + return { + id, + name: id, + parentPath: '/srv', + parentGroupId: null, + connectionId, + createdFrom: 'manual', + tabOrder: 0, + isCollapsed: false, + color: null, + createdAt: 1, + updatedAt: 1 + } +} + +const sourceRepo: Repo = { + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: FENCED.id +} + +function catalog(targets: SshTarget[]): Store { + const groups = [group('source-group', FENCED.id), group('local-group', null)] + const folders: FolderWorkspace[] = [] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the lists read only these four getters. + return { + getSshTargets: () => targets, + getRepos: () => [sourceRepo], + getProjectGroups: () => groups, + getFolderWorkspaces: () => folders + } as unknown as Store +} + +describe('lists while a converted host keeps its source rows', () => { + beforeEach(() => { + userData.dir = mkdtempSync(join(tmpdir(), 'orcad-retained-lists-')) + writeOrcadMigrationSourceCutover(userData.dir, { + ...orcadMigrationCutoverFixture('m-0', FENCED.id), + phase: 'destination-committed' + }) + }) + afterEach(() => rmSync(userData.dir, { recursive: true, force: true })) + + it('shows every row of a fenced host no journal explains, as after an empty-host deploy', () => { + const emptyDeploy = mkdtempSync(join(tmpdir(), 'orcad-retained-lists-')) + try { + // An older build added these after the deploy; no move owns them, so they stay visible. + expect(visibleProjectGroups(catalog([FENCED]), () => emptyDeploy)).toHaveLength(2) + } finally { + rmSync(emptyDeploy, { recursive: true, force: true }) + } + }) + + it('hides the host own project groups, but not a local group', () => { + expect(visibleProjectGroups(catalog([FENCED])).map((entry) => entry.id)).toEqual([ + 'local-group' + ]) + }) + + it('shows them again once an older build changed the host', () => { + const changed = { ...FENCED, orcadFence: { environmentId: 'env-1', sourceChangedAt: 'x' } } + expect(visibleProjectGroups(catalog([changed])).map((entry) => entry.id)).toEqual([ + 'source-group', + 'local-group' + ]) + }) + + it('shows them until the migration commits, while freezing the session from the fence on', () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-retained-lists-')) + try { + const cutover = orcadMigrationCutoverFixture('m-1', FENCED.id) + writeOrcadMigrationSourceCutover(userDataPath, cutover) + const store = catalog([FENCED]) + const lookup = { getSshTarget: () => FENCED } + expect(visibleProjectGroups(store, () => userDataPath)).toHaveLength(2) + expect(isFrozenOrcadSourceSessionPartition(lookup, 'ssh:ssh-box')).toBe(true) + writeOrcadMigrationSourceCutover(userDataPath, { ...cutover, phase: 'destination-committed' }) + expect(visibleProjectGroups(store, () => userDataPath)).toHaveLength(1) + expect(isFrozenOrcadSourceSessionPartition(lookup, 'local')).toBe(false) + const changed = { ...FENCED, orcadFence: { environmentId: 'env-1', sourceChangedAt: 'x' } } + expect( + isFrozenOrcadSourceSessionPartition({ getSshTarget: () => changed }, 'ssh:ssh-box') + ).toBe(false) + } finally { + rmSync(userDataPath, { recursive: true, force: true }) + } + }) + + it('leaves the source repos out of the host catalog the SSH bridge hydrates from', async () => { + const authority: unknown = { targetId: FENCED.id, providerEpoch: 'e1', connectionGeneration: 1 } + if (!isAdmissibleDirectSshAuthority(authority)) { + throw new Error('fixture authority rejected') + } + const snapshot = await listReposForExecutionHost(catalog([FENCED]), { + executionHostId: `ssh:${FENCED.id}`, + expectedAuthority: authority + }) + expect(snapshot).toMatchObject({ authoritative: true, repos: [] }) + }) +}) diff --git a/src/main/ssh/orcad-retained-source.ts b/src/main/ssh/orcad-retained-source.ts new file mode 100644 index 00000000000..1db1db3ea08 --- /dev/null +++ b/src/main/ssh/orcad-retained-source.ts @@ -0,0 +1,241 @@ +/** + * A converted host's source rows: hidden from this build's lists, which show the managed server + * instead, and kept for a downgraded build that still reads them. Nothing here ever deletes them; + * only stopping the server, removing the host or uninstalling does. + * + * On every start the rows are compared with what the migration committed. If an older build + * changed them, the host is marked `sourceChangedAt`: its rows show again, it stays on the relay, + * and it needs a new move. A second manifest is never merged into the server automatically. + */ +import { createHash } from 'node:crypto' +import { getAppEnvironment } from '../../shared/app-environment' +import { parseExecutionHostId } from '../../shared/execution-host' +import type { FolderWorkspace } from '../../shared/folder-workspace-types' +import type { ProjectGroup } from '../../shared/project-group-types' +import type { OrcadMigrationCatalogPayload } from '../../shared/orcad-migration-manifest' +import { + isRetainedOrcadMigrationSourceCutover, + type OrcadMigrationSourceCutover +} from '../../shared/orcad-migration-source-cutover' +import type { Repo } from '../../shared/repo-types' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { SshTarget } from '../../shared/ssh-types' +import type { Store } from '../persistence' +import { collectOrcadMigrationSourceCatalog } from '../persistence/migrating-orcad-catalog/orcad-source-catalog' +import { + orcadSourceFolderWorkspaceIds, + projectGroupBelongsToOrcadSource, + repoBelongsToOrcadSource +} from '../persistence/migrating-orcad-catalog/orcad-source-ownership' +import { findOrcadMigrationSourceCutoverForTarget } from './orcad-migration-cutover-journal' + +const appUserDataPath = (): string => getAppEnvironment().getPath('userData') + +type CatalogStore = Pick +type TargetStore = Pick + +/** + * Whether this build hides the host's source rows: committed to its server and not changed by an + * older build since. Mid-migration the rows stay shown, since the source is still authoritative, + * and so does a fence no journal explains (an empty host's deploy): no move owns those rows. + */ +export function isHiddenRetainedSourceTarget( + userDataPath: string, + target: Pick +): boolean { + if (!target.orcadFence || target.orcadFence.sourceChangedAt) { + return false + } + let head: OrcadMigrationSourceCutover | null + try { + head = findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) + } catch { + return true // An unreadable journal keeps the host fenced, and so hidden. + } + return ( + head !== null && + (head.phase === 'destination-committed' || + head.phase === 'source-retired' || + head.destinationEnvironmentId !== target.orcadFence.environmentId) + ) +} + +/** `getUserDataPath` is read only once a host is fenced, so unfenced lists never touch the journal. */ +export function hiddenRetainedSourceTargetIds( + getUserDataPath: () => string, + targets: readonly SshTarget[] +): string[] { + return targets + .filter( + (target) => target.orcadFence && isHiddenRetainedSourceTarget(getUserDataPath(), target) + ) + .map((target) => target.id) +} + +/** + * A fenced host's `ssh:` session partition is migration source from the fence on: a renderer save + * would change it mid-conversion, and strip the rows this build hides once it commits. + */ +export function isFrozenOrcadSourceSessionPartition( + store: Pick, + hostId: string | null | undefined +): boolean { + const parsed = parseExecutionHostId(hostId) + const fence = parsed?.kind === 'ssh' ? store.getSshTarget(parsed.targetId)?.orcadFence : undefined + return fence !== undefined && !fence.sourceChangedAt +} + +export function visibleRepos( + store: CatalogStore & Pick, + getUserDataPath = appUserDataPath +): Repo[] { + const hidden = hiddenRetainedSourceTargetIds(getUserDataPath, store.getSshTargets()) + const repos = store.getRepos() + if (hidden.length === 0) { + return repos + } + return repos.filter( + (repo) => !hidden.some((targetId) => repoBelongsToOrcadSource(repo, targetId)) + ) +} + +/** Hides only groups the host owns; a local group holding one of its projects stays. */ +export function visibleProjectGroups( + store: Pick, + getUserDataPath = appUserDataPath +): ProjectGroup[] { + const hidden = hiddenRetainedSourceTargetIds(getUserDataPath, store.getSshTargets()) + const groups = store.getProjectGroups() + if (hidden.length === 0) { + return groups + } + return groups.filter( + (group) => !hidden.some((targetId) => projectGroupBelongsToOrcadSource(group, targetId)) + ) +} + +export function visibleFolderWorkspaces( + store: CatalogStore & Pick, + getUserDataPath = appUserDataPath +): FolderWorkspace[] { + const hidden = hiddenRetainedSourceTargetIds(getUserDataPath, store.getSshTargets()) + const folderWorkspaces = store.getFolderWorkspaces() + if (hidden.length === 0) { + return folderWorkspaces + } + const state = { + repos: store.getRepos(), + projectGroups: store.getProjectGroups(), + folderWorkspaces + } + const hiddenIds = new Set( + hidden.flatMap((targetId) => [...orcadSourceFolderWorkspaceIds(state, targetId)]) + ) + return folderWorkspaces.filter((workspace) => !hiddenIds.has(workspace.id)) +} + +/** + * Identity only, hashed to fit the journal: an older build adding, removing or moving a project is + * a change, a touched timestamp is not. + */ +export function orcadCatalogFingerprint(catalog: OrcadMigrationCatalogPayload): string { + return createHash('sha256') + .update( + JSON.stringify([ + catalog.repositories.map((repo) => `${repo.id}\0${repo.path}`).sort(), + catalog.folderWorkspaces.map((folder) => `${folder.id}\0${folder.folderPath}`).sort(), + catalog.projectGroups.map((group) => group.id).sort() + ]) + ) + .digest('hex') +} + +export function currentOrcadSourceFingerprint( + store: CatalogStore, + target: Pick +): string { + return orcadCatalogFingerprint(collectOrcadMigrationSourceCatalog(store, target)) +} + +/** What the retained source must still look like for this build to keep serving it from orcad. */ +export function retainedOrcadSourceBaseline(head: OrcadMigrationSourceCutover): string { + return head.sourceBaselineFingerprint ?? orcadCatalogFingerprint(head.manifest.payload) +} + +type RetainedSourceVerdict = 'unchanged' | 'changed' + +/** Identity only: an older build's edits inside moved projects stay in the retained rows. */ +function compareRetainedOrcadSource( + store: CatalogStore, + target: Pick, + head: OrcadMigrationSourceCutover +): RetainedSourceVerdict { + return currentOrcadSourceFingerprint(store, target) === retainedOrcadSourceBaseline(head) + ? 'unchanged' + : 'changed' +} + +/** + * Startup pass: restore a fence the profile lost from the registered managed server, and mark a + * retained host whose rows an older build changed. Never throws; a failed pass changes nothing. + */ +export function reconcileManagedOrcadSshTargets( + userDataPath: string, + store: CatalogStore & TargetStore, + now: () => Date = () => new Date() +): void { + try { + restoreFencesFromManagedServers(userDataPath, store) + markChangedRetainedSources(userDataPath, store, now) + } catch (error) { + console.warn('[ssh] Could not reconcile managed Orca server hosts:', error) + } +} + +function restoreFencesFromManagedServers(userDataPath: string, store: TargetStore): void { + const targets = new Map(store.getSshTargets().map((target) => [target.id, target])) + for (const environment of listEnvironments(userDataPath)) { + const targetId = environment.orcadDeployment?.sshTargetId + const target = targetId ? targets.get(targetId) : undefined + // Why generation-bound: a host re-created under the same id is a different registration. + if ( + target && + !target.orcadFence && + target.generation === environment.orcadDeployment?.sshTargetGeneration + ) { + store.updateSshTarget(target.id, { orcadFence: { environmentId: environment.id } }) + } + } +} + +function markChangedRetainedSources( + userDataPath: string, + store: CatalogStore & TargetStore, + now: () => Date +): void { + for (const target of store.getSshTargets()) { + const fence = target.orcadFence + const head = fence ? findOrcadMigrationSourceCutoverForTarget(userDataPath, target.id) : null + if ( + !fence || + !head || + fence.environmentId !== head.destinationEnvironmentId || + fence.sourceChangedAt + ) { + continue + } + // A delta move a crash interrupted lost its mark with it; the mark leads back to resuming it. + const interruptedDelta = + head.supersedesMigrationId !== undefined && + (head.phase === 'source-fenced' || head.phase === 'destination-staged') + if ( + interruptedDelta || + (isRetainedOrcadMigrationSourceCutover(head) && + compareRetainedOrcadSource(store, target, head) === 'changed') + ) { + store.updateSshTarget(target.id, { + orcadFence: { ...fence, sourceChangedAt: now().toISOString() } + }) + } + } +} diff --git a/src/main/ssh/orcad-rollback-terminal-barrier.ts b/src/main/ssh/orcad-rollback-terminal-barrier.ts new file mode 100644 index 00000000000..d719910f065 --- /dev/null +++ b/src/main/ssh/orcad-rollback-terminal-barrier.ts @@ -0,0 +1,110 @@ +/** + * The rollback's terminal barrier. The census a rollback plans on is taken while orcad still + * admits work, so it cannot vouch for the moment the older snapshot replaces the state. The + * incumbent is stopped through its managed stop with idle-daemon retirement instead: orcad closes + * terminal admission on every daemon generation, counts live sessions under that fence, and + * retires the daemon only when none exist. Only `retired` proves no terminal ran then or could + * start after; anything else keeps the incumbent's state and puts it back. + */ +import { + readRemoteOrcadManagedStopTarget, + type OrcadManagedStopTarget +} from './orcad-managed-remote-stop' +import { settleOrcadDecommissionStop } from './orcad-decommission-stop' +import type { OrcadSlotOptions } from './orcad-recovery-slot' +import type { OrcadManagedStopContext } from '../../shared/orcad-stop-request' + +export type OrcadRollbackBarrier = + | { state: 'retired' } + /** orcad withdrew the stop and keeps serving: nothing changed. */ + | { state: 'withdrawn'; reason: string } + /** orcad may still be stopping, or the host could not say: the fence must stay. */ + | { state: 'unsettled'; reason: string } + /** orcad exited, but live or unproven work remains: the incumbent must be restarted. */ + | { state: 'unproven'; reason: string } + +/** Names the incumbent instance before any mutation; a build without managed stop is refused. */ +export async function readOrcadRollbackBarrierTarget( + options: OrcadSlotOptions, + version: string +): Promise { + const target = await readRemoteOrcadManagedStopTarget(options, version) + return target.state === 'ready' + ? target + : { + ...target, + code: 'orcad_rollback_terminal_barrier_unavailable', + reason: + `${target.reason} A rollback needs orcad to prove no terminal runs at its stop; ` + + 'nothing was changed.' + } +} + +export async function stopIncumbentBehindTerminalBarrier( + options: OrcadSlotOptions, + transactionId: string, + context: OrcadManagedStopContext +): Promise { + const settlement = await settleOrcadDecommissionStop(options, { + schemaVersion: 1, + transactionId, + ...context, + retireIdleDaemon: true + }) + if (settlement.state === 'withdrawn' || settlement.state === 'unsettled') { + return { state: settlement.state, reason: settlement.reason } + } + if (settlement.retirement === 'retired') { + return { state: 'retired' } + } + return { + state: 'unproven', + reason: + `orcad ${context.version} stopped, but its terminal daemon reported ` + + `${settlement.retirement} work at the stop, so the older snapshot was not restored.` + } +} + +export type OrcadRollbackBarrierRefusal = { + outcome: 'refused' | 'failed' + code: string + reason: string + /** orcad may still be stopping, so the fence stays. */ + retainFence: boolean + /** orcad exited without proving it ran no terminal, so its own state goes back up. */ + restartIncumbent: boolean +} + +/** Null only for a retired daemon, the one answer that lets the older snapshot replace state. */ +export function rollbackBarrierRefusal( + barrier: OrcadRollbackBarrier +): OrcadRollbackBarrierRefusal | null { + switch (barrier.state) { + case 'retired': + return null + case 'withdrawn': + return { + outcome: 'failed', + code: 'orcad_rollback_stop_withdrawn', + reason: barrier.reason, + retainFence: false, + restartIncumbent: false + } + case 'unsettled': + return { + outcome: 'failed', + code: 'orcad_rollback_stop_incomplete', + reason: `${barrier.reason} Nothing was restored.`, + retainFence: true, + restartIncumbent: false + } + case 'unproven': + return { + outcome: 'refused', + code: 'orcad_rollback_terminals_at_stop', + reason: barrier.reason, + retainFence: false, + restartIncumbent: true + } + } +} diff --git a/src/main/ssh/orcad-rollback-transition.ts b/src/main/ssh/orcad-rollback-transition.ts new file mode 100644 index 00000000000..86f3c42ad39 --- /dev/null +++ b/src/main/ssh/orcad-rollback-transition.ts @@ -0,0 +1,272 @@ +/** The locked, journaled half of `rollbackOrcad`. */ +import { orcadRemoteBaseDir } from './orcad-remote-windows-node' +import { randomUUID } from 'node:crypto' +import type { OrcadRollbackOptions, OrcadRollbackResult } from './orcad-remote-rollback' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { withRolledBackVersion } from './orcad-activation-record' +import { writeOrcadActivationRecord } from './orcad-activation-record-store' +import { launchAndJudgeOrcadSlot } from './orcad-candidate-launch-verdict' +import { assessOrcadRollback } from './orcad-update-plan' +import { ORCAD_LOG_FILENAME } from './orcad-remote-launch' +import { + captureOrcadStateSnapshotCommand, + newestStateMtimeCommand, + orcadRollbackRescueDirName, + parseNewestStateMtimeSeconds, + parseOrcadSnapshotCapture, + parseOrcadSnapshotPresence, + parseOrcadSnapshotRestore, + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand +} from './orcad-state-snapshot' +import { joinRemotePath } from './ssh-remote-platform' +import type { OrcadActivationLockControl } from './orcad-activation-lock' +import { + rollbackStartingState, + type OrcadRollbackTransaction +} from './orcad-activation-transaction' +import { + createOrcadRollbackTransaction, + withOrcadRollbackPhase, + withOrcadRollbackRescue +} from './orcad-activation-transaction-transitions' +import { writeOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { execOrcadRemoteOr, withoutAbortSignal } from './orcad-remote-runtime-control' +import { execOrcadStateMutationOr } from './orcad-state-mutation-exec' +import { + orcadSlotDir, + resolveOrcadSlotIdentity, + type OrcadSlotIdentity +} from './orcad-recovery-slot' +import { orcadSnapshotPath } from './orcad-incumbent-recovery' +import { + putTransactionIncumbentBack, + restoreAfterRejectedCandidate +} from './orcad-transaction-incumbent' +import { + readOrcadRollbackBarrierTarget, + rollbackBarrierRefusal, + stopIncumbentBehindTerminalBarrier +} from './orcad-rollback-terminal-barrier' +import { withOrcadLogTail } from './orcad-remote-log-tail' +import { errorMessage } from '../../shared/error-message' + +async function stateWritesSinceActivation(options: OrcadRollbackOptions): Promise { + const activatedAtSeconds = Math.floor(Date.parse(options.record.activatedAt ?? '') / 1000) + if (!Number.isFinite(activatedAtSeconds)) { + return null + } + const newest = parseNewestStateMtimeSeconds( + await execOrcadRemoteOr( + options, + newestStateMtimeCommand( + options.host, + options.userDataDir, + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + ) + ) + return newest === null ? null : newest >= activatedAtSeconds +} + +export async function rollbackOrcadLocked( + options: OrcadRollbackOptions, + lock: OrcadActivationLockControl +): Promise { + const now = options.now ?? ((): Date => new Date()) + const snapshot = options.record.snapshot + const presence = snapshot + ? parseOrcadSnapshotPresence( + await execOrcadRemoteOr( + options, + probeOrcadStateSnapshotCommand( + options.host, + orcadSnapshotPath(options, snapshot.dirName), + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + ) + ) + : 'absent' + const safety = assessOrcadRollback({ + record: options.record, + snapshotPresent: presence === 'unverifiable' ? null : presence === 'present', + census: options.census, + targetDaemonProtocol: options.targetDaemonProtocol, + stateWritesSinceActivation: await stateWritesSinceActivation(options) + }) + if (safety.safety === 'unsafe') { + return { outcome: 'refused', code: safety.code, reason: safety.reason } + } + if (!snapshot || !options.record.active) { + return { + outcome: 'refused', + code: 'orcad_rollback_no_active', + reason: 'No orcad version is active on this host, so there is nothing to roll back from.' + } + } + let incumbent: OrcadSlotIdentity + try { + incumbent = await resolveOrcadSlotIdentity(options, options.record.active) + } catch (error) { + if (isUnconfirmedSshCommandTermination(error)) { + throw error + } + return { + outcome: 'refused', + code: 'orcad_rollback_active_identity_unverifiable', + reason: `The active orcad identity could not be verified: ${errorMessage(error)} Nothing was changed.` + } + } + + const barrierTarget = await readOrcadRollbackBarrierTarget(options, incumbent.version) + if (barrierTarget.state === 'refused') { + return { outcome: 'refused', code: barrierTarget.code, reason: barrierTarget.reason } + } + + const startedAt = now() + let transaction: OrcadRollbackTransaction = createOrcadRollbackTransaction({ + transactionId: randomUUID(), + incumbentVersion: incumbent.version, + targetVersion: safety.target, + recordBefore: options.record, + recordAfter: withRolledBackVersion(options.record, startedAt), + rescueDirName: orcadRollbackRescueDirName(incumbent.version, startedAt.getTime()), + now: startedAt + }) + await writeOrcadActivationTransaction(options, transaction) + lock.retainOnError() + // Past the first mutation a cancel would strand a stopped host, so the run finishes or rolls back. + options = withoutAbortSignal(options) + + // The census above was taken while orcad admitted work; this stop is the proof that counts. + const blocked = rollbackBarrierRefusal( + await stopIncumbentBehindTerminalBarrier( + options, + transaction.transactionId, + barrierTarget.context + ) + ) + if (blocked) { + if (blocked.retainFence) { + lock.retain() + } + const recovered = blocked.restartIncumbent + ? ` ${await putIncumbentBack(options, lock, transaction, incumbent)}` + : '' + return { outcome: blocked.outcome, code: blocked.code, reason: blocked.reason + recovered } + } + transaction = withOrcadRollbackPhase(transaction, 'incumbent-stopped', now()) + await writeOrcadActivationTransaction(options, transaction) + + const rescue = parseOrcadSnapshotCapture( + await execOrcadStateMutationOr( + options, + captureOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + orcadSnapshotPath(options, transaction.rescue.dirName), + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + ) + ) + if (rescue === 'failed') { + const recovered = await putIncumbentBack(options, lock, transaction, incumbent) + return { + outcome: 'failed', + code: 'orcad_rollback_rescue_snapshot_failed', + reason: + 'The current state could not be preserved in a rescue snapshot, so the data root was ' + + `not replaced. ${recovered}` + } + } + transaction = withOrcadRollbackRescue(transaction, rescue, now()) + await writeOrcadActivationTransaction(options, transaction) + + // Why between stop and start: the older build must never load the newer build's state. + const restored = parseOrcadSnapshotRestore( + await execOrcadStateMutationOr( + options, + restoreOrcadStateSnapshotCommand( + options.host, + options.userDataDir, + orcadSnapshotPath(options, snapshot.dirName), + orcadRemoteBaseDir(options.host, options.remoteHome) + ) + ) + ) + if (restored !== 'restored') { + const recovered = await putIncumbentBack(options, lock, transaction, incumbent) + return { + outcome: 'failed', + code: 'orcad_rollback_restore_failed', + reason: `The pre-activation snapshot could not be restored (${restored}). ${recovered}` + } + } + transaction = withOrcadRollbackPhase(transaction, 'rollback-state-restored', now()) + await writeOrcadActivationTransaction(options, transaction) + + const targetDir = orcadSlotDir(options, safety.target) + const { verdict, launchError } = await launchAndJudgeOrcadSlot(options, { + remoteInstallDir: targetDir, + fullVersion: safety.target, + buildHash: options.targetBuildHash + }) + if (verdict.decision === 'reject') { + const reason = + launchError === undefined + ? verdict.reason + : `It failed while starting: ${errorMessage(launchError)}` + const recovered = await restoreAfterRejectedCandidate(options, lock, { + launchedDir: targetDir, + launchedVersion: safety.target, + transactionStartedAt: transaction.startedAt, + incumbent, + restoreState: rescueVerdict(transaction), + launchedFromState: rollbackStartingState(transaction) + }) + return { + outcome: 'failed', + code: launchError === undefined ? verdict.code : 'orcad_rollback_target_launch_failed', + reason: await withOrcadLogTail( + options, + targetDir, + `The rollback target ${safety.target} did not come up healthy: ${reason} ${recovered} ` + + `Its stderr is at ${joinRemotePath(options.host, targetDir, ORCAD_LOG_FILENAME)}.` + ) + } + } + + // The record is written last: until the target is proven serving, `active` still names the + // version an operator would need to bring back. + transaction = withOrcadRollbackPhase(transaction, 'target-ready', now()) + await writeOrcadActivationTransaction(options, transaction) + await writeOrcadActivationRecord(options, transaction.recordAfter) + return { + outcome: 'rolled-back', + target: safety.target, + discarded: safety.safety === 'lossy' ? safety.discards : [], + verdict + } +} + +type RestoreState = Parameters[2]['restoreState'] + +function rescueVerdict(transaction: OrcadRollbackTransaction): RestoreState { + return transaction.rescue.state === 'pending' ? null : transaction.rescue +} + +/** Puts the rescued state back when it was replaced, then restarts the incumbent. */ +async function putIncumbentBack( + options: OrcadRollbackOptions, + lock: OrcadActivationLockControl, + transaction: OrcadRollbackTransaction, + incumbent: OrcadSlotIdentity +): Promise { + const failure = await putTransactionIncumbentBack(options, lock, { + transactionStartedAt: transaction.startedAt, + launchedVersion: null, + incumbent, + restoreState: rescueVerdict(transaction) + }) + return failure ?? `orcad ${incumbent.version} was restored and is serving again.` +} diff --git a/src/main/ssh/orcad-runtime-conversion-wiring.ts b/src/main/ssh/orcad-runtime-conversion-wiring.ts new file mode 100644 index 00000000000..d1bdf06c8c7 --- /dev/null +++ b/src/main/ssh/orcad-runtime-conversion-wiring.ts @@ -0,0 +1,47 @@ +/** The live collaborators a host conversion needs: the relay, the direct session and the server. */ +import { encodePairingOffer } from '../../shared/pairing' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { disconnectRegisteredSshTarget } from '../ipc/ssh-session-teardown' +import type { OrcadMigrationDestinationCatalog } from './orcad-migration-cutover-coordinator' +import { + abortRemoteOrcadMigrationCatalog, + commitRemoteOrcadMigrationCatalog, + readRemoteOrcadMigrationCatalogState, + stageRemoteOrcadMigrationCatalog, + stageRemoteOrcadMigrationSnapshotChunk +} from './orcad-migration-catalog-client' +import { orcadMigrationRelayPtyLister } from './orcad-migration-relay-pty-lister' +import { censusHostRelayTerminalsFor } from './ssh-host-relay-census-for-target' +import type { OrcadManagedConversionArgs } from './orcad-runtime-conversion' + +/** The T6-9 client against this server, pinned to the runtime it paired with. */ +export function orcadMigrationDestinationFor( + environment: KnownRuntimeEnvironment +): OrcadMigrationDestinationCatalog { + const pairingCode = encodePairingOffer(getPreferredPairingOffer(environment)) + const options = environment.runtimeId ? { expectedRuntimeId: environment.runtimeId } : {} + return { + readState: (manifest) => readRemoteOrcadMigrationCatalogState(pairingCode, manifest, options), + stage: (manifest) => stageRemoteOrcadMigrationCatalog(pairingCode, manifest, options), + commit: (manifest) => commitRemoteOrcadMigrationCatalog(pairingCode, manifest, options), + abort: (manifest) => abortRemoteOrcadMigrationCatalog(pairingCode, manifest, options), + stageChunk: (request) => stageRemoteOrcadMigrationSnapshotChunk(pairingCode, request, options) + } +} + +export function conversionCollaborators( + sshTargetId: string +): Pick< + OrcadManagedConversionArgs, + 'destinationFor' | 'listRelayPtyIds' | 'censusHost' | 'releaseDirectSession' +> { + return { + destinationFor: orcadMigrationDestinationFor, + listRelayPtyIds: orcadMigrationRelayPtyLister(sshTargetId), + censusHost: (target) => censusHostRelayTerminalsFor(target)(), + releaseDirectSession: disconnectRegisteredSshTarget + } +} diff --git a/src/main/ssh/orcad-runtime-conversion.test.ts b/src/main/ssh/orcad-runtime-conversion.test.ts new file mode 100644 index 00000000000..254056129f6 --- /dev/null +++ b/src/main/ssh/orcad-runtime-conversion.test.ts @@ -0,0 +1,351 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { + addManagedOrcadEnvironment, + removeManagedOrcadEnvironment +} from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' +import type { ListRelayPtyIds } from './orcad-migration-terminal-gate' +import type { OrcadManagedConversionArgs } from './orcad-runtime-conversion' +import { fakeOrcadMigrationDestination } from './orcad-migration-destination-fake' +import { reconcileManagedOrcadSshTargets, visibleRepos } from './orcad-retained-source' +import { SshConnectionStore } from './ssh-connection-store' + +const mocks = vi.hoisted(() => { + const state: { targetStore: unknown } = { targetStore: null } + return { state, deploy: vi.fn(), ensureTunnel: vi.fn(), directAuthority: vi.fn() } +}) +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({}), + getSshTargetRegistryStore: () => mocks.state.targetStore, + hasRegisteredDirectSshAuthority: mocks.directAuthority +})) +vi.mock('./orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('./orcad-managed-tunnel', () => ({ + ensureOrcadManagedTunnel: mocks.ensureTunnel, + closeOrcadManagedTunnel: async () => {} +})) + +const { convertSshTargetToManagedOrcad } = await import('./orcad-runtime-conversion') +const { abandonOrcadConversion } = await import('./orcad-conversion-abandon') + +const TARGET: SshTarget = { + id: 'ssh-prod', + label: 'Production', + host: 'prod.example.com', + port: 22, + username: 'deploy', + generation: 2 +} + +let userDataPath: string +let store: Store +let destination: ReturnType + +beforeEach(() => { + vi.resetAllMocks() + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-conversion-')) + store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SshConnectionStore wraps the real test store it is given. + mocks.state.targetStore = new SshConnectionStore(store as never) + destination = fakeOrcadMigrationDestination() + mocks.directAuthority.mockReturnValue(false) + mocks.ensureTunnel.mockResolvedValue(undefined) + // Registers the server the fence named, as the real deploy would after pairing. + mocks.deploy.mockImplementation(async (path: string, args: { name: string }) => { + const owner = getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id)) + if (!owner) { + throw new Error('deploy without a fence') + } + if (!listEnvironments(path).some((entry) => entry.id === owner)) { + addManagedOrcadEnvironment(path, { + id: owner, + name: args.name, + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/', + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: TARGET.id, + sshTargetGeneration: 2, + localPort: 46_768, + remotePort: 6_768 + } + }) + } + return { outcome: 'created', environment: {}, activeVersion: '1.0.0' } + }) +}) + +afterEach(async () => { + await closeTestStores() + rmSync(userDataPath, { recursive: true, force: true }) +}) + +const releaseDirectSession = vi.fn(async () => {}) +// Every relay, this build's and earlier ones, answers that nothing runs. +const everyRelayEmpty = (): ListRelayPtyIds => + Object.assign(async () => [], { previous: async () => [] }) +const convert = ( + listRelayPtyIds: ListRelayPtyIds | null = everyRelayEmpty(), + // The account-wide census found no relay with work unless a test says otherwise. + censusHost: OrcadManagedConversionArgs['censusHost'] = async () => ({ + verdict: 'exited', + count: 0 + }) +) => + convertSshTargetToManagedOrcad(userDataPath, { + sshTargetId: TARGET.id, + name: 'Managed', + listRelayPtyIds, + censusHost, + destinationFor: () => destination, + releaseDirectSession, + now: () => new Date('2026-10-02T00:00:00.000Z') + }) + +describe('converting an SSH host into a managed server', () => { + it('fences, deploys, marks the server, commits once, then keeps the source rows', async () => { + const result = await convert() + expect(result).toMatchObject({ outcome: 'converted' }) + expect(releaseDirectSession).toHaveBeenCalledWith(TARGET.id) + expect(mocks.deploy).toHaveBeenCalledWith( + userDataPath, + expect.objectContaining({ migration: true }) + ) + const [environment] = listEnvironments(userDataPath) + expect(environment?.orcadMigratedAt).toBe('2026-10-02T00:00:00.000Z') + expect(destination.commits).toBe(1) + // An older build reads these rows and reaches the host over its relay. + expect(store.getRepos().map((repo) => repo.id)).toEqual(['repo-1']) + const [journal] = listOrcadMigrationSourceCutovers(userDataPath) + expect(journal).toMatchObject({ + phase: 'destination-committed', + sourceRetainedAt: '2026-10-02T00:00:00.000Z' + }) + expect(store.getSshTarget(TARGET.id)?.owner).toBeUndefined() + expect(getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))).toBe(environment?.id) + // Converting again only resumes the finished migration: nothing commits twice. + await expect(convert()).resolves.toMatchObject({ + outcome: 'converted', + migrationId: journal?.migrationId + }) + expect(destination.commits).toBe(1) + }) + + it('refuses before touching the host while terminals run or cannot be counted', async () => { + store.upsertSshRemotePtyLease({ targetId: TARGET.id, ptyId: 'p', state: 'expired' }) + await expect(convert(null)).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable' + }) + expect(releaseDirectSession).not.toHaveBeenCalled() + expect(store.getSshTarget(TARGET.id)?.orcadFence).toBeUndefined() + expect(listOrcadMigrationSourceCutovers(userDataPath)).toEqual([]) + }) + + // No relay session and no lease: only a host census may prove nothing runs, never the silence. + it('converts with no relay session only on a host census that proves its relays idle', async () => { + await expect(convert(null, null)).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable' + }) + await expect(convert(null, async () => ({ verdict: 'live', count: 1 }))).resolves.toMatchObject( + { outcome: 'refused', verdict: 'live' } + ) + expect(store.getSshTarget(TARGET.id)?.orcadFence).toBeUndefined() + + await expect( + convert(null, async () => ({ verdict: 'exited', count: 0 })) + ).resolves.toMatchObject({ outcome: 'converted' }) + }) + + it('keeps the fence across a deferred deploy and resumes the same migration', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'deferred', + candidateVersion: '1.0.0', + code: 'orcad_update_terminals_running', + reason: 'busy' + }) + await expect(convert()).resolves.toMatchObject({ outcome: 'deferred' }) + const [fenced] = listOrcadMigrationSourceCutovers(userDataPath) + expect(fenced?.phase).toBe('source-fenced') + await expect(convert(null)).resolves.toMatchObject({ + outcome: 'converted', + migrationId: fenced?.migrationId + }) + expect(destination.commits).toBe(1) + }) + + it('resumes after a lost commit reply without committing twice', async () => { + const read = destination.readState.getMockImplementation() + const commit = destination.commit.getMockImplementation() + let reachable = true + destination.readState.mockImplementation(async (manifest) => { + if (!reachable) { + throw new Error('socket closed') + } + return read!(manifest) + }) + // The commit lands, then contact is lost before either the reply or a re-read arrives. + destination.commit.mockImplementationOnce(async (manifest) => { + await commit!(manifest) + reachable = false + throw new Error('socket closed') + }) + await expect(convert()).rejects.toThrow('socket closed') + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]?.phase).toBe('destination-staged') + reachable = true + await expect(convert(null)).resolves.toMatchObject({ outcome: 'converted' }) + expect(destination.commits).toBe(1) + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]?.sourceRetainedAt).toBeDefined() + expect(store.getRepos().map((repo) => repo.id)).toEqual(['repo-1']) + }) +}) + +describe('the source frozen during a conversion', () => { + it('commits despite the UI moving focus to the host mid-conversion', async () => { + const stage = destination.stage.getMockImplementation()! + destination.stage.mockImplementationOnce(async (manifest) => { + store.updateUI({ lastActiveRepoId: 'repo-1' }) + return stage(manifest) + }) + await expect(convert()).resolves.toMatchObject({ outcome: 'converted' }) + }) + + it('still refuses a catalog change mid-conversion', async () => { + const stage = destination.stage.getMockImplementation()! + destination.stage.mockImplementationOnce(async (manifest) => { + store.updateRepo('repo-1', { displayName: 'Renamed' }) + return stage(manifest) + }) + await expect(convert()).rejects.toThrow('orcad_migration_source_changed') + }) +}) + +describe('backing out a conversion whose server is registered but never committed', () => { + const abandon = () => + abandonOrcadConversion({ + userDataPath, + store, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the test registry is the SshConnectionStore built above. + claims: (mocks.state.targetStore as SshConnectionStore).getOrcadRuntimeClaims(), + targetId: TARGET.id, + destinationFor: () => destination + }) + + it('aborts the staged catalog, unregisters the server and gives the host back', async () => { + destination.commit.mockRejectedValueOnce(new Error('orcad_migration_source_changed')) + await expect(convert()).rejects.toThrow('orcad_migration_source_changed') + expect(listOrcadMigrationSourceCutovers(userDataPath)[0]?.phase).toBe('destination-staged') + + await expect(abandon()).resolves.toMatchObject({ outcome: 'released' }) + expect(destination.abort).toHaveBeenCalled() + expect(listEnvironments(userDataPath)).toEqual([]) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toEqual([]) + expect(store.getSshTarget(TARGET.id)?.orcadFence).toBeUndefined() + // The next start finds no server to re-fence the host to. + reconcileManagedOrcadSshTargets(userDataPath, store) + expect(store.getSshTarget(TARGET.id)?.orcadFence).toBeUndefined() + }) + + it('keeps the fence while the server cannot say what it holds', async () => { + destination.commit.mockRejectedValueOnce(new Error('socket closed')) + await expect(convert()).rejects.toThrow('socket closed') + destination.readState.mockRejectedValue(new Error('socket closed')) + await expect(abandon()).rejects.toThrow('socket closed') + expect(listEnvironments(userDataPath)).toHaveLength(1) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + }) + + it('clears a stale journal a stopped server left, so the host converts again', async () => { + await expect(convert()).resolves.toMatchObject({ outcome: 'converted' }) + // A stopped server from a build that left its journal behind. + const [environment] = listEnvironments(userDataPath) + removeManagedOrcadEnvironment(userDataPath, environment!.id) + store.updateSshTarget(TARGET.id, { orcadFence: undefined }) + await expect(convert()).resolves.toMatchObject({ outcome: 'converted' }) + expect(listOrcadMigrationSourceCutovers(userDataPath)).toHaveLength(1) + }) +}) + +/** What v1.4.218 and current main do with a stored target: keep every field, hide only `owner`. */ +function shippedBuildView(targets: SshTarget[]): SshTarget[] { + return JSON.parse(JSON.stringify(targets)).filter( + (target: SshTarget) => target.owner?.type !== 'on-demand-runtime' + ) +} + +describe('a converted host across a downgrade and back', () => { + async function convertRetained(): Promise { + await expect(convert()).resolves.toMatchObject({ outcome: 'converted' }) + } + + it('stays visible, with its projects, to an older build', async () => { + await convertRetained() + const visible = shippedBuildView(store.getSshTargets()) + expect(visible.map((target) => target.id)).toEqual([TARGET.id]) + expect(store.getRepos().filter((repo) => repo.connectionId === TARGET.id)).toHaveLength(1) + // This build hides the retained rows and serves the host from its managed server instead. + expect(visibleRepos(store, () => userDataPath)).toEqual([]) + }) + + it('goes back to managed after a downgrade that changed nothing', async () => { + await convertRetained() + reconcileManagedOrcadSshTargets(userDataPath, store) + expect(store.getSshTarget(TARGET.id)?.orcadFence?.sourceChangedAt).toBeUndefined() + expect(visibleRepos(store, () => userDataPath)).toEqual([]) + }) + + it('keeps a host an older build changed on the relay, never merging a second manifest', async () => { + await convertRetained() + store.addRepo({ + id: 'repo-2', + path: '/srv/tool', + displayName: 'Tool', + badgeColor: '#737373', + addedAt: 2, + kind: 'git', + connectionId: TARGET.id + }) + reconcileManagedOrcadSshTargets(userDataPath, store, () => new Date('2026-10-05T00:00:00Z')) + expect(store.getSshTarget(TARGET.id)?.orcadFence).toMatchObject({ + sourceChangedAt: '2026-10-05T00:00:00.000Z' + }) + expect( + visibleRepos(store, () => userDataPath) + .map((repo) => repo.id) + .sort() + ).toEqual(['repo-1', 'repo-2']) + expect(destination.commits).toBe(1) + }) + + it('restores a fence the profile lost from the registered managed server', async () => { + await convertRetained() + store.updateSshTarget(TARGET.id, { orcadFence: undefined }) + reconcileManagedOrcadSshTargets(userDataPath, store) + expect(getManagedOrcadFenceEnvironmentId(store.getSshTarget(TARGET.id))).toBe( + listEnvironments(userDataPath)[0]?.id + ) + }) +}) diff --git a/src/main/ssh/orcad-runtime-conversion.ts b/src/main/ssh/orcad-runtime-conversion.ts new file mode 100644 index 00000000000..32f4de50a0e --- /dev/null +++ b/src/main/ssh/orcad-runtime-conversion.ts @@ -0,0 +1,205 @@ +/** + * Converting a relay-hosted SSH target that holds Orca state into a managed orcad server: + * fence the source, deploy and pair the server, stage and commit the dormant catalog, then keep + * the source rows, hidden, for a downgraded build. + * + * Every step is keyed by the journal, so calling this again after a crash or lost contact resumes + * the same migration instead of starting another. The source stays authoritative until the + * destination proves its commit, and is never deleted afterwards. + */ +import { randomUUID } from 'node:crypto' +import type { OrcadManagedConversionResult } from '../../shared/orcad-managed-runtime' +import type { OrcadMigrationSourceCutover } from '../../shared/orcad-migration-source-cutover' +import { recordManagedOrcadMigration } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { + redactRuntimeEnvironment, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import { environmentMatchesManagedOrcadCutover } from './orcad-managed-migration-status' +import { requireManagedOrcadInfrastructure } from './orcad-managed-runtime-context' +import { ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { + commitOrcadMigrationDestination, + type OrcadMigrationDestinationCatalog +} from './orcad-migration-cutover-coordinator' +import { removeOrcadMigrationJournalsForDestination } from './orcad-migration-cutover-journal' +import { + fenceOrcadMigrationSource, + resolveOrcadMigrationFence +} from './orcad-migration-source-fence' +import type { SshTarget } from '../../shared/ssh-types' +import { + assessOrcadMigrationTerminals, + type CensusHostRelayTerminals, + retireProvenExitedLeases, + type ListRelayPtyIds +} from './orcad-migration-terminal-gate' +import { createManagedOrcadEnvironment } from './orcad-runtime-deployment' +import { retainOrcadMigrationSource } from './orcad-migration-source-retention' +import { hasRegisteredDirectSshAuthority } from './ssh-target-registry' + +export type OrcadManagedConversionArgs = { + sshTargetId: string + name: string + /** The relay's process list, asked while the host is still connected directly. */ + listRelayPtyIds: ListRelayPtyIds | null + /** With no relay session to ask, the census of the host's relay endpoints that must prove exit. */ + censusHost?: ((target: SshTarget) => ReturnType) | null + /** The destination's T6-9 catalog client, reached through the server's tunnel. */ + destinationFor: (environment: KnownRuntimeEnvironment) => OrcadMigrationDestinationCatalog + /** Releases the direct SSH session after the terminal check, before the fence. */ + releaseDirectSession: (sshTargetId: string) => Promise + now?: () => Date + signal?: AbortSignal +} + +export async function convertSshTargetToManagedOrcad( + userDataPath: string, + args: OrcadManagedConversionArgs +): Promise { + const fenced = await fenceOrResume(userDataPath, args) + if ('outcome' in fenced) { + return fenced + } + const deployed = await createManagedOrcadEnvironment(userDataPath, { + name: fenced.destinationName, + sshTargetId: fenced.sshTargetId, + migration: true, + signal: args.signal + }) + if (deployed.outcome === 'deferred') { + return deployed // The fence and journal stay; a later call resumes here. + } + const environment = listEnvironments(userDataPath).find( + (entry) => entry.id === fenced.destinationEnvironmentId + ) + if (!environment || !environmentMatchesManagedOrcadCutover(environment, fenced)) { + throw new Error('orcad_migration_destination_environment_mismatch') + } + // Before any commit can land, so a crash after it still blocks rollbacks across it. + const marked = recordManagedOrcadMigration( + userDataPath, + environment.id, + (args.now ?? (() => new Date()))().toISOString() + ) + await ensureOrcadManagedTunnel(userDataPath, environment.id) + const { claims, targetStore } = requireManagedOrcadInfrastructure() + const committed = await runTargetLifecycle(fenced.sshTargetId, () => + commitOrcadMigrationDestination( + { + userDataPath, + store: targetStore.getOrcadMigrationSource(), + claims, + destination: args.destinationFor(marked), + now: args.now + }, + fenced.migrationId + ) + ) + if (committed.phase !== 'destination-committed' && committed.phase !== 'source-retired') { + throw new Error(`orcad_migration_commit_not_proven:${committed.phase}`) + } + if (committed.phase === 'destination-committed') { + retainOrcadMigrationSource(userDataPath, committed.migrationId, args.now) + } + return { + outcome: 'converted', + environment: redactRuntimeEnvironment(marked), + migrationId: committed.migrationId + } +} + +async function fenceOrResume( + userDataPath: string, + args: OrcadManagedConversionArgs +): Promise< + OrcadMigrationSourceCutover | Extract +> { + const { claims, targetStore } = requireManagedOrcadInfrastructure() + const target = targetStore.getTarget(args.sshTargetId) + if (!target) { + return refuse('unverifiable', 'orcad_migration_target_not_found', 'The SSH host is gone.') + } + const existing = resolveOrcadMigrationFence(userDataPath, target) + if (existing.state === 'fenced') { + return existing.cutover + } + const stale = existing.state === 'stale-journal' ? existing.cutover : null + if (stale && !isRegistered(userDataPath, stale.destinationEnvironmentId)) { + // Its fence is gone and its server unregistered, so nothing it records can still be acted on. + removeOrcadMigrationJournalsForDestination( + userDataPath, + target.id, + stale.destinationEnvironmentId + ) + } + const converted = listEnvironments(userDataPath).some( + (environment) => environment.orcadDeployment?.sshTargetId === target.id + ) + if (converted) { + return refuse( + 'live', + 'orcad_migration_already_managed', + 'This SSH host is already a managed server.' + ) + } + const store = targetStore.getOrcadMigrationSource() + // Asked while the relay still answers; the fence re-checks leases once it holds. + const censusHost = args.censusHost + const terminalProof = await assessOrcadMigrationTerminals( + store, + target.id, + args.listRelayPtyIds, + censusHost ? () => censusHost(target) : null + ) + if (terminalProof.verdict !== 'exited') { + return refuse(terminalProof.verdict, 'orcad_migration_terminals', terminalProof.reason) + } + retireProvenExitedLeases(store, target.id, terminalProof) + await args.releaseDirectSession(target.id) + const result = await runTargetLifecycle(target.id, () => + fenceOrcadMigrationSource({ + userDataPath, + store, + claims, + targetId: target.id, + destinationEnvironmentId: randomUUID(), + destinationName: args.name, + terminalProof, + hasDirectSshAuthority: hasRegisteredDirectSshAuthority, + now: args.now, + signal: args.signal + }) + ) + if (result.outcome === 'fenced') { + return result.cutover + } + if (result.blockers?.length) { + // Why logged: the connect surfaces only the reason, and support needs which state blocked. + console.warn( + '[ssh] Conversion refused by:', + JSON.stringify( + result.blockers.map((blocker) => + 'dependencies' in blocker + ? { code: blocker.code, dependencies: blocker.dependencies } + : { code: blocker.code } + ) + ) + ) + } + return refuse(result.verdict, result.code, result.reason) +} + +function isRegistered(userDataPath: string, environmentId: string): boolean { + return listEnvironments(userDataPath).some((environment) => environment.id === environmentId) +} + +function refuse( + verdict: 'live' | 'unverifiable', + code: string, + reason: string +): Extract { + return { outcome: 'refused', verdict, code, reason } +} diff --git a/src/main/ssh/orcad-runtime-decommission.test.ts b/src/main/ssh/orcad-runtime-decommission.test.ts new file mode 100644 index 00000000000..6c06a8e0b29 --- /dev/null +++ b/src/main/ssh/orcad-runtime-decommission.test.ts @@ -0,0 +1,253 @@ +import { existsSync, readFileSync, rmSync } from 'node:fs' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { getRuntimeEnvironmentSidecarPath } from '../../shared/runtime-environment-sidecar' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { + listOrcadMigrationSourceCutovers, + writeOrcadMigrationSourceCutover +} from './orcad-migration-cutover-journal' +import { orcadMigrationCutoverFixture } from './orcad-migration-cutover-fixture' +import { + createManagedLifecycleHarness, + MANAGED_VERSION +} from './orcad-managed-lifecycle-test-fixture' + +const mocks = vi.hoisted(() => { + const state: { store: unknown; transaction: unknown } = { store: null, transaction: null } + return { + state, + resolveContext: vi.fn(), + census: vi.fn(), + decommission: vi.fn(), + recover: vi.fn(), + cancel: vi.fn(), + keepServing: vi.fn(), + closeTunnel: vi.fn(), + ensureTunnel: vi.fn(), + retire: vi.fn() + } +}) + +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({ connect: async () => ({}) }), + getSshTargetRegistryStore: () => mocks.state.store +})) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext: mocks.resolveContext })) +vi.mock('./orcad-terminal-census-client', () => ({ collectManagedTerminalCensus: mocks.census })) +vi.mock('./orcad-remote-stop', () => ({ decommissionRemoteOrcad: mocks.decommission })) +vi.mock('./orcad-activation-recovery', () => ({ recoverInterruptedOrcadActivation: mocks.recover })) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: async () => mocks.state.transaction +})) +vi.mock('./orcad-activation-lock', () => ({ + withStaleOrcadActivationRecoveryLock: async ( + _options: unknown, + run: (lock: { retain: () => void }) => Promise + ) => run({ retain: () => {} }) +})) +vi.mock('./orcad-managed-remote-stop', () => ({ cancelRemoteOrcadManagedStop: mocks.cancel })) +vi.mock('./orcad-decommission-recovery', () => ({ reconcileOrcadDecommission: mocks.keepServing })) +vi.mock('./orcad-managed-tunnel', () => ({ + closeOrcadManagedTunnel: mocks.closeTunnel, + ensureOrcadManagedTunnel: mocks.ensureTunnel +})) + +const { cancelManagedOrcadStop, stopManagedOrcadEnvironment } = + await import('./orcad-runtime-lifecycle') + +const idle = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 7 } +let harness: ReturnType + +const stop = (isActive = false) => + stopManagedOrcadEnvironment( + harness.userDataPath, + { selector: 'Managed' }, + { isActiveEnvironment: () => isActive, retireLocalState: mocks.retire } + ) + +function expectStillLinked(): void { + expect(listEnvironments(harness.userDataPath)[0]?.orcadDeployment).toBeDefined() + expect(getManagedOrcadFenceEnvironmentId(harness.current())).toBe('environment-1') + expect(mocks.retire).not.toHaveBeenCalled() + expect(mocks.closeTunnel).not.toHaveBeenCalled() +} + +beforeEach(() => { + vi.resetAllMocks() + harness = createManagedLifecycleHarness() + mocks.state.store = harness.targetStore + mocks.state.transaction = null + mocks.resolveContext.mockImplementation(async () => harness.context()) + mocks.census.mockResolvedValue(idle) +}) + +afterEach(() => rmSync(harness.userDataPath, { recursive: true, force: true })) + +describe('stopManagedOrcadEnvironment', () => { + it('unlinks the server only after the host proves orcad exited', async () => { + writeOrcadMigrationSourceCutover(harness.userDataPath, { + ...orcadMigrationCutoverFixture('migration-1', 'ssh-1', { environmentId: 'environment-1' }), + phase: 'destination-committed', + sourceRetainedAt: '2026-10-01T00:00:00.000Z' + }) + mocks.decommission.mockResolvedValueOnce({ + outcome: 'decommissioned', + version: MANAGED_VERSION, + retirement: 'retired' + }) + await expect(stop()).resolves.toEqual({ + outcome: 'unlinked', + verdict: 'exited', + environmentId: 'environment-1', + sshTargetId: 'ssh-1', + stoppedVersion: MANAGED_VERSION, + retirement: 'retired' + }) + expect(mocks.decommission.mock.calls[0]?.[0]).toMatchObject({ census: idle, port: 6_768 }) + expect(listEnvironments(harness.userDataPath)).toEqual([]) + expect( + readFileSync(getRuntimeEnvironmentSidecarPath(harness.userDataPath), 'utf8') + ).not.toContain('orcadDeployment') + expect(harness.current().orcadFence).toBeUndefined() + expect(harness.flushes).toHaveLength(1) + expect(mocks.retire).toHaveBeenCalledWith('environment-1') + expect(mocks.closeTunnel).toHaveBeenCalledWith('environment-1') + // Its retained journal would otherwise block every later conversion of the host. + expect(listOrcadMigrationSourceCutovers(harness.userDataPath)).toEqual([]) + }) + + it.each([ + ['live', 'orcad_decommission_terminals_running'], + ['unverifiable', 'orcad_decommission_census_unavailable'], + ['unverifiable', 'orcad_decommission_stop_unsettled'] + ] as const)('keeps every link when the verdict is %s (%s)', async (verdict, code) => { + mocks.decommission.mockResolvedValueOnce({ outcome: 'refused', verdict, code, reason: 'no' }) + await expect(stop()).resolves.toMatchObject({ outcome: 'refused', verdict, code }) + expectStillLinked() + expect(existsSync(getRuntimeEnvironmentSidecarPath(harness.userDataPath))).toBe(true) + }) + + it('refuses the Active Server before contacting the host', async () => { + await expect(stop(true)).resolves.toMatchObject({ code: 'orcad_stop_active_environment' }) + expect(mocks.resolveContext).not.toHaveBeenCalled() + expectStillLinked() + }) + + it('finishes an interrupted stop from its journal and unlinks only on proven exit', async () => { + mocks.state.transaction = { operation: 'decommission', activeVersion: MANAGED_VERSION } + mocks.recover.mockResolvedValueOnce({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_decommission_unsettled', + reason: 'no answer' + }) + await expect(stop()).resolves.toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expectStillLinked() + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'committed', + activeVersion: null, + readiness: null + }) + await expect(stop()).resolves.toMatchObject({ outcome: 'unlinked', verdict: 'exited' }) + expect(mocks.decommission).not.toHaveBeenCalled() + }) + + it('starts over, never reading live, when another run settled the journal first', async () => { + mocks.state.transaction = { operation: 'decommission', activeVersion: MANAGED_VERSION } + mocks.recover.mockImplementationOnce(async () => { + mocks.state.transaction = null + return { outcome: 'none' } + }) + mocks.decommission.mockResolvedValueOnce({ + outcome: 'decommissioned', + version: MANAGED_VERSION, + retirement: null + }) + await expect(stop()).resolves.toMatchObject({ outcome: 'unlinked', verdict: 'exited' }) + + rmSync(harness.userDataPath, { recursive: true, force: true }) + harness = createManagedLifecycleHarness() + mocks.state.store = harness.targetStore + mocks.state.transaction = { operation: 'decommission', activeVersion: MANAGED_VERSION } + mocks.recover.mockResolvedValue({ outcome: 'none' }) + await expect(stop()).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_stop_journal_changed' + }) + }) + + it('refuses while another interrupted operation holds the journal', async () => { + mocks.state.transaction = { operation: 'activate' } + await expect(stop()).resolves.toMatchObject({ code: 'orcad_activation_recovery_required' }) + expectStillLinked() + }) + + it('unlinks a server whose record already shows nothing active', async () => { + mocks.resolveContext.mockImplementation(async () => harness.context({ active: null })) + await expect(stop()).resolves.toMatchObject({ outcome: 'unlinked', stoppedVersion: null }) + expect(mocks.decommission).not.toHaveBeenCalled() + }) +}) + +describe('cancelManagedOrcadStop', () => { + const request = { transactionId: 't-1', version: MANAGED_VERSION } + const cancel = () => cancelManagedOrcadStop(harness.userDataPath, { selector: 'Managed' }) + + it('has nothing to cancel without a decommission journal', async () => { + await expect(cancel()).resolves.toEqual({ outcome: 'none' }) + expect(mocks.cancel).not.toHaveBeenCalled() + }) + + it('withdraws a stop orcad never acted on and keeps the server serving', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'stop-dispatched', + activeVersion: MANAGED_VERSION, + request + } + mocks.cancel.mockResolvedValueOnce({ outcome: 'canceled' }) + mocks.keepServing.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: MANAGED_VERSION, + readiness: null + }) + await expect(cancel()).resolves.toEqual({ outcome: 'canceled', activeVersion: MANAGED_VERSION }) + expect(mocks.keepServing.mock.calls[0]?.[1]).toEqual({ + action: 'keep-serving', + version: MANAGED_VERSION + }) + expect(mocks.ensureTunnel).toHaveBeenCalledOnce() + expectStillLinked() + }) + + it('refuses once orcad already acted on the stop', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'stop-dispatched', + activeVersion: MANAGED_VERSION, + request + } + mocks.cancel.mockResolvedValueOnce({ outcome: 'dispatched' }) + await expect(cancel()).resolves.toMatchObject({ + outcome: 'refused', + verdict: 'live', + code: 'orcad_stop_already_dispatched' + }) + expect(mocks.keepServing).not.toHaveBeenCalled() + }) + + it('reports an already-exited stop for stop to finish', async () => { + mocks.state.transaction = { + operation: 'decommission', + phase: 'process-exited', + activeVersion: MANAGED_VERSION, + request + } + await expect(cancel()).resolves.toEqual({ outcome: 'already-stopped' }) + expectStillLinked() + }) +}) diff --git a/src/main/ssh/orcad-runtime-decommission.ts b/src/main/ssh/orcad-runtime-decommission.ts new file mode 100644 index 00000000000..3ac50ddcd60 --- /dev/null +++ b/src/main/ssh/orcad-runtime-decommission.ts @@ -0,0 +1,220 @@ +/** + * Stopping a managed orcad and unlinking it, on T6-4's journaled decommission. The server stays + * linked — its SSH claim, tunnel and deployment record — unless the host proves orcad exited. + */ +import type { + OrcadManagedCancelStopResult, + OrcadManagedStopResult +} from '../../shared/orcad-managed-runtime' +import type { OrcadDaemonRetirementVerdict } from '../../shared/orcad-stop-request' +import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import type { + KnownRuntimeEnvironment, + OrcadDeploymentLink +} from '../../shared/runtime-environments' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { withStaleOrcadActivationRecoveryLock } from './orcad-activation-lock' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { reconcileOrcadDecommission } from './orcad-decommission-recovery' +import { cancelRemoteOrcadManagedStop } from './orcad-managed-remote-stop' +import { + managedOrcadSlot, + requireManagedOrcadInfrastructure, + resolveLinkedOrcadContext +} from './orcad-managed-runtime-context' +import { closeOrcadManagedTunnel, ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { clearManagedOrcadUpdateDeferral } from './orcad-managed-update-deferrals' +import { removeOrcadMigrationJournalsForDestination } from './orcad-migration-cutover-journal' +import { decommissionRemoteOrcad } from './orcad-remote-stop' +import { withManagedOrcadLifecycle } from './orcad-runtime-maintenance' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' +import { RemoteInstallLockBusyError } from './ssh-relay-install-lock' + +export type ManagedOrcadStopPolicy = { + isActiveEnvironment: (environmentId: string) => boolean + /** Invalidates the server's transport and retires its client-side state once it is unlinked. */ + retireLocalState: (environmentId: string) => Promise | void +} + +type Refusal = Extract +type Stopped = { version: string | null; retirement: OrcadDaemonRetirementVerdict | null } + +function refuse(verdict: Refusal['verdict'], code: string, reason: string): Refusal { + return { outcome: 'refused', verdict, code, reason } +} + +export function stopManagedOrcadEnvironment( + userDataPath: string, + args: { selector: string; signal?: AbortSignal }, + policy: ManagedOrcadStopPolicy +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async (managed) => { + const { environment, deployment } = managed + if (policy.isActiveEnvironment(environment.id)) { + return refuse( + 'live', + 'orcad_stop_active_environment', + 'Choose another Active Server in Advanced before stopping this server.' + ) + } + const stopped = await stopRemote(userDataPath, environment, deployment, args.signal) + if ('outcome' in stopped) { + return stopped + } + await unlinkStoppedEnvironment(userDataPath, environment, deployment, policy) + return { + outcome: 'unlinked', + verdict: 'exited', + environmentId: environment.id, + sshTargetId: deployment.sshTargetId, + stoppedVersion: stopped.version, + retirement: stopped.retirement + } + }) +} + +/** Proven exit, or the refusal that keeps the server linked. */ +async function stopRemote( + userDataPath: string, + environment: KnownRuntimeEnvironment, + deployment: OrcadDeploymentLink, + signal?: AbortSignal, + retried = false +): Promise { + const context = await resolveLinkedOrcadContext(environment, deployment, signal) + const slot = managedOrcadSlot(context, deployment.remotePort, signal) + const transaction = await readOrcadActivationTransaction(slot) + if (transaction?.operation === 'decommission') { + // An earlier stop was interrupted: its journal decides, so finish exactly that one. + const recovered = await recoverInterruptedOrcadActivation(slot) + if (recovered.outcome === 'recovered' && recovered.activeVersion === null) { + return { version: transaction.activeVersion, retirement: null } + } + if (recovered.outcome === 'refused') { + return refuse(recovered.verdict, recovered.code, recovered.reason) + } + if (recovered.outcome === 'pending') { + return refuse('unverifiable', recovered.code, recovered.reason) + } + // Another run settled the journal first; start over from what it left, never assume live. + if (recovered.outcome === 'none') { + return retried + ? refuse( + 'unverifiable', + 'orcad_stop_journal_changed', + 'Another run on this server changed its stop while this one waited. Refresh and retry.' + ) + : stopRemote(userDataPath, environment, deployment, signal, true) + } + return refuse( + 'live', + 'orcad_stop_withdrawn', + 'The interrupted stop was withdrawn and the server keeps serving. Stop it again.' + ) + } + if (transaction) { + return refuse( + 'unverifiable', + 'orcad_activation_recovery_required', + 'An earlier update on this server was interrupted. Recover it before stopping.' + ) + } + const record = context.activationRecord + if (!record.active) { + // Only a proven exit (or a deploy that never activated) leaves the record without a version. + return { version: null, retirement: null } + } + const census = await collectManagedTerminalCensus(userDataPath, environment, record) + const result = await decommissionRemoteOrcad({ ...slot, record, census }) + if (result.outcome === 'refused') { + return refuse(result.verdict, result.code, result.reason) + } + return { version: result.version, retirement: result.retirement } +} + +async function unlinkStoppedEnvironment( + userDataPath: string, + environment: KnownRuntimeEnvironment, + deployment: OrcadDeploymentLink, + policy: ManagedOrcadStopPolicy +): Promise { + // Environment first: a crash before the claim is released leaves a hidden target to resume, + // never a linked server whose SSH target was already handed back. + removeManagedOrcadEnvironment(userDataPath, environment.id) + await policy.retireLocalState(environment.id) + await closeOrcadManagedTunnel(environment.id) + const { claims } = requireManagedOrcadInfrastructure() + claims.release(deployment.sshTargetId, environment.id) + clearManagedOrcadUpdateDeferral(environment.id) + await claims.flush() + // After the fence: a crash between leaves a stale journal a later conversion clears. + removeOrcadMigrationJournalsForDestination(userDataPath, deployment.sshTargetId, environment.id) +} + +/** Withdraws a stop orcad has not acted on yet; the server then keeps serving. */ +export function cancelManagedOrcadStop( + userDataPath: string, + args: { selector: string; signal?: AbortSignal } +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async (managed) => { + const { environment, deployment } = managed + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + let result: OrcadManagedCancelStopResult + try { + result = await withStaleOrcadActivationRecoveryLock(slot, async (lock) => { + const transaction = await readOrcadActivationTransaction(slot) + if (transaction?.operation !== 'decommission') { + if (transaction) { + lock.retain() + } + return { outcome: 'none' } + } + if (transaction.phase === 'process-exited') { + lock.retain() + return { outcome: 'already-stopped' } + } + if (transaction.request) { + const cancellation = await cancelRemoteOrcadManagedStop(slot, transaction.request) + if (cancellation.outcome !== 'canceled') { + lock.retain() + return { + outcome: 'refused', + verdict: 'live', + code: 'orcad_stop_already_dispatched', + reason: 'orcad already acted on the stop and is shutting down; it cannot be canceled.' + } + } + } + const kept = await reconcileOrcadDecommission(slot, { + action: 'keep-serving', + version: transaction.activeVersion + }) + if (kept.outcome !== 'recovered' || !kept.activeVersion) { + lock.retain() + return { + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_stop_cancel_unverifiable', + reason: 'The stop was withdrawn but the server could not be shown to be serving.' + } + } + return { outcome: 'canceled', activeVersion: kept.activeVersion } + }) + } catch (error) { + if (error instanceof RemoteInstallLockBusyError) { + return { + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_stop_still_running', + reason: 'A stop on this server may still be running. Retry after it settles.' + } + } + throw error + } + if (result.outcome === 'canceled') { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return result + }) +} diff --git a/src/main/ssh/orcad-runtime-deployment.test.ts b/src/main/ssh/orcad-runtime-deployment.test.ts new file mode 100644 index 00000000000..f0f5be86e48 --- /dev/null +++ b/src/main/ssh/orcad-runtime-deployment.test.ts @@ -0,0 +1,432 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { writeOrcadMigrationSourceCutover } from './orcad-migration-cutover-journal' +import { orcadMigrationCutoverFixture } from './orcad-migration-cutover-fixture' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { SshTarget } from '../../shared/ssh-types' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { emptyDependentStateStore } from './ssh-target-orcad-dependents-fixture' + +const mocks = vi.hoisted(() => { + const state: { store: unknown } = { store: null } + return { + state, + events: new Array(), + connect: vi.fn(), + hasDirectAuthority: vi.fn(), + resolveContext: vi.fn(), + recover: vi.fn(), + readRecord: vi.fn(), + deploy: vi.fn(), + probe: vi.fn(), + startTunnel: vi.fn(), + ensureTunnel: vi.fn(), + closeTunnel: vi.fn(), + readTransaction: vi.fn() + } +}) + +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({ connect: mocks.connect }), + getSshTargetRegistryStore: () => mocks.state.store, + hasRegisteredDirectSshAuthority: mocks.hasDirectAuthority +})) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext: mocks.resolveContext })) +vi.mock('./orcad-activation-recovery', () => ({ recoverInterruptedOrcadActivation: mocks.recover })) +vi.mock('./orcad-activation-record-store', () => ({ readOrcadActivationRecord: mocks.readRecord })) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: mocks.readTransaction +})) +vi.mock('./orcad-remote-deploy', () => ({ deployOrcad: mocks.deploy })) +vi.mock('./orcad-artifact-materializer', () => ({ + materializeOrcadArtifact: async () => '/local/orcad' +})) +vi.mock('./orcad-local-build-hash', () => ({ computeLocalOrcadBuildHash: () => 'build-hash' })) +vi.mock('./orcad-active-readiness', () => ({ probeActiveOrcadReadiness: mocks.probe })) +vi.mock('./orcad-terminal-census-client', () => ({ + collectManagedTerminalCensus: async () => ({ + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: 39 + }) +})) +vi.mock('./orcad-managed-tunnel', () => ({ + startOrcadManagedTunnel: mocks.startTunnel, + ensureOrcadManagedTunnel: mocks.ensureTunnel, + closeOrcadManagedTunnel: mocks.closeTunnel +})) + +const { createManagedOrcadEnvironment, getManagedOrcadRuntimeStatus } = + await import('./orcad-runtime-lifecycle') + +const VERSION = '0.1.0+abc123' +const emptyRecord = { active: null, previous: null, activatedAt: null, snapshot: null } + +function readiness(endpoint = 'ws://127.0.0.1:6768') { + return { + runtimeId: 'runtime-1', + boundEndpoint: endpoint, + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { + available: true, + url: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint, + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + endpoint, + deviceId: 'device-1', + webClientUrl: null, + scope: 'runtime', + qr: null + } + } +} + +let userDataPath: string +let target: SshTarget +let flushes: number + +function setupStore(overrides: { repos?: { connectionId: string }[] } = {}) { + const store = { + allocateSshTargetGeneration: () => 9, + flushPendingOrThrowAsync: async () => { + flushes += 1 + mocks.events.push('flush') + }, + getFolderWorkspaces: () => [], + getRepos: () => overrides.repos ?? [], + ...emptyDependentStateStore(), + getSshTarget: (id: string) => (id === target.id ? target : undefined), + getSshTargets: () => [target], + updateSshTarget: (_id: string, updates: Partial) => { + target = { ...target, ...updates } + return target + } + } + mocks.state.store = { + getTarget: (id: string) => (id === target.id ? target : undefined), + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the claims read only the store methods stubbed above. + getOrcadRuntimeClaims: () => new SshTargetOrcadClaims(store as never) + } +} + +beforeEach(() => { + vi.resetAllMocks() + mocks.events.length = 0 + flushes = 0 + userDataPath = mkdtempSync(join(tmpdir(), 'orcad-deployment-test-')) + target = { id: 'ssh-1', label: 'Builder', host: 'builder', port: 22, username: 'dev' } + setupStore() + mocks.hasDirectAuthority.mockReturnValue(false) + mocks.connect.mockImplementation(async () => { + mocks.events.push('connect') + return {} + }) + mocks.resolveContext.mockImplementation(async (claimed: SshTarget) => ({ + activationRecord: emptyRecord, + serverTarget: 'linux-x64-glibc', + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target: claimed, + userDataDir: '/home/dev/.orca' + })) + mocks.recover.mockResolvedValue({ outcome: 'none' }) + mocks.deploy.mockResolvedValue({ outcome: 'installed-and-activated', fullVersion: VERSION }) + mocks.probe.mockResolvedValue(readiness()) + mocks.startTunnel.mockResolvedValue(46_768) + mocks.closeTunnel.mockResolvedValue(undefined) + mocks.ensureTunnel.mockResolvedValue(undefined) +}) + +afterEach(() => rmSync(userDataPath, { recursive: true, force: true })) + +const deploy = () => + createManagedOrcadEnvironment(userDataPath, { name: 'Managed', sshTargetId: 'ssh-1' }) + +describe('createManagedOrcadEnvironment', () => { + it('claims the target durably before contacting it, then registers a tunneled server', async () => { + const result = await deploy() + + expect(mocks.events.slice(0, 2)).toEqual(['flush', 'connect']) + expect(result).toMatchObject({ outcome: 'created', activeVersion: VERSION }) + const [environment] = listEnvironments(userDataPath) + expect(environment?.orcadDeployment).toEqual({ + sshTargetId: 'ssh-1', + sshTargetGeneration: 9, + localPort: 46_768, + remotePort: 6_768 + }) + expect(environment?.connectionDependency).toBe('ssh-tunnel') + expect(getManagedOrcadFenceEnvironmentId(target)).toBe(environment?.id) + expect(target.orcadProvisioning).toEqual({ requestId: environment?.id, name: 'Managed' }) + expect(JSON.stringify(result)).not.toContain('device-token') + expect(mocks.probe).toHaveBeenCalledWith( + expect.objectContaining({ remoteInstallDir: expect.any(String) }), + { + buildHash: 'build-hash', + fullVersion: VERSION + } + ) + expect(mocks.closeTunnel).not.toHaveBeenCalled() + }) + + it('tunnels to the port orcad bound when 6768 is taken, and keeps 6768 as its launch port', async () => { + mocks.probe.mockResolvedValue(readiness('ws://127.0.0.1:58520')) + await deploy() + + const [, , , remotePort, checks] = mocks.startTunnel.mock.calls[0] ?? [] + expect(remotePort).toBe(58_520) + expect(checks).toMatchObject({ preferredPort: 6_768 }) + expect(listEnvironments(userDataPath)[0]?.orcadDeployment?.remotePort).toBe(6_768) + }) + + it('deploys an empty host with a zero census and an unknown one over an active slot', async () => { + await deploy() + expect(mocks.deploy.mock.calls[0]?.[0]).toMatchObject({ + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null }, + nodePath: '' + }) + }) + + it('refuses a host with direct SSH projects before claiming or connecting', async () => { + setupStore({ repos: [{ connectionId: 'ssh-1' }] }) + await expect(deploy()).rejects.toThrow('repositories or folder workspaces') + expect(target.orcadFence).toBeUndefined() + expect(mocks.connect).not.toHaveBeenCalled() + }) + + it('refuses a host that saved state still references, naming it, before claiming', async () => { + const leases = [{ ptyId: 'pty-1', state: 'expired' }] + const store = { + ...emptyDependentStateStore({ + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the census reads only ptyId and state. + getSshRemotePtyLeases: () => leases as never + }), + allocateSshTargetGeneration: () => 9, + flushPendingOrThrowAsync: async () => {}, + getFolderWorkspaces: () => [], + getRepos: () => [], + getSshTarget: () => target, + getSshTargets: () => [target], + updateSshTarget: (_id: string, updates: Partial) => + (target = { ...target, ...updates }) + } + mocks.state.store = { + getTarget: () => target, + getOrcadRuntimeClaims: () => new SshTargetOrcadClaims(store) + } + await expect(deploy()).rejects.toThrow('terminal-lease ×1 (pty-1 (expired))') + expect(target.orcadFence).toBeUndefined() + expect(mocks.connect).not.toHaveBeenCalled() + }) + + it('refuses a connected direct SSH session and a taken server name', async () => { + mocks.hasDirectAuthority.mockReturnValueOnce(true) + await expect(deploy()).rejects.toThrow('Disconnect this SSH host') + await deploy() + target = { ...target, id: 'ssh-2', orcadFence: undefined } + await expect( + createManagedOrcadEnvironment(userDataPath, { name: 'Managed', sshTargetId: 'ssh-2' }) + ).rejects.toThrow('already exists') + }) + + it('does not contact the host when the claim cannot be made durable', async () => { + const claims = new SshTargetOrcadClaims({ + allocateSshTargetGeneration: () => 9, + flushPendingOrThrowAsync: async () => { + throw new Error('disk full') + }, + getFolderWorkspaces: () => [], + getRepos: () => [], + ...emptyDependentStateStore(), + getSshTarget: () => target, + getSshTargets: () => [target], + updateSshTarget: (_id, updates) => (target = { ...target, ...updates }) + }) + mocks.state.store = { getTarget: () => target, getOrcadRuntimeClaims: () => claims } + await expect(deploy()).rejects.toThrow('disk full') + expect(mocks.connect).not.toHaveBeenCalled() + }) + + it('returns a deferral, keeps the claim and closes nothing it did not open', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: VERSION, + code: 'orcad_update_terminal_census_unavailable', + reason: 'unknown census' + }) + await expect(deploy()).resolves.toMatchObject({ outcome: 'deferred', forceable: false }) + expect(listEnvironments(userDataPath)).toEqual([]) + expect(getManagedOrcadFenceEnvironmentId(target)).not.toBeNull() + expect(mocks.startTunnel).not.toHaveBeenCalled() + }) + + it('resumes an interrupted deploy under the environment id its claim recorded', async () => { + mocks.probe.mockRejectedValueOnce(new Error('readiness unverifiable')) + await expect(deploy()).rejects.toThrow('readiness unverifiable') + const claimedId = getManagedOrcadFenceEnvironmentId(target) + expect(claimedId).not.toBeNull() + mocks.deploy.mockResolvedValueOnce({ outcome: 'already-active', fullVersion: VERSION }) + const result = await deploy() + expect(result).toMatchObject({ outcome: 'already-current' }) + expect(listEnvironments(userDataPath).map((entry) => entry.id)).toEqual([claimedId]) + }) + + it('closes the tunnel it opened when registration fails', async () => { + mocks.probe.mockResolvedValueOnce({ + ...readiness(), + pairing: { available: false, guidance: 'off' } + }) + mocks.startTunnel.mockResolvedValueOnce(46_768) + await expect(deploy()).rejects.toThrow('did not publish a pairing offer') + expect(mocks.closeTunnel).toHaveBeenCalledOnce() + expect(JSON.stringify(mocks.closeTunnel.mock.calls)).not.toContain('device-token') + }) + + it('finishes an already registered server by ensuring its tunnel, without redeploying', async () => { + await deploy() + mocks.resolveContext.mockImplementation(async (claimed: SshTarget) => ({ + activationRecord: { ...emptyRecord, active: VERSION }, + serverTarget: 'linux-x64-glibc', + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target: claimed, + userDataDir: '/home/dev/.orca' + })) + await expect(deploy()).resolves.toMatchObject({ outcome: 'already-current' }) + expect(mocks.deploy).toHaveBeenCalledOnce() + expect(mocks.ensureTunnel).toHaveBeenCalledOnce() + }) + + it('stops at an interrupted activation the host cannot reconcile yet', async () => { + mocks.recover.mockResolvedValueOnce({ + outcome: 'pending', + code: 'fresh', + reason: 'still fresh' + }) + await expect(deploy()).rejects.toThrow('still fresh') + expect(mocks.deploy).not.toHaveBeenCalled() + }) +}) + +describe('createManagedOrcadEnvironment for a migration', () => { + it('deploys into its own journaled fence without leaving a provisioning intent', async () => { + target = { ...target, orcadFence: { environmentId: 'env-m' }, generation: 9 } + writeOrcadMigrationSourceCutover( + userDataPath, + orcadMigrationCutoverFixture('m-1', 'ssh-1', { generation: 9, environmentId: 'env-m' }) + ) + await expect( + createManagedOrcadEnvironment(userDataPath, { + name: 'Managed', + sshTargetId: 'ssh-1', + migration: true + }) + ).resolves.toMatchObject({ outcome: 'created' }) + expect(listEnvironments(userDataPath).map((entry) => entry.id)).toEqual(['env-m']) + expect(target.orcadProvisioning).toBeUndefined() + }) + + it('refuses a migration deploy whose fence has no journal, before connecting', async () => { + target = { ...target, orcadFence: { environmentId: 'env-m' }, generation: 9 } + await expect( + createManagedOrcadEnvironment(userDataPath, { + name: 'Managed', + sshTargetId: 'ssh-1', + migration: true + }) + ).rejects.toThrow('orcad_migration_fence_required') + expect(mocks.connect).not.toHaveBeenCalled() + }) +}) + +describe('getManagedOrcadRuntimeStatus', () => { + it('reports the activation record and an interrupted transaction without repairing it', async () => { + await deploy() + const [environment] = listEnvironments(userDataPath) + mocks.resolveContext.mockImplementation(async (claimed: SshTarget) => ({ + activationRecord: { ...emptyRecord, active: VERSION, activatedAt: 'now' }, + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target: claimed + })) + mocks.readTransaction.mockResolvedValueOnce({ + operation: 'activate', + phase: 'candidate-ready', + candidateVersion: '0.2.0+def', + startedAt: 'then' + }) + await expect(getManagedOrcadRuntimeStatus(userDataPath, 'Managed')).resolves.toEqual({ + environmentId: environment?.id, + sshTargetId: 'ssh-1', + activeVersion: VERSION, + previousVersion: null, + activatedAt: 'now', + rollbackAvailable: false, + recovery: { + operation: 'activate', + phase: 'candidate-ready', + version: '0.2.0+def', + startedAt: 'then' + }, + terminals: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 39 }, + migration: null, + deferredUpdate: null + }) + expect(mocks.recover).toHaveBeenCalledTimes(1) + }) + + it('reports an interrupted decommission without finishing it', async () => { + await deploy() + mocks.readTransaction.mockResolvedValueOnce({ + operation: 'decommission', + phase: 'stop-dispatched', + activeVersion: VERSION, + startedAt: 'then' + }) + await expect(getManagedOrcadRuntimeStatus(userDataPath, 'Managed')).resolves.toMatchObject({ + recovery: { operation: 'decommission', phase: 'stop-dispatched', version: VERSION } + }) + }) + + it('reports an unfinished migration into the server', async () => { + await deploy() + const [environment] = listEnvironments(userDataPath) + writeOrcadMigrationSourceCutover(userDataPath, { + ...orcadMigrationCutoverFixture('m-2', 'ssh-1', { + generation: 9, + environmentId: environment!.id + }), + phase: 'destination-staged' + }) + mocks.resolveContext.mockImplementation(async (claimed: SshTarget) => ({ + activationRecord: { ...emptyRecord, active: VERSION }, + connection: {}, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/dev', + target: claimed + })) + await expect(getManagedOrcadRuntimeStatus(userDataPath, 'Managed')).resolves.toMatchObject({ + migration: { migrationId: 'm-2', phase: 'destination-staged' } + }) + }) + + it('refuses a server whose SSH registration was re-created', async () => { + await deploy() + target = { ...target, generation: 10 } + await expect(getManagedOrcadRuntimeStatus(userDataPath, 'Managed')).rejects.toThrow( + 'no longer valid' + ) + }) +}) diff --git a/src/main/ssh/orcad-runtime-deployment.ts b/src/main/ssh/orcad-runtime-deployment.ts new file mode 100644 index 00000000000..8688607f0cc --- /dev/null +++ b/src/main/ssh/orcad-runtime-deployment.ts @@ -0,0 +1,201 @@ +/** + * Deploys orcad onto an empty SSH host and pairs this client with it through a loopback tunnel. + * The target claim is the resume point: an interrupted deploy leaves the target owned by the + * environment id it was creating, and the next deploy of that target finishes the same one. + */ +import { getAppEnvironment } from '../../shared/app-environment' +import { randomUUID } from 'node:crypto' +import { assertRuntimeEnvironmentNotReconciling } from '../../shared/runtime-environment-reconciliation-record' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { redactRuntimeEnvironment } from '../../shared/runtime-environments' +import { + ORCAD_MANAGED_REMOTE_PORT, + type OrcadManagedDeployResult +} from '../../shared/orcad-managed-runtime' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import { materializeOrcadArtifact } from './orcad-artifact-materializer' +import { + closeOrcadManagedTunnel, + ensureOrcadManagedTunnel, + startOrcadManagedTunnel +} from './orcad-managed-tunnel' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { deployOrcad } from './orcad-remote-deploy' +import { pruneManagedOrcadVersions } from './orcad-managed-version-gc' +import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' +import { hasRegisteredDirectSshAuthority } from './ssh-target-registry' +import { resolveOrcadMigrationFence } from './orcad-migration-source-fence' +import type { SshTarget } from '../../shared/ssh-types' +import { deployedOrcadTunnelChecks } from './orcad-managed-tunnel-identity' +import { + isForceableOrcadDeferral, + managedOrcadSlot, + probeManagedOrcadReadiness, + requireManagedOrcadInfrastructure +} from './orcad-managed-runtime-context' + +export async function createManagedOrcadEnvironment( + userDataPath: string, + args: { + name: string + sshTargetId: string + force?: boolean + signal?: AbortSignal + /** Deploying into a target fenced by a migration journal rather than claiming an empty one. */ + migration?: boolean + } +): Promise { + return runTargetLifecycle(args.sshTargetId, async () => { + const { connectionManager, targetStore, claims } = requireManagedOrcadInfrastructure() + const environmentId = + getManagedOrcadFenceEnvironmentId(targetStore.getTarget(args.sshTargetId)) ?? randomUUID() + const environments = listEnvironments(userDataPath) + const registered = environments.find((entry) => entry.id === environmentId) + if (registered) { + assertRuntimeEnvironmentNotReconciling(registered) + if (registered.orcadDeployment?.sshTargetId !== args.sshTargetId) { + throw new Error('The SSH target is owned by a server that is not deployed on it.') + } + } + if (environments.some((entry) => entry.id !== environmentId && entry.name === args.name)) { + throw new Error(`A server named "${args.name}" already exists.`) + } + if (hasRegisteredDirectSshAuthority(args.sshTargetId)) { + throw new Error('Disconnect this SSH host before converting it to a managed Orca server.') + } + const current = targetStore.getTarget(args.sshTargetId) + if (args.migration) { + assertMigrationFence(userDataPath, current, environmentId, args.name) + } + const claimed = claims.claim(args.sshTargetId, environmentId, { + // A migration records itself in its journal, not as a provisioning intent. + ...(args.migration ? {} : { deployName: args.name }), + // Why: this deploy's own claim left a provisioning intent or a journal, or registered a server. + ownerRecorded: + Boolean(args.migration) || Boolean(current?.orcadProvisioning) || Boolean(registered) + }) + await claims.flush(args.signal) + const targetGeneration = claimed.generation + if (targetGeneration === undefined) { + throw new Error('The managed Orca SSH registration has no durable generation.') + } + if (registered && registered.orcadDeployment?.sshTargetGeneration !== targetGeneration) { + throw new Error('The saved managed Orca server has a stale SSH target generation.') + } + let environmentRegistered = false + try { + const connection = await connectionManager.connect(claimed) + let context = await resolveOrcadRemoteContext(claimed, connection, args.signal) + const slot = managedOrcadSlot(context, ORCAD_MANAGED_REMOTE_PORT, args.signal) + const recovery = await recoverInterruptedOrcadActivation(slot) + if (recovery.outcome === 'pending' || recovery.outcome === 'refused') { + throw new Error(recovery.reason) + } + if (recovery.outcome === 'recovered') { + context = { ...context, activationRecord: await readOrcadActivationRecord(slot) } + } + if (registered) { + environmentRegistered = true + await ensureOrcadManagedTunnel(userDataPath, registered.id) + const activeVersion = context.activationRecord.active + if (!activeVersion) { + throw new Error('The managed Orca server has no active runtime version.') + } + return { + outcome: 'already-current', + environment: redactRuntimeEnvironment(registered), + activeVersion + } + } + const localOrcadDir = await materializeOrcadArtifact(context.serverTarget, { + signal: args.signal + }) + const deployResult = await deployOrcad({ + ...slot, + conn: connection, + localOrcadDir, + target: context.serverTarget, + // Why unknown when a version is active: only the daemon can count its sessions, and + // a deploy that guessed zero would restart over live terminals. + census: context.activationRecord.active + ? { liveSessions: null, startedSinceActivation: null, daemonProtocolVersion: null } + : { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null }, + force: args.force, + appVersion: getAppEnvironment().getVersion() + }) + if (deployResult.outcome === 'installed-not-activated') { + return { + outcome: 'deferred', + candidateVersion: deployResult.fullVersion, + code: deployResult.code, + reason: deployResult.reason, + forceable: isForceableOrcadDeferral(deployResult.code) + } + } + const readiness = await probeManagedOrcadReadiness( + context, + localOrcadDir, + deployResult.fullVersion, + args.signal + ) + await pruneManagedOrcadVersions({ + slot, + serverTarget: context.serverTarget, + activeVersion: deployResult.fullVersion, + readiness + }) + // Why read back: orcad binds another port when the preferred one is taken on the host. + const tunnel = deployedOrcadTunnelChecks(readiness, () => + probeManagedOrcadReadiness(context, localOrcadDir, deployResult.fullVersion, args.signal) + ) + const localPort = await startOrcadManagedTunnel( + environmentId, + claimed, + connection, + tunnel.remotePort, + { ...tunnel, preferredPort: ORCAD_MANAGED_REMOTE_PORT } + ) + const environment = addManagedOrcadEnvironment(userDataPath, { + id: environmentId, + name: args.name, + pairingCode: tunneledOrcadPairingCode(tunnel.readiness(), localPort), + orcadDeployment: { + sshTargetId: claimed.id, + sshTargetGeneration: targetGeneration, + localPort, + remotePort: ORCAD_MANAGED_REMOTE_PORT + } + }) + environmentRegistered = true + return { + outcome: deployResult.outcome === 'already-active' ? 'already-current' : 'created', + environment: redactRuntimeEnvironment(environment), + activeVersion: deployResult.fullVersion + } + } finally { + if (!environmentRegistered) { + await closeOrcadManagedTunnel(environmentId).catch(() => undefined) + } + } + }) +} + +function assertMigrationFence( + userDataPath: string, + target: SshTarget | undefined, + environmentId: string, + name: string +): void { + const fence = target ? resolveOrcadMigrationFence(userDataPath, target) : null + if ( + fence?.state !== 'fenced' || + fence.cutover.destinationEnvironmentId !== environmentId || + fence.cutover.destinationName !== name + ) { + throw new Error('orcad_migration_fence_required') + } +} diff --git a/src/main/ssh/orcad-runtime-lifecycle.ts b/src/main/ssh/orcad-runtime-lifecycle.ts new file mode 100644 index 00000000000..0562ccce7c7 --- /dev/null +++ b/src/main/ssh/orcad-runtime-lifecycle.ts @@ -0,0 +1,8 @@ +export { createManagedOrcadEnvironment } from './orcad-runtime-deployment' +export { getManagedOrcadRuntimeStatus } from './orcad-runtime-status' +export { + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} from './orcad-runtime-maintenance' +export { cancelManagedOrcadStop, stopManagedOrcadEnvironment } from './orcad-runtime-decommission' diff --git a/src/main/ssh/orcad-runtime-maintenance.test.ts b/src/main/ssh/orcad-runtime-maintenance.test.ts new file mode 100644 index 00000000000..33506c397ca --- /dev/null +++ b/src/main/ssh/orcad-runtime-maintenance.test.ts @@ -0,0 +1,241 @@ +import { rmSync } from 'node:fs' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { recordManagedOrcadMigration } from '../../shared/runtime-environment-managed-orcad-store' +import { orcadMigrationCutoverFixture } from './orcad-migration-cutover-fixture' +import { writeOrcadMigrationSourceCutover } from './orcad-migration-cutover-journal' +import { + createManagedLifecycleHarness, + MANAGED_PREVIOUS_VERSION, + MANAGED_VERSION, + managedReadiness +} from './orcad-managed-lifecycle-test-fixture' + +const mocks = vi.hoisted(() => { + const state: { store: unknown } = { store: null } + return { + state, + resolveContext: vi.fn(), + census: vi.fn(), + deploy: vi.fn(), + rollback: vi.fn(), + recover: vi.fn(), + probe: vi.fn(), + buildHash: vi.fn(), + ensureTunnel: vi.fn(), + pendingMigration: vi.fn() + } +}) + +vi.mock('./ssh-target-registry', () => ({ + getSshConnectionManager: () => ({ connect: async () => ({}) }), + getSshTargetRegistryStore: () => mocks.state.store +})) +vi.mock('./orcad-remote-context', () => ({ resolveOrcadRemoteContext: mocks.resolveContext })) +vi.mock('./orcad-terminal-census-client', () => ({ collectManagedTerminalCensus: mocks.census })) +vi.mock('./orcad-remote-deploy', () => ({ deployOrcad: mocks.deploy })) +vi.mock('./orcad-remote-rollback', () => ({ rollbackOrcad: mocks.rollback })) +vi.mock('./orcad-activation-recovery', () => ({ recoverInterruptedOrcadActivation: mocks.recover })) +vi.mock('./orcad-active-readiness', () => ({ probeActiveOrcadReadiness: mocks.probe })) +vi.mock('./orcad-remote-build-hash', () => ({ readRemoteOrcadBuildHash: mocks.buildHash })) +vi.mock('./orcad-artifact-materializer', () => ({ + materializeOrcadArtifact: async () => '/local/orcad' +})) +vi.mock('./orcad-local-build-hash', () => ({ computeLocalOrcadBuildHash: () => 'local-hash' })) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensureTunnel })) +vi.mock('./orcad-managed-migration-status', () => ({ + findIncompleteManagedOrcadMigration: mocks.pendingMigration +})) +vi.mock('./orcad-activation-transaction-store', () => ({ + readOrcadActivationTransaction: async () => null +})) + +const { + getManagedOrcadRuntimeStatus, + recoverManagedOrcadEnvironment, + rollbackManagedOrcadEnvironment, + updateManagedOrcadEnvironment +} = await import('./orcad-runtime-lifecycle') + +const idle = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 7 } +let harness: ReturnType + +beforeEach(() => { + vi.resetAllMocks() + harness = createManagedLifecycleHarness() + mocks.state.store = harness.targetStore + mocks.resolveContext.mockImplementation(async () => harness.context()) + mocks.census.mockResolvedValue(idle) + mocks.probe.mockResolvedValue(managedReadiness()) + mocks.buildHash.mockResolvedValue('previous-hash') + mocks.recover.mockResolvedValue({ outcome: 'none' }) +}) + +afterEach(() => rmSync(harness.userDataPath, { recursive: true, force: true })) + +describe('updateManagedOrcadEnvironment', () => { + it('defers over live or unverifiable terminals and reports the deferral in status', async () => { + mocks.census.mockResolvedValue({ ...idle, liveSessions: null }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '0.3.0+new', + code: 'orcad_update_terminal_census_unavailable', + reason: 'The terminal daemon did not answer a session count.' + }) + const result = await updateManagedOrcadEnvironment(harness.userDataPath, { + selector: 'Managed' + }) + expect(result).toMatchObject({ outcome: 'deferred', forceable: false }) + expect(mocks.deploy.mock.calls[0]?.[0]).toMatchObject({ + census: { liveSessions: null }, + port: 6_768, + nodePath: '' + }) + const status = await getManagedOrcadRuntimeStatus(harness.userDataPath, 'Managed') + expect(status.deferredUpdate).toMatchObject({ + candidateVersion: '0.3.0+new', + code: 'orcad_update_terminal_census_unavailable' + }) + expect(status.terminals.liveSessions).toBeNull() + }) + + it('clears the deferral once an update goes through and keeps an unchanged pairing as is', async () => { + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-not-activated', + fullVersion: '0.3.0+new', + code: 'orcad_update_terminals_running', + reason: 'busy' + }) + await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + mocks.deploy.mockResolvedValueOnce({ + outcome: 'installed-and-activated', + fullVersion: '0.3.0+new' + }) + const result = await updateManagedOrcadEnvironment(harness.userDataPath, { + selector: 'Managed', + force: true + }) + expect(result).toMatchObject({ outcome: 'updated', activeVersion: '0.3.0+new' }) + expect(mocks.probe).toHaveBeenCalledWith(expect.anything(), { + buildHash: 'local-hash', + fullVersion: '0.3.0+new' + }) + const [environment] = listEnvironments(harness.userDataPath) + expect(environment?.pairingRevision).toBe(harness.environment.pairingRevision) + expect(environment?.orcadDeployment).toEqual(harness.environment.orcadDeployment) + const status = await getManagedOrcadRuntimeStatus(harness.userDataPath, 'Managed') + expect(status.deferredUpdate).toBeNull() + }) + + it('re-pairs through the same tunnel, keeping the deployment link, when the offer changed', async () => { + mocks.deploy.mockResolvedValueOnce({ outcome: 'installed-and-activated', fullVersion: 'v3' }) + mocks.probe.mockResolvedValueOnce(managedReadiness('rotated-token')) + await updateManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + const [environment] = listEnvironments(harness.userDataPath) + expect(environment?.endpoints[0]?.deviceToken).toBe('rotated-token') + expect(environment?.orcadDeployment).toEqual(harness.environment.orcadDeployment) + }) +}) + +describe('rollbackManagedOrcadEnvironment', () => { + it.each([ + [{ ...idle, liveSessions: 2 }, 'orcad_rollback_terminals_running'], + [{ ...idle, liveSessions: null }, 'orcad_rollback_census_unavailable'] + ])('refuses while terminals run or cannot be counted: %j', async (census, code) => { + mocks.census.mockResolvedValue(census) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code }) + expect(mocks.rollback).not.toHaveBeenCalled() + }) + + it('refuses with no previous version without contacting the slot', async () => { + mocks.resolveContext.mockImplementation(async () => harness.context({ previous: null })) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code: 'orcad_rollback_no_target' }) + expect(mocks.census).not.toHaveBeenCalled() + }) + + it('refuses while a migration into the server is unfinished', async () => { + mocks.pendingMigration.mockReturnValueOnce({ migrationId: 'm-1', phase: 'destination-staged' }) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code: 'orcad_rollback_migration_in_progress' }) + expect(mocks.rollback).not.toHaveBeenCalled() + }) + + it('refuses a rollback to a snapshot older than the migrated catalog, not a newer one', async () => { + // The harness's active version activated 2026-01-01. + recordManagedOrcadMigration(harness.userDataPath, 'environment-1', '2026-02-01T00:00:00.000Z') + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code: 'orcad_rollback_crosses_migration' }) + expect(mocks.rollback).not.toHaveBeenCalled() + mocks.resolveContext.mockImplementation(async () => + harness.context({ activatedAt: '2026-03-01T00:00:00.000Z' }) + ) + mocks.rollback.mockResolvedValueOnce({ outcome: 'refused', code: 'x', reason: 'y' }) + await rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + expect(mocks.rollback).toHaveBeenCalledOnce() + }) + + it('refuses a rollback across a retained delta move that predates the mark', async () => { + // A delta finished by an earlier build left no mark; its retained journal still counts. + writeOrcadMigrationSourceCutover(harness.userDataPath, { + ...orcadMigrationCutoverFixture('delta-1', 'ssh-1', { environmentId: 'environment-1' }), + startedAt: '2026-02-01T00:00:00.000Z', + phase: 'destination-committed', + sourceRetainedAt: '2026-02-01T00:00:00.000Z' + }) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', code: 'orcad_rollback_crosses_migration' }) + expect(mocks.rollback).not.toHaveBeenCalled() + }) + + it('rolls back against the installed bytes of the previous slot', async () => { + mocks.rollback.mockResolvedValueOnce({ + outcome: 'rolled-back', + target: MANAGED_PREVIOUS_VERSION, + discarded: [MANAGED_VERSION], + verdict: { decision: 'activate', coverage: 'pty-spawn', warnings: [] } + }) + await expect( + rollbackManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'rolled-back', activeVersion: MANAGED_PREVIOUS_VERSION }) + expect(mocks.rollback.mock.calls[0]?.[0]).toMatchObject({ targetBuildHash: 'previous-hash' }) + expect(mocks.probe).toHaveBeenCalledWith(expect.anything(), { + buildHash: 'previous-hash', + fullVersion: MANAGED_PREVIOUS_VERSION + }) + }) +}) + +describe('recoverManagedOrcadEnvironment', () => { + it('passes a refusal through and leaves the server linked', async () => { + mocks.recover.mockResolvedValueOnce({ + outcome: 'refused', + verdict: 'unverifiable', + code: 'orcad_recovery_unverifiable', + reason: 'host fenced' + }) + await expect( + recoverManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expect(listEnvironments(harness.userDataPath)).toHaveLength(1) + }) + + it('re-ensures the tunnel once a serving slot is restored', async () => { + mocks.recover.mockResolvedValueOnce({ + outcome: 'recovered', + resolution: 'restored-incumbent', + activeVersion: MANAGED_VERSION, + readiness: managedReadiness() + }) + await expect( + recoverManagedOrcadEnvironment(harness.userDataPath, { selector: 'Managed' }) + ).resolves.toMatchObject({ outcome: 'recovered', activeVersion: MANAGED_VERSION }) + expect(mocks.ensureTunnel).toHaveBeenCalledWith(harness.userDataPath, 'environment-1') + }) +}) diff --git a/src/main/ssh/orcad-runtime-maintenance.ts b/src/main/ssh/orcad-runtime-maintenance.ts new file mode 100644 index 00000000000..a300d409d04 --- /dev/null +++ b/src/main/ssh/orcad-runtime-maintenance.ts @@ -0,0 +1,284 @@ +/** + * Updating, rolling back and recovering a managed orcad, on T6-2's deploy, rollback and recovery. + * Every step reads the terminal census through the server's tunnel first; an unanswered census + * is unverifiable, never zero, so an update over live or uncounted terminals defers (D7). + */ +import { getAppEnvironment } from '../../shared/app-environment' +import { refreshManagedOrcadPairing } from '../../shared/runtime-environment-managed-orcad-store' +import { assertRuntimeEnvironmentNotReconciling } from '../../shared/runtime-environment-reconciliation-record' +import { + redactRuntimeEnvironment, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { + OrcadManagedDeployResult, + OrcadManagedRecoveryResult, + OrcadManagedRollbackResult +} from '../../shared/orcad-managed-runtime' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import type { ServeReadiness } from '../server/serve-readiness' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { probeActiveOrcadReadiness } from './orcad-active-readiness' +import { materializeOrcadArtifact } from './orcad-artifact-materializer' +import { CURRENT_ORCAD_DAEMON_PROTOCOL } from './orcad-daemon-protocol-crossing' +import { ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { + clearManagedOrcadUpdateDeferral, + recordManagedOrcadUpdateDeferral +} from './orcad-managed-update-deferrals' +import { + isForceableOrcadDeferral, + managedOrcadInstallDir, + managedOrcadSlot, + probeManagedOrcadReadiness, + requireManagedOrcadEnvironment, + resolveLinkedOrcadContext +} from './orcad-managed-runtime-context' +import type { OrcadRemoteContext } from './orcad-remote-context' +import { readRemoteOrcadBuildHash } from './orcad-remote-build-hash' +import { deployOrcad } from './orcad-remote-deploy' +import { pruneManagedOrcadVersions } from './orcad-managed-version-gc' +import { rollbackOrcad } from './orcad-remote-rollback' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' +import { findIncompleteManagedOrcadMigration } from './orcad-managed-migration-status' +import { latestOrcadMigrationInto } from './orcad-migration-rollback-mark' +import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' + +type LifecycleArgs = { selector: string; signal?: AbortSignal } + +export function withManagedOrcadLifecycle( + userDataPath: string, + selector: string, + run: (managed: ReturnType) => Promise +): Promise { + const { environment, deployment } = requireManagedOrcadEnvironment(userDataPath, selector) + return runTargetLifecycle(deployment.sshTargetId, async () => { + // Re-read under the queue: a reconciliation or stop may have won the race for it. + const current = requireManagedOrcadEnvironment(userDataPath, environment.id) + assertRuntimeEnvironmentNotReconciling(current.environment) + return run(current) + }) +} + +function refreshPairing( + userDataPath: string, + environment: KnownRuntimeEnvironment, + readiness: ServeReadiness, + localPort: number +): KnownRuntimeEnvironment { + return refreshManagedOrcadPairing( + userDataPath, + environment.id, + tunneledOrcadPairingCode(readiness, localPort) + ) +} + +export function updateManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs & { force?: boolean } +): Promise { + return withManagedOrcadLifecycle(userDataPath, args.selector, async (managed) => + runManagedOrcadUpdate( + userDataPath, + managed, + await resolveLinkedOrcadContext(managed.environment, managed.deployment, args.signal), + args + ) + ) +} + +/** The Managed servers update, run inside the target's lifecycle queue with a resolved context. */ +export async function runManagedOrcadUpdate( + userDataPath: string, + { environment, deployment }: ReturnType, + context: OrcadRemoteContext, + args: { force?: boolean; signal?: AbortSignal; localOrcadDir?: string } +): Promise { + // Why release: finished automation shells would otherwise defer every update on a host with + // schedules; the update restarts the server anyway. + const census = await collectManagedTerminalCensus( + userDataPath, + environment, + context.activationRecord, + undefined, + { releaseFinishedAutomationTerminals: true } + ) + const localOrcadDir = + args.localOrcadDir ?? + (await materializeOrcadArtifact(context.serverTarget, { signal: args.signal })) + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + const result = await deployOrcad({ + ...slot, + localOrcadDir, + target: context.serverTarget, + census, + force: args.force, + appVersion: getAppEnvironment().getVersion() + }) + if (result.outcome === 'installed-not-activated') { + const deferral = { + outcome: 'deferred' as const, + candidateVersion: result.fullVersion, + code: result.code, + reason: result.reason, + forceable: isForceableOrcadDeferral(result.code) + } + recordManagedOrcadUpdateDeferral(environment.id, deferral) + return deferral + } + clearManagedOrcadUpdateDeferral(environment.id) + const readiness = await probeManagedOrcadReadiness( + context, + localOrcadDir, + result.fullVersion, + args.signal + ) + await pruneManagedOrcadVersions({ + slot, + serverTarget: context.serverTarget, + activeVersion: result.fullVersion, + readiness + }) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: result.outcome === 'already-active' ? 'already-current' : 'updated', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.fullVersion + } +} + +export function rollbackManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs +): Promise { + return withManagedOrcadLifecycle( + userDataPath, + args.selector, + async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const record = context.activationRecord + const target = record.previous + if (!target) { + return { + outcome: 'refused', + code: 'orcad_rollback_no_target', + reason: 'This server has no previous version to roll back to.' + } + } + const crossing = migrationRollbackRefusal(userDataPath, environment, record.activatedAt) + if (crossing) { + return crossing + } + const census = await collectManagedTerminalCensus( + userDataPath, + environment, + record, + undefined, + { + releaseFinishedAutomationTerminals: true + } + ) + // Why idle only: this client cannot read the older build's daemon protocol, so it cannot show + // that build would reach terminals that are still running. + if (census.liveSessions !== 0) { + return { + outcome: 'refused', + code: + census.liveSessions === null + ? 'orcad_rollback_census_unavailable' + : 'orcad_rollback_terminals_running', + reason: + census.liveSessions === null + ? 'The server did not answer how many terminals it runs. Retry when it answers.' + : 'Close the terminals running on this server before rolling it back.' + } + } + const slot = managedOrcadSlot(context, deployment.remotePort, args.signal) + const targetDir = managedOrcadInstallDir(context, target) + const targetBuildHash = await readRemoteOrcadBuildHash(slot, targetDir) + const result = await rollbackOrcad({ + ...slot, + record, + census, + targetBuildHash, + targetDaemonProtocol: CURRENT_ORCAD_DAEMON_PROTOCOL + }) + if (result.outcome !== 'rolled-back') { + return result + } + const readiness = await probeActiveOrcadReadiness( + { ...slot, remoteInstallDir: targetDir }, + { buildHash: targetBuildHash, fullVersion: result.target } + ) + const updated = refreshPairing(userDataPath, environment, readiness, deployment.localPort) + return { + outcome: 'rolled-back', + environment: redactRuntimeEnvironment(updated), + activeVersion: result.target, + discarded: result.discarded + } + } + ) +} + +/** Finishes or undoes an interrupted activation, rollback or decommission on the host. */ +export function recoverManagedOrcadEnvironment( + userDataPath: string, + args: LifecycleArgs & { acceptChangedState?: boolean } +): Promise { + return withManagedOrcadLifecycle( + userDataPath, + args.selector, + async ({ environment, deployment }) => { + const context = await resolveLinkedOrcadContext(environment, deployment, args.signal) + const result = await recoverInterruptedOrcadActivation({ + ...managedOrcadSlot(context, deployment.remotePort, args.signal), + acceptChangedState: args.acceptChangedState === true + }) + if (result.outcome !== 'recovered') { + return result + } + const updated = result.readiness + ? refreshPairing(userDataPath, environment, result.readiness, deployment.localPort) + : environment + if (result.activeVersion) { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + } + return { + outcome: 'recovered', + resolution: result.resolution, + activeVersion: result.activeVersion, + environment: redactRuntimeEnvironment(updated) + } + } + ) +} + +/** + * A rollback restores the snapshot taken when the current version activated. If a migration + * began after that, the snapshot predates the imported catalog and restoring it would drop it. + */ +function migrationRollbackRefusal( + userDataPath: string, + environment: KnownRuntimeEnvironment, + activatedAt: string | null +): OrcadManagedRollbackResult | null { + if (findIncompleteManagedOrcadMigration(userDataPath, environment.id)) { + return { + outcome: 'refused', + code: 'orcad_rollback_migration_in_progress', + reason: 'A migration into this server is still running. Finish it before rolling back.' + } + } + const migratedAt = latestOrcadMigrationInto(userDataPath, environment) + if (migratedAt && (activatedAt === null || Date.parse(activatedAt) < Date.parse(migratedAt))) { + return { + outcome: 'refused', + code: 'orcad_rollback_crosses_migration', + reason: + "The previous version's state predates the projects migrated onto this server; " + + 'rolling back would lose them. Deploy forward instead.' + } + } + return null +} diff --git a/src/main/ssh/orcad-runtime-status.ts b/src/main/ssh/orcad-runtime-status.ts new file mode 100644 index 00000000000..191a2e569be --- /dev/null +++ b/src/main/ssh/orcad-runtime-status.ts @@ -0,0 +1,80 @@ +import type { OrcadManagedRuntimeStatus } from '../../shared/orcad-managed-runtime' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import type { OrcadActivationTransaction } from './orcad-activation-transaction' +import { readOrcadActivationTransaction } from './orcad-activation-transaction-store' +import { + requireManagedOrcadEnvironment, + resolveLinkedOrcadContext +} from './orcad-managed-runtime-context' +import { currentManagedOrcadUpdateDeferral } from './orcad-managed-update-deferrals' +import { findIncompleteManagedOrcadMigration } from './orcad-managed-migration-status' +import { collectManagedTerminalCensus } from './orcad-terminal-census-client' + +/** Read-only: what the host's activation record and journal say, without repairing either. */ +export async function getManagedOrcadRuntimeStatus( + userDataPath: string, + selector: string, + signal?: AbortSignal +): Promise { + const { environment, deployment } = requireManagedOrcadEnvironment(userDataPath, selector) + return runTargetLifecycle(deployment.sshTargetId, async () => { + const context = await resolveLinkedOrcadContext(environment, deployment, signal) + const record = context.activationRecord + const transaction = await readOrcadActivationTransaction({ + conn: context.connection, + host: context.host, + remoteHome: context.remoteHome, + signal + }) + return { + environmentId: environment.id, + sshTargetId: context.target.id, + activeVersion: record.active, + previousVersion: record.previous, + activatedAt: record.activatedAt, + rollbackAvailable: Boolean(record.previous && record.snapshot), + recovery: transaction ? managedRecoveryStatus(transaction) : null, + terminals: await collectManagedTerminalCensus(userDataPath, environment, record), + migration: migrationStatus(userDataPath, environment.id), + deferredUpdate: currentManagedOrcadUpdateDeferral(environment.id, record.active) + } + }) +} + +function managedRecoveryStatus( + transaction: OrcadActivationTransaction +): NonNullable { + switch (transaction.operation) { + case 'activate': + return { + operation: transaction.operation, + phase: transaction.phase, + version: transaction.candidateVersion, + startedAt: transaction.startedAt + } + case 'rollback': + return { + operation: transaction.operation, + phase: transaction.phase, + version: transaction.targetVersion, + startedAt: transaction.startedAt + } + case 'decommission': + return { + operation: transaction.operation, + phase: transaction.phase, + version: transaction.activeVersion, + startedAt: transaction.startedAt + } + } +} + +function migrationStatus( + userDataPath: string, + environmentId: string +): OrcadManagedRuntimeStatus['migration'] { + const migration = findIncompleteManagedOrcadMigration(userDataPath, environmentId) + return migration + ? { migrationId: migration.migrationId, phase: migration.phase, startedAt: migration.startedAt } + : null +} diff --git a/src/main/ssh/orcad-runtime-target.test.ts b/src/main/ssh/orcad-runtime-target.test.ts new file mode 100644 index 00000000000..242a7023aa1 --- /dev/null +++ b/src/main/ssh/orcad-runtime-target.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resolveOrcadRuntimeTarget } from './orcad-runtime-target' +import { SshConnection } from './ssh-connection' +import { createCallbacks, createTarget } from './ssh-connection-test-fixtures' +import { execCommand } from './ssh-relay-deploy-helpers' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn() })) +beforeEach(() => vi.mocked(execCommand).mockReset()) + +function resolveOn(platform: Parameters[0], ldd: string) { + vi.mocked(execCommand).mockResolvedValueOnce(ldd) + return resolveOrcadRuntimeTarget({ + conn: new SshConnection(createTarget(), createCallbacks()), + host: getRemoteHostPlatform(platform) + }) +} + +describe('managed orcad runtime target', () => { + it.each([ + ['CentOS 7 (glibc 2.17)', 'ldd (GNU libc) 2.17', 'linux-x64-glibc217'], + ['glibc 2.27, still below the default floor', 'ldd (GNU libc) 2.27', 'linux-x64-glibc217'], + ['Debian 10 (glibc 2.28)', 'ldd (Debian GLIBC 2.28-10+deb10u2) 2.28', 'linux-x64-glibc'], + ['Ubuntu 22.04 (glibc 2.35)', 'ldd (Ubuntu GLIBC 2.35-0ubuntu3.8) 2.35', 'linux-x64-glibc'], + ['Alpine (musl)', 'musl libc (x86_64)\nVersion 1.2.5', 'linux-x64-musl'] + ])('deploys %s on the runtime the relay ladder would pick', async (_host, ldd, target) => { + await expect(resolveOn('linux-x64', ldd)).resolves.toBe(target) + }) + + it('keeps the host target when the glibc version is unreadable; the self-test decides', async () => { + await expect(resolveOn('linux-x64', 'ldd (GNU libc) unknown')).resolves.toBe('linux-x64-glibc') + }) + + it('refuses as unsupported where no runtime serves the glibc, so the connect keeps the relay', async () => { + await expect(resolveOn('linux-arm64', 'ldd (GNU libc) 2.17')).rejects.toMatchObject({ + name: 'OrcadHostUnsupportedError', + message: expect.stringContaining('linux-arm64-glibc with glibc 2.17') + }) + await expect(resolveOn('linux-x64', 'ldd (GNU libc) 2.12')).rejects.toMatchObject({ + name: 'OrcadHostUnsupportedError' + }) + }) + + it('needs no libc probe off Linux', async () => { + await expect( + resolveOrcadRuntimeTarget({ + conn: new SshConnection(createTarget(), createCallbacks()), + host: getRemoteHostPlatform('darwin-arm64') + }) + ).resolves.toBe('darwin-arm64') + expect(execCommand).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-runtime-target.ts b/src/main/ssh/orcad-runtime-target.ts new file mode 100644 index 00000000000..509725d7517 --- /dev/null +++ b/src/main/ssh/orcad-runtime-target.ts @@ -0,0 +1,27 @@ +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import { resolveOrcadDeploymentTargetFacts } from './orcad-deployment-target' +import { OrcadHostUnsupportedError } from './orcad-host-unavailable' +import type { SshConnection } from './ssh-connection' +import { pinnedRuntimeTargetForHost } from './ssh-relay-runtime-ladder' +import type { RemoteHostPlatform } from './ssh-remote-platform' + +/** + * The runtime target managed orcad deploys on a host, picked by glibc exactly as the relay ladder + * picks rung A or B: the host's own target, or its compat runtime below the default's glibc floor. + */ +export async function resolveOrcadRuntimeTarget(options: { + conn: SshConnection + host: RemoteHostPlatform + signal?: AbortSignal + exec?: (command: string) => Promise +}): Promise { + const facts = await resolveOrcadDeploymentTargetFacts(options) + const target = pinnedRuntimeTargetForHost(facts) + if (!target) { + const glibc = facts.glibc ? `${facts.glibc.major}.${facts.glibc.minor}` : 'unknown' + throw new OrcadHostUnsupportedError( + `No Orca runtime supports ${facts.target} with glibc ${glibc}.` + ) + } + return target +} diff --git a/src/main/ssh/orcad-snapshot-capture-command.ts b/src/main/ssh/orcad-snapshot-capture-command.ts new file mode 100644 index 00000000000..35567fd5e4c --- /dev/null +++ b/src/main/ssh/orcad-snapshot-capture-command.ts @@ -0,0 +1,4 @@ +/** A fake host's test for the capture's own tar flag: a random fence token can contain `-cf`. */ +export function isSnapshotCaptureCommand(command: string): boolean { + return /\btar -C \S+ -cf /u.test(command) +} diff --git a/src/main/ssh/orcad-ssh-provisioning.test.ts b/src/main/ssh/orcad-ssh-provisioning.test.ts new file mode 100644 index 00000000000..4640f0b93d5 --- /dev/null +++ b/src/main/ssh/orcad-ssh-provisioning.test.ts @@ -0,0 +1,245 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SshTarget } from '../../shared/ssh-types' +import { normalizeSshTarget } from '../persistence/leasing-ssh-ptys/ssh-normalization' + +const mocks = vi.hoisted(() => { + const targets: SshTarget[] = [] + const deployedTargets: string[] = [] + const targetStore: Record = {} + return { + targets, + deployedTargets, + update: vi.fn(), + deploy: vi.fn(), + flush: vi.fn(), + add: vi.fn(), + rotate: vi.fn(), + targetStore + } +}) + +vi.mock('./orcad-managed-runtime-context', () => ({ + requireManagedOrcadTargetStore: () => mocks.targetStore +})) +vi.mock('./orcad-runtime-deployment', () => ({ createManagedOrcadEnvironment: mocks.deploy })) +vi.mock('./ssh-provider-authority', () => ({ rotateSshProviderAuthority: mocks.rotate })) +vi.mock('../../shared/runtime-environment-store', () => ({ + listEnvironments: () => + mocks.deployedTargets.map((sshTargetId) => ({ orcadDeployment: { sshTargetId } })) +})) + +import { + createOrcadSshHost, + listPendingOrcadSshProvisioning, + resumeOrcadSshHost +} from './orcad-ssh-provisioning' + +const request = { + requestId: 'request-1', + name: 'Build host', + target: { label: 'Build host', host: 'builder', port: 22, username: 'dev' } +} +const deployed = { + outcome: 'created', + environment: { id: 'environment-1' }, + activeVersion: 'node-1' +} + +beforeEach(() => { + vi.clearAllMocks() + mocks.targets.length = 0 + mocks.deployedTargets.length = 0 + mocks.flush.mockReset().mockResolvedValue(undefined) + mocks.deploy.mockReset().mockResolvedValue(deployed) + mocks.add.mockImplementation((input) => { + const target = normalizeSshTarget({ + ...input, + id: `ssh-${mocks.targets.length + 1}`, + generation: 1 + }) + mocks.targets.push(target) + return target + }) + mocks.update.mockImplementation((id: string, updates: Partial) => { + const index = mocks.targets.findIndex((target) => target.id === id) + mocks.targets[index] = { ...mocks.targets[index]!, ...updates } + return mocks.targets[index] + }) + mocks.targetStore = { + addTarget: mocks.add, + updateTarget: mocks.update, + lastRepoReadoptions: [], + getOrcadRuntimeClaims: () => ({ listTargets: () => mocks.targets, flush: mocks.flush }) + } +}) + +describe('managed SSH host provisioning', () => { + it('persists intent before running the existing migration/deployment preflight', async () => { + mocks.deploy.mockImplementation(async () => { + expect(mocks.flush).toHaveBeenCalledOnce() + expect(mocks.targets[0]?.orcadProvisioning).toEqual({ + requestId: 'request-1', + name: 'Build host' + }) + return deployed + }) + await expect(createOrcadSshHost('/profile', request)).resolves.toMatchObject({ + requestId: 'request-1', + name: 'Build host', + sshTargetId: 'ssh-1', + result: deployed + }) + expect(mocks.deploy).toHaveBeenCalledWith('/profile', { + name: 'Build host', + sshTargetId: 'ssh-1' + }) + }) + + it('does not contact a host if durable intent publication fails', async () => { + mocks.flush.mockRejectedValueOnce(new Error('disk full')) + await expect(createOrcadSshHost('/profile', request)).rejects.toThrow('disk full') + expect(mocks.deploy).not.toHaveBeenCalled() + await resumeOrcadSshHost('/profile', 'request-1') + expect(mocks.add).toHaveBeenCalledOnce() + }) + + it('retains the same request across preflight refusal and a reconstructed target store', async () => { + mocks.deploy.mockRejectedValueOnce(new Error('live-or-unverifiable terminal leases')) + await expect(createOrcadSshHost('/profile', request)).resolves.toMatchObject({ + result: { outcome: 'pending', reason: 'live-or-unverifiable terminal leases' } + }) + mocks.targets.splice(0, 1, JSON.parse(JSON.stringify(mocks.targets[0]))) + expect(listPendingOrcadSshProvisioning('/profile')).toEqual([ + { requestId: 'request-1', name: 'Build host', sshTargetId: 'ssh-1' } + ]) + await expect(resumeOrcadSshHost('/profile', 'request-1')).resolves.toMatchObject({ + result: deployed + }) + expect(mocks.add).toHaveBeenCalledOnce() + expect(mocks.deploy).toHaveBeenCalledTimes(2) + }) + + it('restores a host with re-adopted projects as direct SSH instead of hiding them', async () => { + const readoptions = [{ oldTargetId: 'old', newTargetId: 'ssh-1', repoIds: ['repo-1'] }] + mocks.targetStore.lastRepoReadoptions = readoptions + const result = await createOrcadSshHost('/profile', request) + expect(result.repoReadoptions).toEqual(readoptions) + expect(result.result).toMatchObject({ outcome: 'pending' }) + expect(mocks.rotate).toHaveBeenCalledWith('old') + expect(mocks.rotate).toHaveBeenCalledWith('ssh-1') + expect(mocks.targets[0]?.orcadProvisioning).toBeUndefined() + expect(listPendingOrcadSshProvisioning('/profile')).toEqual([]) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('returns activation deferral without force, deletion, or another target', async () => { + const deferred = { + outcome: 'deferred', + reason: 'unverifiable', + code: 'blocked', + candidateVersion: 'node-1' + } + mocks.deploy.mockResolvedValueOnce(deferred) + await expect(createOrcadSshHost('/profile', request)).resolves.toMatchObject({ + result: deferred + }) + expect(listPendingOrcadSshProvisioning('/profile')).toHaveLength(1) + expect(mocks.targets).toHaveLength(1) + }) + + it('serializes repeated requests and preserves normalization on exact retries', async () => { + const normalizedRequest = { + ...request, + target: { ...request.target, relayGracePeriodSeconds: 10800 } + } + await Promise.all([ + createOrcadSshHost('/profile', normalizedRequest), + createOrcadSshHost('/profile', normalizedRequest) + ]) + expect(mocks.add).toHaveBeenCalledOnce() + expect(mocks.targets).toHaveLength(1) + }) + + it('rejects request identity reuse for different host data or name', async () => { + await createOrcadSshHost('/profile', request) + await expect(createOrcadSshHost('/profile', { ...request, name: 'Other' })).rejects.toThrow( + 'already belongs' + ) + await expect( + createOrcadSshHost('/profile', { + ...request, + target: { ...request.target, identityFile: 'other' } + }) + ).rejects.toThrow('already belongs') + expect(mocks.deploy).toHaveBeenCalledOnce() + }) + + it('does not create a duplicate raw host when another request registered the endpoint', async () => { + await createOrcadSshHost('/profile', request) + await expect( + createOrcadSshHost('/profile', { ...request, requestId: 'request-2' }) + ).rejects.toThrow('already registered') + expect(mocks.add).toHaveBeenCalledOnce() + }) + + it('does not duplicate an occupied config alias or endpoint under a different label', async () => { + await createOrcadSshHost('/profile', request) + await expect( + createOrcadSshHost('/profile', { + ...request, + requestId: 'request-2', + target: { ...request.target, configHost: 'BUILDER', host: 'resolved-address' } + }) + ).rejects.toThrow('already registered') + await expect( + createOrcadSshHost('/profile', { + ...request, + requestId: 'request-3', + target: { ...request.target, label: 'Different', configHost: 'other-alias' } + }) + ).rejects.toThrow('already registered') + }) + + it.each([ + { configHost: 3 }, + { identityFile: false }, + { gssapiAuthentication: 'yes' }, + { systemSshConnectionReuse: 1 }, + { portForwards: [{ localPort: -1 }] } + ])('rejects malformed optional connection fields before durable registration: %j', (invalid) => { + expect(() => + createOrcadSshHost('/profile', { + ...request, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: deliberately malformed renderer input. + target: { ...request.target, ...invalid } as never + }) + ).toThrow() + expect(mocks.add).not.toHaveBeenCalled() + }) + + it('omits completed requests from pending discovery but retains their retry identity', async () => { + await createOrcadSshHost('/profile', request) + mocks.deployedTargets.push('ssh-1') + expect(listPendingOrcadSshProvisioning('/profile')).toEqual([]) + await resumeOrcadSshHost('/profile', 'request-1') + expect(mocks.add).toHaveBeenCalledOnce() + expect(mocks.deploy).toHaveBeenCalledTimes(2) + }) + + it('does not provision imported config entries without explicit intent', () => { + mocks.targets.push({ ...request.target, id: 'imported', source: 'ssh-config' }) + expect(listPendingOrcadSshProvisioning('/profile')).toEqual([]) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('rejects invalid input and unknown retry IDs without writing a target', async () => { + expect(() => createOrcadSshHost('/profile', { ...request, requestId: '' })).toThrow( + 'request id' + ) + expect(() => + createOrcadSshHost('/profile', { ...request, target: { ...request.target, port: 0 } }) + ).toThrow('port') + await expect(resumeOrcadSshHost('/profile', 'missing')).rejects.toThrow('not found') + expect(mocks.add).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ssh/orcad-ssh-provisioning.ts b/src/main/ssh/orcad-ssh-provisioning.ts new file mode 100644 index 00000000000..8b904e7f272 --- /dev/null +++ b/src/main/ssh/orcad-ssh-provisioning.ts @@ -0,0 +1,200 @@ +import { z } from 'zod' +import type { + OrcadSshPendingProvisioning, + OrcadSshProvisioningRequest, + OrcadSshProvisioningResult +} from '../../shared/orcad-ssh-provisioning' +import type { SshRepoReadoption, SshTarget, SshTargetCreateInput } from '../../shared/ssh-types' +import { EphemeralVmRecipeSshTargetSchema } from '../../shared/ephemeral-vm-recipes' +import { listEnvironments } from '../../shared/runtime-environment-store' +import { normalizeSshConfigAlias } from '../../shared/ssh-config-alias' +import { runTargetLifecycle } from '../ipc/ssh-target-lifecycle-queue' +import { normalizeSshTarget } from '../persistence/leasing-ssh-ptys/ssh-normalization' +import { requireManagedOrcadTargetStore } from './orcad-managed-runtime-context' +import { createManagedOrcadEnvironment } from './orcad-runtime-deployment' +import { rotateSshProviderAuthority } from './ssh-provider-authority' + +// Why no port forwards: a managed server is only created on an empty host. +const provisioningTargetSchema = EphemeralVmRecipeSshTargetSchema.omit({ + portForwards: true +}).extend({ + gssapiAuthentication: z.boolean().optional(), + systemSshConnectionReuse: z.boolean().optional(), + source: z.enum(['manual', 'ssh-config']).optional(), + lastRequiredPassphrase: z.boolean().optional() +}) + +/** Requests whose server is not registered yet; a completed one keeps its intent for idempotent retries. */ +export function listPendingOrcadSshProvisioning( + userDataPath: string +): OrcadSshPendingProvisioning[] { + const deployed = new Set( + listEnvironments(userDataPath).flatMap((environment) => + environment.orcadDeployment ? [environment.orcadDeployment.sshTargetId] : [] + ) + ) + return requireManagedOrcadTargetStore() + .getOrcadRuntimeClaims() + .listTargets() + .flatMap((target) => + target.orcadProvisioning && !deployed.has(target.id) + ? [{ ...target.orcadProvisioning, sshTargetId: target.id }] + : [] + ) +} + +export function createOrcadSshHost( + userDataPath: string, + request: OrcadSshProvisioningRequest +): Promise { + const requestId = requireRequestId(request?.requestId) + const name = requireText(request?.name, 'Server name') + const targetInput = parseTarget(request?.target) + return runTargetLifecycle(`orcad-provision:${userDataPath}:${requestId}`, async () => { + const targetStore = requireManagedOrcadTargetStore() + const targets = targetStore.getOrcadRuntimeClaims().listTargets() + const existing = targets.find((entry) => entry.orcadProvisioning?.requestId === requestId) + if (existing) { + if (existing.orcadProvisioning?.name !== name || !matchesRequest(existing, targetInput)) { + throw new Error('This provisioning request already belongs to another host or server name.') + } + return provision(userDataPath, existing, []) + } + if (targets.some((entry) => sameEndpoint(entry, targetInput))) { + throw new Error('That SSH host is already registered. Use its existing server or SSH entry.') + } + const target = targetStore.addTarget({ + ...targetInput, + source: 'manual', + orcadProvisioning: { requestId, name } + }) + const repoReadoptions = [...targetStore.lastRepoReadoptions] + targetStore.lastRepoReadoptions = [] + for (const targetId of new Set( + repoReadoptions.flatMap(({ oldTargetId, newTargetId }) => [oldTargetId, newTargetId]) + )) { + rotateSshProviderAuthority(targetId) + } + if (repoReadoptions.length > 0) { + // Why visible: re-adopted projects make the host non-empty; hiding it would hide them too. + targetStore.updateTarget(target.id, { orcadProvisioning: undefined }) + return { + requestId, + name, + sshTargetId: target.id, + repoReadoptions, + result: { + outcome: 'pending', + reason: + 'This host already has Orca projects, so it was restored as a direct SSH host. ' + + 'A managed server can only be created on an empty host.' + } + } + } + return provision(userDataPath, target, repoReadoptions) + }) +} + +export function resumeOrcadSshHost( + userDataPath: string, + requestIdInput: string +): Promise { + const requestId = requireRequestId(requestIdInput) + return runTargetLifecycle(`orcad-provision:${userDataPath}:${requestId}`, async () => { + const target = requireManagedOrcadTargetStore() + .getOrcadRuntimeClaims() + .listTargets() + .find((entry) => entry.orcadProvisioning?.requestId === requestId) + if (!target) { + throw new Error('The managed SSH provisioning request was not found.') + } + return provision(userDataPath, target, []) + }) +} + +async function provision( + userDataPath: string, + target: SshTarget, + repoReadoptions: SshRepoReadoption[] +): Promise { + const intent = target.orcadProvisioning + if (!intent) { + throw new Error('The managed SSH provisioning request was not found.') + } + // Why first: the intent must survive a crash before the host is contacted. + await requireManagedOrcadTargetStore().getOrcadRuntimeClaims().flush() + const base = { ...intent, sshTargetId: target.id, repoReadoptions } + try { + return { + ...base, + result: await createManagedOrcadEnvironment(userDataPath, { + name: intent.name, + sshTargetId: target.id + }) + } + } catch (error) { + return { + ...base, + result: { + outcome: 'pending', + reason: error instanceof Error ? error.message : 'Managed SSH provisioning failed.' + } + } + } +} + +function requireText(value: unknown, label: string): string { + if (typeof value !== 'string' || !value.trim() || value.length > 1_024) { + throw new Error(`${label} is required and must not exceed 1024 characters.`) + } + return value.trim() +} + +function requireRequestId(value: unknown): string { + if (typeof value !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/.test(value)) { + throw new Error('A stable managed SSH provisioning request id is required.') + } + return value +} + +function parseTarget(value: unknown): SshTargetCreateInput { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('A new unowned SSH host is required.') + } + const { + id: _id, + generation: _generation, + orcadProvisioning: _intent, + owner, + ...raw + }: Record = { ...value } + if (owner !== undefined) { + throw new Error('A new unowned SSH host is required.') + } + const target = provisioningTargetSchema.parse(raw) + return { + ...target, + label: requireText(target.label, 'SSH host label'), + host: requireText(target.host, 'SSH host'), + username: target.username.trim(), + configHost: target.configHost?.trim() || target.host.trim() + } +} + +function sameEndpoint(left: SshTarget, right: SshTargetCreateInput): boolean { + const alias = normalizeSshConfigAlias(right.configHost ?? right.host) + if ( + alias && + [left.configHost, left.label].some((value) => normalizeSshConfigAlias(value) === alias) + ) { + return true + } + return left.host === right.host && left.port === right.port && left.username === right.username +} + +function matchesRequest(target: SshTarget, input: SshTargetCreateInput): boolean { + const stored: Record = { ...target } + return Object.entries(normalizeSshTarget({ ...input, id: target.id })).every( + ([key, value]) => key === 'source' || JSON.stringify(stored[key]) === JSON.stringify(value) + ) +} diff --git a/src/main/ssh/orcad-state-mutation-exec.test.ts b/src/main/ssh/orcad-state-mutation-exec.test.ts new file mode 100644 index 00000000000..2d1a02927b2 --- /dev/null +++ b/src/main/ssh/orcad-state-mutation-exec.test.ts @@ -0,0 +1,93 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ClientChannel } from 'ssh2' +import { + ORCAD_STATE_MUTATION_CLIENT_TIMEOUT_MS, + execOrcadStateMutation, + execOrcadStateMutationOr +} from './orcad-state-mutation-exec' +import { execOrcadRemoteOr } from './orcad-remote-runtime-control' +import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import type { SshConnection } from './ssh-connection' + +/** An ssh2 exec channel whose remote command answers only when the test says so. */ +class SlowChannel extends EventEmitter { + readonly stderr = new EventEmitter() + readonly stdin = this + closed = false + // sshd acknowledges the close at once, which ssh2 reports as a confirmed close. + close(): void { + this.closed = true + queueMicrotask(() => this.emit('close', 0)) + } + resume(): void {} + finish(output: string): void { + this.emit('data', Buffer.from(output)) + this.emit('close', 0) + } +} + +let channel: SlowChannel +const conn = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand reads only the channel events, close() and resume() SlowChannel implements. + exec: async () => channel as unknown as ClientChannel, + usesSystemSshTransport: () => false +} +const target = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only exec and usesSystemSshTransport are read. + conn: conn as unknown as SshConnection, + host: getRemoteHostPlatform('linux-x64') +} + +beforeEach(() => { + vi.useFakeTimers() + channel = new SlowChannel() +}) +afterEach(() => { + vi.useRealTimers() +}) + +describe('a snapshot restore that runs past the 30s exec timeout', () => { + it('was read as a confirmed failure by the generic exec, though the host kept restoring', async () => { + const generic = execOrcadRemoteOr(target, 'restore', 'FALLBACK') + await vi.advanceTimersByTimeAsync(31_000) + await expect(generic).resolves.toBe('FALLBACK') + expect(channel.closed).toBe(true) + }) + + it('keeps waiting past 30s and returns the restore’s own answer', async () => { + const restore = execOrcadStateMutation(target, 'restore') + await vi.advanceTimersByTimeAsync(45_000) + expect(channel.closed).toBe(false) + channel.finish('RESTORED\n') + await expect(restore).resolves.toBe('RESTORED\n') + }) + + it('treats giving up as unconfirmed, even when sshd confirms the channel close', async () => { + const restore = execOrcadStateMutationOr(target, 'restore', 'FALLBACK') + const settled = restore.catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(ORCAD_STATE_MUTATION_CLIENT_TIMEOUT_MS + 1_000) + const error = await settled + expect(channel.closed).toBe(true) + expect(isUnconfirmedSshCommandTermination(error)).toBe(true) + }) + + it.each(['STATE_MUTATION_BUSY', 'STATE_MUTATION_DEADLINE'])( + 'reads the host’s %s answer as unconfirmed, never as a failed restore', + async (answer) => { + const restore = execOrcadStateMutationOr(target, 'restore', 'FALLBACK') + await vi.advanceTimersByTimeAsync(0) + channel.finish(`${answer}\n`) + const error = await restore.catch((caught: unknown) => caught) + expect(isUnconfirmedSshCommandTermination(error)).toBe(true) + } + ) + + it('reads a command that exited non-zero as finished, so the fallback applies', async () => { + const restore = execOrcadStateMutationOr(target, 'restore', 'FALLBACK') + await vi.advanceTimersByTimeAsync(0) + channel.emit('close', 2) + await expect(restore).resolves.toBe('FALLBACK') + }) +}) diff --git a/src/main/ssh/orcad-state-mutation-exec.ts b/src/main/ssh/orcad-state-mutation-exec.ts new file mode 100644 index 00000000000..a940d88029f --- /dev/null +++ b/src/main/ssh/orcad-state-mutation-exec.ts @@ -0,0 +1,74 @@ +/** + * Running a snapshot capture, restore or clear on the host. + * + * A closed channel is not a stopped command: sshd keeps a pty-less child running, and these + * commands print nothing until they finish. So a timeout, abort or dropped channel leaves the + * work unconfirmed, which keeps the activation fence fresh instead of letting a second restore + * or a launch follow. Only the command's own answer, or its exit, is a verdict. + */ +import { ORCAD_STATE_MUTATION_DEADLINE_SECONDS } from './orcad-state-snapshot' +import { + ORCAD_STATE_MUTATION_BUSY, + ORCAD_STATE_MUTATION_DEADLINE +} from './orcad-state-snapshot-members' +import type { OrcadRemoteExecTarget } from './orcad-remote-runtime-control' +import { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' +import { isOrcadFenceLost, OrcadFenceLostError } from './orcad-activation-fence-scope' + +// Longer than the host's own deadline, so the host kills the work before the client gives up. +export const ORCAD_STATE_MUTATION_CLIENT_TIMEOUT_MS = + (ORCAD_STATE_MUTATION_DEADLINE_SECONDS + 60) * 1000 + +function unconfirmed(error: Error): Error & { sshChannelCloseConfirmed: false } { + return Object.assign(error, { sshChannelCloseConfirmed: false as const }) +} + +/** Never aborted: cancelling the request would only stop the client from seeing the outcome. */ +export async function execOrcadStateMutation( + target: OrcadRemoteExecTarget, + command: string +): Promise { + let output: string + try { + output = await execCommand(target.conn, command, { + wrapCommand: target.host.commandDialect !== 'powershell', + timeoutMs: ORCAD_STATE_MUTATION_CLIENT_TIMEOUT_MS + }) + } catch (error) { + // A termination error carries this flag; an exit-status error means the command finished. + if (error instanceof Error && 'sshChannelCloseConfirmed' in error) { + throw unconfirmed(error) + } + throw isOrcadFenceLost(error) ? new OrcadFenceLostError() : error + } + const verdict = output.trim().split('\n').pop()?.trim() + if (verdict === ORCAD_STATE_MUTATION_BUSY) { + throw unconfirmed( + new Error('Another snapshot capture or restore is still running on the host.') + ) + } + if (verdict === ORCAD_STATE_MUTATION_DEADLINE) { + throw unconfirmed( + new Error( + `A snapshot capture or restore ran past ${ORCAD_STATE_MUTATION_DEADLINE_SECONDS}s and ` + + 'the host stopped it part way through.' + ) + ) + } + return output +} + +/** A finished command that failed reads as `fallback`; an unconfirmed one propagates. */ +export function execOrcadStateMutationOr( + target: OrcadRemoteExecTarget, + command: string, + fallback = '' +): Promise { + return execOrcadStateMutation(target, command).catch((error: unknown) => { + // A lost fence is a refusal, never a failed capture to fall back from. + if (isUnconfirmedSshCommandTermination(error) || error instanceof OrcadFenceLostError) { + throw error + } + return fallback + }) +} diff --git a/src/main/ssh/orcad-state-mutation-fence-heartbeat.test.ts b/src/main/ssh/orcad-state-mutation-fence-heartbeat.test.ts new file mode 100644 index 00000000000..11c19a1456e --- /dev/null +++ b/src/main/ssh/orcad-state-mutation-fence-heartbeat.test.ts @@ -0,0 +1,118 @@ +/** + * A state mutation can outlast the activation fence's stale window, so it keeps the fence + * fresh while it runs. Run for real with a one-second beat and the real steal command. + */ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { RELAY_INSTALL_LOCK_NAME } from '../../shared/relay-install-lock-name' +import { ORCAD_ACTIVATION_TRANSACTION_DIRNAME } from './orcad-activation-transaction' +import { serializedStateMutationCommand } from './orcad-state-snapshot' +import { ORCAD_FENCE_OWNER_FILENAME } from './orcad-activation-fence-scope' +import { tryStealInstallLockCommand } from './ssh-relay-install-lock-commands' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') +const STALE_SECONDS = 3 + +async function sh(command: string): Promise { + return (await runProcess({ program: '/bin/sh', args: ['-c', command] })).stdout.trim() +} +const pause = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +describe.skipIf(process.platform === 'win32')( + 'the fence heartbeat of a running state mutation', + () => { + let base = '' + let fence = '' + + beforeEach(async () => { + base = mkdtempSync(join(tmpdir(), 'orcad-fence-beat-')) + fence = join(base, ORCAD_ACTIVATION_TRANSACTION_DIRNAME, RELAY_INSTALL_LOCK_NAME) + mkdirSync(fence, { recursive: true }) + writeFileSync(join(fence, ORCAD_FENCE_OWNER_FILENAME), 'holder-1') + }) + // The mutation runs under the fence its run holds, generation `holder-1`. + const mutation = (script: string): string => + serializedStateMutationCommand(base, script, 1, { lockDir: fence, token: 'holder-1' }) + afterEach(() => { + rmSync(base, { recursive: true, force: true }) + }) + + const backdate = (): Promise => sh(`touch -m -t 200001010000 '${fence}'`) + const steal = (): Promise => sh(tryStealInstallLockCommand(posix, fence, STALE_SECONDS)) + const age = (): number => Date.now() / 1000 - statSync(fence).mtimeMs / 1000 + + it('is not stolen while a long mutation runs, and is once its holder dies', async () => { + // Stands in for a restore that outlasts the stale window. + const run = spawnProcess({ + program: '/bin/sh', + args: ['-c', mutation('sleep 30')] + }) + try { + const pidFile = join(base, 'orcad-state-mutation.lock', 'pid') + await expect + .poll(() => existsSync(pidFile) && readFileSync(pidFile, 'utf8').trim()) + .toBeTruthy() + + await backdate() + await pause(2_500) + expect(age()).toBeLessThan(STALE_SECONDS) + expect(await steal()).toBe('BUSY') + + // The host process dies (OOM, reboot of the session): nothing refreshes the fence now. + process.kill(Number(readFileSync(pidFile, 'utf8').trim()), 'SIGKILL') + await pause(1_500) + await backdate() + await pause(2_500) + expect(age()).toBeGreaterThan(STALE_SECONDS) + expect(await steal()).toMatch(/OK$/u) + } finally { + run.kill('SIGKILL') + } + }, 20_000) + + it('stops refreshing a fence another run took over mid-mutation, and keeps its token', async () => { + const run = spawnProcess({ program: '/bin/sh', args: ['-c', mutation('sleep 6')] }) + try { + await pause(1_500) + writeFileSync(join(fence, ORCAD_FENCE_OWNER_FILENAME), 'successor') + await backdate() + await pause(2_500) + expect(age()).toBeGreaterThan(STALE_SECONDS) + expect(readFileSync(join(fence, ORCAD_FENCE_OWNER_FILENAME), 'utf8')).toBe('successor') + } finally { + run.kill('SIGKILL') + } + }, 20_000) + + it('never starts a mutation once its run no longer owns the fence', async () => { + writeFileSync(join(fence, ORCAD_FENCE_OWNER_FILENAME), 'successor') + const marker = join(base, 'mutated') + expect(await sh(mutation(`touch '${marker}'`))).toBe('__ORCAD_FENCE_LOST__') + expect(existsSync(marker)).toBe(false) + expect(existsSync(join(base, 'orcad-state-mutation.lock'))).toBe(false) + }, 20_000) + + it('stops refreshing once the mutation finishes, and never creates a missing fence', async () => { + await sh(mutation('sleep 2')) + await backdate() + await pause(2_500) + expect(age()).toBeGreaterThan(STALE_SECONDS) + + rmSync(fence, { recursive: true }) + await sh(serializedStateMutationCommand(base, 'sleep 2', 1, null)) + expect(existsSync(fence)).toBe(false) + }, 20_000) + } +) diff --git a/src/main/ssh/orcad-state-mutation-owner-record.ts b/src/main/ssh/orcad-state-mutation-owner-record.ts new file mode 100644 index 00000000000..fd4e8d7acb0 --- /dev/null +++ b/src/main/ssh/orcad-state-mutation-owner-record.ts @@ -0,0 +1,33 @@ +/** + * How a holder records itself in the POSIX state-mutation lock: a mutation, or the exited-owner + * fence steal that holds the lock across a takeover. A leaf, so the relay's lock commands can use it. + */ + +/** + * Prints `pid`'s process group. Why /proc first: BusyBox `ps` has no `-p`. The comm field + * may hold spaces and parens, so the fields are read after its last `)`. + */ +export function posixProcessGroupCommand(pid: string, procRoot = '/proc'): string { + const stat = `${procRoot}/${pid}/stat` + return [ + `if [ -r ${stat} ]; then stat=$(cat ${stat} 2>/dev/null); set -- \${stat##*")"}; echo "$3";`, + `else ps -o pgid= -p ${pid} 2>/dev/null | tr -d " "; fi` + ].join(' ') +} + +/** + * The holder's pid in the mutation lock `lock` (a quoted shell word); `onTaken` runs when another + * holder's is already there. Noclobber: a run that resumes after a takeover backs off. + */ +export function posixStateMutationPidRecord(lock: string, onTaken: string): string { + return `set -C; { echo $$ > ${lock}/pid; } 2>/dev/null || { ${onTaken} }; set +C;` +} + +/** The holder's own process group, recorded only when it was started as one. */ +export function posixStateMutationGroupRecord(lock: string): string { + return [ + 'if [ "${ORCA_STATE_MUTATION_GROUP:-}" = 1 ]; then', + `group=$(${posixProcessGroupCommand('$$')});`, + `case "$group" in ""|*[!0-9]*) ;; *) echo "$group" > ${lock}/pgid;; esac; fi;` + ].join(' ') +} diff --git a/src/main/ssh/orcad-state-mutation-timeout.test.ts b/src/main/ssh/orcad-state-mutation-timeout.test.ts new file mode 100644 index 00000000000..e7ab0f40511 --- /dev/null +++ b/src/main/ssh/orcad-state-mutation-timeout.test.ts @@ -0,0 +1,184 @@ +/** + * A snapshot capture or restore that outlives the client's wait, driven through the real + * deploy, rollback and recovery code on the fake host. The host keeps applying the slow one; + * a second mutation that takes the host lock answers busy, and one that doesn't (the old + * command shape) runs beside it, as the two restores did on a real host. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as DeployHelpers from './ssh-relay-deploy-helpers' +import type * as RecordFile from './orcad-remote-record-file' +import type * as InstallLock from './ssh-relay-install-lock' +import type * as VersionedInstall from './ssh-relay-versioned-install' + +vi.mock('./ssh-relay-deploy-helpers', async (importOriginal) => ({ + ...(await importOriginal()), + execCommand: vi.fn() +})) +vi.mock('./ssh-connection-utils', () => ({ shellEscape: (s: string) => `'${s}'` })) +vi.mock('./ssh-relay-install-lock', async (importOriginal) => ({ + ...(await importOriginal()), + acquireInstallLock: vi.fn() +})) +vi.mock('./orcad-remote-record-file', async (importOriginal) => ({ + ...(await importOriginal()), + writeAtomicOrcadRemoteRecord: vi.fn() +})) +vi.mock('./ssh-relay-versioned-install', async (importOriginal) => ({ + ...(await importOriginal()), + readLocalFullVersion: () => '0.2.0+bb01' +})) +vi.mock('./orcad-remote-install', () => ({ installOrcadBundle: vi.fn() })) +vi.mock('./orcad-remote-preflight', () => ({ preflightInstalledOrcad: vi.fn() })) +vi.mock('./orcad-local-build-hash', () => ({ + computeLocalOrcadBuildHash: () => 'abc123def4567890' +})) + +import { execCommand } from './ssh-relay-deploy-helpers' +import { acquireInstallLock } from './ssh-relay-install-lock' +import { writeAtomicOrcadRemoteRecord } from './orcad-remote-record-file' +import { deployOrcad } from './orcad-remote-deploy' +import { rollbackOrcad } from './orcad-remote-rollback' +import { recoverInterruptedOrcadActivation } from './orcad-activation-recovery' +import { getRemoteHostPlatform } from './ssh-remote-platform' +import { SSH_EXEC_TIMEOUT_CODE, isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command' +import type { SshConnection } from './ssh-connection' +import { BUILD_HASH, FakeOrcadHost, NEW, OLD } from './orcad-activation-host-test-harness' +import { isSnapshotCaptureCommand } from './orcad-snapshot-capture-command' + +let host = new FakeOrcadHost() +type StateMutation = 'capture' | 'restore' +let slow: StateMutation | null = null +let running = false + +const slot = { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: execCommand is mocked, so nothing reads the connection. + conn: {} as SshConnection, + host: getRemoteHostPlatform('linux-x64'), + remoteHome: '/home/u', + nodePath: '/usr/bin/node', + userDataDir: '/home/u/.orca', + bindHost: '127.0.0.1', + port: 7777, + readinessTimeoutMs: 50, + sleep: async () => {}, + now: () => new Date('2026-02-02T00:00:00.000Z') +} +const census = { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 } +const deploy = (): Promise => + deployOrcad({ ...slot, localOrcadDir: '/local/out/orcad', target: 'linux-x64-glibc', census }) +const rollback = (): Promise => + rollbackOrcad({ + ...slot, + record: FakeOrcadHost.newRecord(), + census, + targetBuildHash: BUILD_HASH, + targetDaemonProtocol: { protocolVersion: 3, previousProtocolVersions: [1, 2] } + }) + +function stateMutation(command: string): StateMutation | null { + if (!command.includes('orcad-state-snapshots/')) { + return null + } + if (command.includes('.orcad-state-restore-stage')) { + return 'restore' + } + return isSnapshotCaptureCommand(command) ? 'capture' : null +} + +/** What ssh2 reports when the 30s timer closes the channel and sshd acknowledges the close. */ +function confirmedTimeout(): Error { + return Object.assign(new Error('Command timed out after 30s'), { + code: SSH_EXEC_TIMEOUT_CODE, + sshChannelCloseConfirmed: true + }) +} + +async function exec(command: string): Promise { + const kind = stateMutation(command) + if (kind && running && command.includes('orcad-state-mutation.lock')) { + return 'STATE_MUTATION_BUSY\n' + } + if (kind && kind === slow) { + slow = null + running = true + // The host finishes the work; the client only stopped waiting for it. + host.exec(command) + throw confirmedTimeout() + } + return host.exec(command) +} + +const count = (kind: StateMutation): number => + host.commands.filter((command) => stateMutation(command) === kind).length +const launchedAfter = (kind: StateMutation): boolean => { + const first = host.commands.findIndex((command) => stateMutation(command) === kind) + return host.commands.slice(first + 1).some((command) => command.includes('nohup')) +} +const orphanedFence = (): boolean => + host.commands.some((command) => command.startsWith('touch -m -t 200001010000')) + +beforeEach(() => { + vi.clearAllMocks() + slow = null + running = false + vi.mocked(execCommand).mockImplementation(async (_conn, command) => exec(command)) + vi.mocked(acquireInstallLock).mockImplementation(async (_conn, dir, _host, options) => { + if (!host.acquireFence(options)) { + const { RemoteInstallLockBusyError } = await vi.importActual( + './ssh-relay-install-lock' + ) + throw new RemoteInstallLockBusyError(dir, 0) + } + }) + vi.mocked(writeAtomicOrcadRemoteRecord).mockImplementation(async (_target, path, contents) => + host.write(path, contents) + ) +}) + +describe('a state mutation that outlives the client’s wait', () => { + it('stops a rollback before a rescue restore or a launch can follow it', async () => { + host = FakeOrcadHost.activatedNew() + slow = 'restore' + const error = await rollback().catch((caught: unknown) => caught) + expect(isUnconfirmedSshCommandTermination(error)).toBe(true) + expect(count('restore')).toBe(1) + expect(launchedAfter('restore')).toBe(false) + expect(host.fence).toBe(true) + expect(host.journal).not.toBeNull() + expect(orphanedFence()).toBe(false) + }) + + it('keeps recovery’s fence fresh, so the next recover waits instead of restoring again', async () => { + host = FakeOrcadHost.deployedOld() + await deploy() + const total = host.mutations + // Lost after the candidate launched: candidate-ready journal, record and release remain. + host = FakeOrcadHost.deployedOld() + host.crashAt = total - 2 + await deploy().catch(() => undefined) + host.crashAt = null + expect(host.alive.has(NEW)).toBe(true) + slow = 'restore' + const first = await recoverInterruptedOrcadActivation({ ...slot, acceptChangedState: true }) + expect(first).toMatchObject({ outcome: 'refused', verdict: 'unverifiable' }) + expect(orphanedFence()).toBe(false) + expect(count('restore')).toBe(1) + + const second = await recoverInterruptedOrcadActivation({ ...slot, acceptChangedState: true }) + expect(second).toMatchObject({ outcome: 'pending' }) + expect(count('restore')).toBe(1) + expect(host.alive.size).toBe(0) + }) + + it('never restarts the incumbent beside a snapshot capture that is still copying', async () => { + host = FakeOrcadHost.deployedOld() + slow = 'capture' + const error = await deploy().catch((caught: unknown) => caught) + expect(isUnconfirmedSshCommandTermination(error)).toBe(true) + expect(launchedAfter('capture')).toBe(false) + expect(host.alive.has(OLD)).toBe(false) + expect(host.alive.has(NEW)).toBe(false) + expect(host.fence).toBe(true) + expect(orphanedFence()).toBe(false) + }) +}) diff --git a/src/main/ssh/orcad-state-snapshot-members.ts b/src/main/ssh/orcad-state-snapshot-members.ts new file mode 100644 index 00000000000..b462b1f4aaa --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot-members.ts @@ -0,0 +1,39 @@ +/** What the pre-activation snapshot holds; shared by the POSIX commands and the Windows host script. */ + +/** + * Root-relative paths a rollback needs restored. Everything else under the data root is + * either regenerable, or owned by a process that survives the rollback. + */ +export const ORCAD_SNAPSHOT_MEMBERS = [ + 'orca-profile-index.json', + // Pre-profiles layout; still read as a migration source. + 'orca-data.json', + 'profiles', + // Cross-profile SQLite moves must survive an orcad rollback too. + 'profile-move-intents' +] as const + +/** Never captured and never restored — see `orcad-state-snapshot.ts`. */ +export const ORCAD_SNAPSHOT_EXCLUDED = ['daemon', 'logs'] as const + +export const ORCAD_STATE_RESTORE_STAGE_DIRNAME = '.orcad-state-restore-stage' + +/** Under `~/.orca-remote`: held by one snapshot capture, restore or clear at a time. */ +export const ORCAD_STATE_MUTATION_LOCK_DIRNAME = 'orcad-state-mutation.lock' + +/** + * How often a running mutation refreshes the activation fence's mtime. Why: the fence goes + * stale by age (INSTALL_LOCK_STALE_MS), and a mutation may outlast that, so a live run keeps + * it fresh while a dead one stops within a beat. + */ +export const ORCAD_STATE_MUTATION_FENCE_HEARTBEAT_SECONDS = 60 +/** How long a lock an exited desktop process left must sit untouched before it is reclaimed. */ +export const ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS = 3 * ORCAD_STATE_MUTATION_FENCE_HEARTBEAT_SECONDS + +/** A state mutation found another still running, so it did nothing. */ +export const ORCAD_STATE_MUTATION_BUSY = 'STATE_MUTATION_BUSY' +/** The host-side deadline killed a state mutation part way through. */ +export const ORCAD_STATE_MUTATION_DEADLINE = 'STATE_MUTATION_DEADLINE' + +/** Windows keeps the snapshot as a directory copy, where POSIX keeps `state.tar`. */ +export const ORCAD_WINDOWS_SNAPSHOT_STATE_DIRNAME = 'state' diff --git a/src/main/ssh/orcad-state-snapshot-shell.test.ts b/src/main/ssh/orcad-state-snapshot-shell.test.ts new file mode 100644 index 00000000000..9cf0192a63f --- /dev/null +++ b/src/main/ssh/orcad-state-snapshot-shell.test.ts @@ -0,0 +1,233 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { + captureOrcadStateSnapshotCommand, + clearOrcadStateSnapshotMembersCommand, + parseOrcadSnapshotCapture, + parseOrcadSnapshotRestore, + restoreOrcadStateSnapshotCommand, + serializedStateMutationCommand +} from './orcad-state-snapshot' +import { posixProcessGroupCommand } from './orcad-state-mutation-owner-record' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +const posix = getRemoteHostPlatform('linux-x64') + +// Linux hosts often run dash as /bin/sh; its builtins differ from bash's (`kill -- -pgid`). +const SHELL = process.env.ORCA_TEST_POSIX_SHELL ?? '/bin/sh' + +async function sh(command: string): Promise { + const result = await runProcess({ program: SHELL, args: ['-c', command] }) + return result.stdout +} + +describe.skipIf(process.platform === 'win32')('snapshot commands on a real shell', () => { + let base: string + let root: string + let snapshot: string + let remoteBase: string + + beforeEach(() => { + base = mkdtempSync(join(tmpdir(), 'orcad-snapshot-shell-')) + root = join(base, 'root') + snapshot = join(base, 'snapshots', 'pre-1') + remoteBase = join(base, '.orca-remote') + mkdirSync(join(root, 'profiles'), { recursive: true }) + mkdirSync(join(root, 'daemon'), { recursive: true }) + writeFileSync(join(root, 'profiles', 'p.json'), 'old') + writeFileSync(join(root, 'daemon', 'token'), 'live-daemon') + }) + + afterEach(() => { + rmSync(base, { recursive: true, force: true }) + }) + + it('restores members, drops files the newer build added, and leaves the daemon alone', async () => { + expect( + parseOrcadSnapshotCapture( + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + ) + ).toBe('captured') + writeFileSync(join(root, 'profiles', 'p.json'), 'migrated') + writeFileSync(join(root, 'profiles', 'added.json'), 'new') + writeFileSync(join(root, 'daemon', 'token'), 'rotated') + + expect( + parseOrcadSnapshotRestore( + await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + ) + ).toBe('restored') + expect(readFileSync(join(root, 'profiles', 'p.json'), 'utf8')).toBe('old') + expect(existsSync(join(root, 'profiles', 'added.json'))).toBe(false) + expect(readFileSync(join(root, 'daemon', 'token'), 'utf8')).toBe('rotated') + expect(existsSync(join(root, '.orcad-state-restore-stage'))).toBe(false) + }) + + it('keeps live state when the archive cannot be extracted', async () => { + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + writeFileSync(join(snapshot, 'state.tar'), 'not a tar archive') + writeFileSync(join(root, 'profiles', 'p.json'), 'current') + + expect( + parseOrcadSnapshotRestore( + await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + ) + ).toBe('failed') + expect(readFileSync(join(root, 'profiles', 'p.json'), 'utf8')).toBe('current') + }) + + it('reruns cleanly after a restore interrupted between removal and replacement', async () => { + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + // Simulates a crash that left the stage behind and the live members already removed. + mkdirSync(join(root, '.orcad-state-restore-stage', 'profiles'), { recursive: true }) + rmSync(join(root, 'profiles'), { recursive: true }) + + expect( + parseOrcadSnapshotRestore( + await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + ) + ).toBe('restored') + expect(readFileSync(join(root, 'profiles', 'p.json'), 'utf8')).toBe('old') + }) + + it('clears only the snapshot members for a root that started empty', async () => { + expect( + parseOrcadSnapshotRestore( + await sh(clearOrcadStateSnapshotMembersCommand(posix, root, remoteBase)) + ) + ).toBe('restored') + expect(existsSync(join(root, 'profiles'))).toBe(false) + expect(readFileSync(join(root, 'daemon', 'token'), 'utf8')).toBe('live-daemon') + }) + + it('answers busy and leaves state alone while another state mutation holds the host lock', async () => { + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + writeFileSync(join(root, 'profiles', 'p.json'), 'current') + const lock = join(remoteBase, 'orcad-state-mutation.lock') + mkdirSync(lock) + // This test process is alive, so it reads as a restore still running. + writeFileSync(join(lock, 'pid'), String(process.pid)) + + const output = await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + expect(output.trim()).toBe('STATE_MUTATION_BUSY') + expect(readFileSync(join(root, 'profiles', 'p.json'), 'utf8')).toBe('current') + expect(existsSync(lock)).toBe(true) + }) + + it('takes over a lock whose whole process group is gone, and releases it when done', async () => { + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + writeFileSync(join(root, 'profiles', 'p.json'), 'current') + const lock = join(remoteBase, 'orcad-state-mutation.lock') + mkdirSync(lock) + const exited = (await runProcess({ program: '/bin/sh', args: ['-c', 'echo $$'] })).stdout + writeFileSync(join(lock, 'pid'), exited.trim()) + writeFileSync(join(lock, 'pgid'), exited.trim()) + + expect( + parseOrcadSnapshotRestore( + await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + ) + ).toBe('restored') + expect(readFileSync(join(root, 'profiles', 'p.json'), 'utf8')).toBe('old') + expect(existsSync(lock)).toBe(false) + }) + + it('on a host that recorded no group, waits on a live pid or a recent beat, then takes over', async () => { + await sh(captureOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase)) + const lock = join(remoteBase, 'orcad-state-mutation.lock') + mkdirSync(lock, { recursive: true }) + const restore = async (): Promise => + (await sh(restoreOrcadStateSnapshotCommand(posix, root, snapshot, remoteBase))).trim() + const old = new Date(Date.now() - 10 * 60_000) + + writeFileSync(join(lock, 'pid'), String(process.pid)) + utimesSync(lock, old, old) + expect(await restore()).toBe('STATE_MUTATION_BUSY') + + const exited = (await runProcess({ program: '/bin/sh', args: ['-c', 'echo $$'] })).stdout + writeFileSync(join(lock, 'pid'), exited.trim()) + utimesSync(lock, new Date(), new Date()) + expect(await restore()).toBe('STATE_MUTATION_BUSY') + + // A dead pid and three missed beats: nothing of that run is provably left. + utimesSync(lock, old, old) + expect(parseOrcadSnapshotRestore(await restore())).toBe('restored') + expect(existsSync(lock)).toBe(false) + }) + + it('reads a process group from a proc stat whose command holds spaces and parens', async () => { + const proc = join(base, 'proc') + mkdirSync(join(proc, '4321'), { recursive: true }) + writeFileSync(join(proc, '4321', 'stat'), '4321 (we ird) (x)) S 1 777 777 0 -1 4194560 0 0') + expect((await sh(posixProcessGroupCommand('4321', proc))).trim()).toBe('777') + }) + + it.skipIf(!existsSync('/proc/self/stat'))( + 'records the group from /proc where ps has no -p (BusyBox)', + async () => { + const shim = join(base, 'bin') + mkdirSync(shim) + writeFileSync( + join(shim, 'ps'), + '#!/bin/sh\necho "ps: unrecognized option: p" >&2\nexit 1\n', + { + mode: 0o755 + } + ) + const lock = join(remoteBase, 'orcad-state-mutation.lock') + const run = spawnProcess({ + program: SHELL, + args: [ + '-c', + `PATH='${shim}':"$PATH"; ${serializedStateMutationCommand(remoteBase, 'sleep 5', 1)}` + ] + }) + try { + await expect.poll(() => existsSync(join(lock, 'pgid'))).toBe(true) + expect(Number(readFileSync(join(lock, 'pgid'), 'utf8'))).toBeGreaterThan(1) + } finally { + run.kill('SIGKILL') + } + }, + 20_000 + ) + + it('keeps the lock while a killed shell’s child still runs, and frees it once the group is gone', async () => { + const lock = join(remoteBase, 'orcad-state-mutation.lock') + // Stands in for a restore whose shell dies while its rm or tar keeps going. + const run = spawnProcess({ + program: SHELL, + args: ['-c', serializedStateMutationCommand(remoteBase, 'sleep 30; echo DONE', 1)] + }) + try { + await expect + .poll(() => existsSync(join(lock, 'pid')) && existsSync(join(lock, 'pgid'))) + .toBe(true) + const shell = Number(readFileSync(join(lock, 'pid'), 'utf8')) + const group = Number(readFileSync(join(lock, 'pgid'), 'utf8')) + expect(group).toBeGreaterThan(1) + process.kill(shell, 'SIGKILL') + + const next = (): Promise => + sh(serializedStateMutationCommand(remoteBase, 'echo RAN', 1)) + expect((await next()).trim()).toBe('STATE_MUTATION_BUSY') + + process.kill(-group, 'SIGKILL') + await expect.poll(async () => (await next()).trim(), { timeout: 5_000 }).toBe('RAN') + expect(existsSync(lock)).toBe(false) + } finally { + run.kill('SIGKILL') + } + }, 20_000) +}) diff --git a/src/main/ssh/orcad-state-snapshot.test.ts b/src/main/ssh/orcad-state-snapshot.test.ts index f660e95e5a4..2bda2a2e54f 100644 --- a/src/main/ssh/orcad-state-snapshot.test.ts +++ b/src/main/ssh/orcad-state-snapshot.test.ts @@ -1,27 +1,36 @@ import { describe, expect, it } from 'vitest' import { - ORCAD_SNAPSHOT_EXCLUDED, - ORCAD_SNAPSHOT_MEMBERS, captureOrcadStateSnapshotCommand, + clearOrcadStateSnapshotMembersCommand, compareOrcadStateSnapshotCommand, newestStateMtimeCommand, orcadSnapshotDirName, parseNewestStateMtimeSeconds, parseOrcadSnapshotCapture, + parseOrcadSnapshotPresence, parseOrcadSnapshotRestore, - restoreOrcadStateSnapshotCommand + probeOrcadStateSnapshotCommand, + restoreOrcadStateSnapshotCommand, + ORCAD_STATE_MUTATION_DEADLINE_SECONDS } from './orcad-state-snapshot' import { getRemoteHostPlatform } from './ssh-remote-platform' +import { ORCAD_SNAPSHOT_EXCLUDED, ORCAD_SNAPSHOT_MEMBERS } from './orcad-state-snapshot-members' const posix = getRemoteHostPlatform('linux-x64') const windows = getRemoteHostPlatform('win32-x64') const ROOT = '/home/u/.orca' const SNAP = '/home/u/.orca-remote/orcad-state-snapshots/pre-0.2.0+bb01-1000' +const BASE = '/home/u/.orca-remote' + +/** The work inside the lock-and-deadline wrapper, with its quoting undone. */ +function innerScript(command: string): string { + return command.replaceAll(`'\\''`, `'`) +} describe('capturing the pre-activation snapshot', () => { it('captures the profile state a rollback needs', () => { - const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE) for (const member of ORCAD_SNAPSHOT_MEMBERS) { expect(command).toContain(`'${member}'`) } @@ -30,15 +39,15 @@ describe('capturing the pre-activation snapshot', () => { // The live daemon owns /daemon and outlives every restart. Restoring a stale copy of // its socket, PID record and token would break the fence that keeps its terminals adoptable. it.each(ORCAD_SNAPSHOT_EXCLUDED)('never captures %s', (excluded) => { - expect(captureOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + expect(captureOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)).not.toContain(`'${excluded}'`) }) it.each(ORCAD_SNAPSHOT_EXCLUDED)('never removes or restores over %s', (excluded) => { - expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).not.toContain(`'${excluded}'`) + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)).not.toContain(`'${excluded}'`) }) it('writes the archive under a temp name and renames, so a killed deploy leaves no torn tar', () => { - const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP) + const command = captureOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE) expect(command).toContain('.partial') expect(command.indexOf('tar -C')).toBeLessThan(command.indexOf('mv ')) }) @@ -59,18 +68,42 @@ describe('capturing the pre-activation snapshot', () => { }) }) +describe('the host-side guard around every state mutation', () => { + it.each([ + ['capture', () => captureOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)], + ['restore', () => restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)], + ['clear', () => clearOrcadStateSnapshotMembersCommand(posix, ROOT, BASE)] + ])('%s runs under the host lock and a host-enforced deadline', (_label, build) => { + const command = build() + expect(command).toContain(`timeout -s KILL ${ORCAD_STATE_MUTATION_DEADLINE_SECONDS} sh -c`) + expect(innerScript(command)).toContain(`lock='${BASE}/orcad-state-mutation.lock'`) + expect(command).toContain('echo STATE_MUTATION_DEADLINE') + }) +}) + describe('restoring the snapshot', () => { - it('clears the members before extracting, so files the new build added do not survive', () => { - const command = restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP) - expect(command.indexOf('rm -rf')).toBeLessThan(command.indexOf('tar -C')) + it('proves the archive extracts before clearing the live members', () => { + const command = innerScript(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)) + const extract = command.indexOf(`tar -C '${ROOT}/.orcad-state-restore-stage'`) + expect(extract).toBeGreaterThan(-1) + expect(extract).toBeLessThan(command.indexOf(`rm -rf '${ROOT}'/'profiles'`)) }) it('reports a missing archive instead of extracting nothing and claiming success', () => { - expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP)).toContain('echo MISSING') + expect(restoreOrcadStateSnapshotCommand(posix, ROOT, SNAP, BASE)).toContain('echo MISSING') expect(parseOrcadSnapshotRestore('MISSING')).toBe('missing') expect(parseOrcadSnapshotRestore('RESTORED')).toBe('restored') expect(parseOrcadSnapshotRestore('FAILED')).toBe('failed') }) + + it.each([ + ['PRESENT', 'present'], + ['ABSENT', 'absent'], + ['', 'unverifiable'], + ['bash: tar: command not found', 'unverifiable'] + ])('reads snapshot presence %j as %s, never treating silence as absence', (out, expected) => { + expect(parseOrcadSnapshotPresence(out)).toBe(expected) + }) }) describe('detecting writes since activation', () => { @@ -91,12 +124,39 @@ describe('detecting writes since activation', () => { }) describe('Windows hosts', () => { + const BASE = 'C:/Users/u/.orca-remote' it.each([ - ['capture', () => captureOrcadStateSnapshotCommand(windows, ROOT, SNAP)], - ['restore', () => restoreOrcadStateSnapshotCommand(windows, ROOT, SNAP)], - ['compare', () => compareOrcadStateSnapshotCommand(windows, ROOT, SNAP)], - ['mtime', () => newestStateMtimeCommand(windows, ROOT)] - ])('refuses %s rather than emitting a POSIX command', (_label, build) => { - expect(build).toThrow('orcad to a Windows host is not implemented') + [ + 'capture', + 'snapshot-capture', + (base?: string) => captureOrcadStateSnapshotCommand(windows, ROOT, SNAP, base ?? '') + ], + [ + 'restore', + 'snapshot-restore', + (base?: string) => restoreOrcadStateSnapshotCommand(windows, ROOT, SNAP, base ?? '') + ], + [ + 'clear', + 'snapshot-clear', + (base?: string) => clearOrcadStateSnapshotMembersCommand(windows, ROOT, base ?? '') + ], + [ + 'presence', + 'snapshot-probe', + (base?: string) => probeOrcadStateSnapshotCommand(windows, SNAP, base) + ], + [ + 'compare', + 'snapshot-compare', + (base?: string) => compareOrcadStateSnapshotCommand(windows, ROOT, SNAP, base) + ], + ['mtime', 'state-newest-mtime', (base?: string) => newestStateMtimeCommand(windows, ROOT, base)] + ])('%s runs the host script op %s with node.exe, never a POSIX command', (_label, op, build) => { + const command = build(BASE) + expect(command).toContain(` ${op} `) + expect(command).toMatch(/^C:\\Users\\u\\\.orca-remote\\runtimes\\node-[0-9a-f]+\\node\.exe /u) + expect(command).not.toMatch(/tar |find |diff |EncodedCommand|powershell/u) + expect(() => build()).toThrow('~/.orca-remote') }) }) diff --git a/src/main/ssh/orcad-state-snapshot.ts b/src/main/ssh/orcad-state-snapshot.ts index 7d062275442..581c0269bb8 100644 --- a/src/main/ssh/orcad-state-snapshot.ts +++ b/src/main/ssh/orcad-state-snapshot.ts @@ -15,25 +15,30 @@ * fence that keeps its terminals adoptable — turning a rollback into the exact terminal * massacre the daemon exists to prevent. */ +import { + currentOrcadFence, + ORCAD_FENCE_LOST_EXIT, + ORCAD_FENCE_LOST_MARKER, + posixOrcadFenceGuard, + posixOrcadFenceOwnedTest, + type OrcadFence +} from './orcad-activation-fence-scope' import { shellEscape } from './ssh-connection-utils' -import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' -import { assertPosixOrcadHost as assertPosixHost } from './orcad-remote-host-support' - -/** - * Root-relative paths a rollback needs restored. Everything else under the data root is - * either regenerable, or owned by a process that survives the rollback. - */ -export const ORCAD_SNAPSHOT_MEMBERS = [ - 'orca-profile-index.json', - // Pre-profiles layout; still read as a migration source. - 'orca-data.json', - 'profiles', - // Cross-profile SQLite moves must survive an orcad rollback too. - 'profile-move-intents' -] as const - -/** Never captured and never restored — see the module comment. */ -export const ORCAD_SNAPSHOT_EXCLUDED = ['daemon', 'logs'] as const +import { isWindowsRemoteHost, joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' +import type { OrcadWindowsHostStateOp } from './orcad-windows-host-state-ops' +import { + ORCAD_SNAPSHOT_MEMBERS, + ORCAD_STATE_MUTATION_BUSY, + ORCAD_STATE_MUTATION_DEADLINE, + ORCAD_STATE_MUTATION_FENCE_HEARTBEAT_SECONDS, + ORCAD_STATE_MUTATION_LOCK_DIRNAME, + ORCAD_STATE_RESTORE_STAGE_DIRNAME +} from './orcad-state-snapshot-members' +import { orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { + posixStateMutationGroupRecord, + posixStateMutationPidRecord +} from './orcad-state-mutation-owner-record' /** * The member names go into the command unquoted (see `captureOrcadStateSnapshotCommand`), so @@ -47,6 +52,86 @@ function assertPlainMemberName(member: string): string { return member } +/** The Windows host-script op, or null on POSIX; `baseDir` is `~/.orca-remote`. */ +function windowsStateCommand( + host: RemoteHostPlatform, + baseDir: string | undefined, + op: OrcadWindowsHostStateOp, + args: string[] +): string | null { + if (!isWindowsRemoteHost(host)) { + return null + } + if (!baseDir) { + throw new Error('Windows orcad state commands need the ~/.orca-remote directory') + } + return orcadWindowsHostOpCommand(host, baseDir, op, args) +} + +/** Past this the host kills a capture, restore or clear; the client waits a minute longer. */ +export const ORCAD_STATE_MUTATION_DEADLINE_SECONDS = 15 * 60 + +/** + * Runs a state mutation under the host's lock and deadline. Why on the host: sshd keeps a + * pty-less command running after its channel closes, so a client that stops waiting has not + * stopped the work, and a rerun beside it would mix two restores in one stage. + */ +export function serializedStateMutationCommand( + baseDir: string, + script: string, + heartbeatSeconds = ORCAD_STATE_MUTATION_FENCE_HEARTBEAT_SECONDS, + // The holder's fence; null (a call outside any fence's run) refreshes no fence at all. + owned: OrcadFence | null = currentOrcadFence() +): string { + const lock = shellEscape(`${baseDir}/${ORCAD_STATE_MUTATION_LOCK_DIRNAME}`) + const busy = `echo ${ORCAD_STATE_MUTATION_BUSY}; exit 0;` + const staleMinutes = Math.max(1, Math.ceil((3 * heartbeatSeconds) / 60)) + const guarded = [ + `lock=${lock};`, + `mkdir -p ${shellEscape(baseDir)} 2>/dev/null;`, + 'if ! mkdir "$lock" 2>/dev/null; then', + 'holder=$(cat "$lock/pid" 2>/dev/null); group=$(cat "$lock/pgid" 2>/dev/null);', + // No pid yet: no work began, and the pid write is exclusive, so a late writer backs off. + 'if [ -z "$holder" ]; then', + `[ -n "$(find "$lock" -maxdepth 0 -mmin +1 2>/dev/null)" ] || { ${busy} };`, + // Why the group: a killed shell can leave its tar or rm running; any live member keeps it. + `elif [ -n "$group" ]; then kill -0 "-$group" 2>/dev/null && { ${busy} };`, + // No group recorded: a live pid, or a beat within three, still holds; past that it is gone. + `elif kill -0 "$holder" 2>/dev/null || [ -z "$(find "$lock" -maxdepth 0 -mmin +${staleMinutes} 2>/dev/null)" ]; then ${busy}`, + 'fi;', + `rm -rf "$lock"; mkdir "$lock" 2>/dev/null || { ${busy} }; fi;`, + posixStateMutationPidRecord('"$lock"', busy), + // Rechecked once the lock is held: an exited-owner steal holds it across the fence takeover. + owned + ? `${posixOrcadFenceOwnedTest(owned)} || { rm -rf "$lock"; echo ${ORCAD_FENCE_LOST_MARKER}; exit ${ORCAD_FENCE_LOST_EXIT}; };` + : '', + posixStateMutationGroupRecord('"$lock"'), + // `-c` never creates a fence that is gone; the beat ends within one sleep of this shell. + // Only a fence this run still owns: a superseded or foreign one ages toward takeover. + `beat_fence() { touch -c -m "$lock" 2>/dev/null; ${ + owned + ? `${posixOrcadFenceOwnedTest(owned)} && touch -c -m ${shellEscape(owned.lockDir)} 2>/dev/null;` + : ':;' + } };`, + 'beat_fence;', + `( while sleep ${heartbeatSeconds} && kill -0 $$ 2>/dev/null; do beat_fence; done ) >/dev/null 2>&1 & beat=$!;`, + `trap 'kill "$beat" 2>/dev/null; rm -rf "$lock"' EXIT;`, + script + ].join(' ') + const run = `sh -c ${shellEscape(guarded)}` + return [ + // Outermost: a superseded fence holder never takes the mutation lock or touches state. + ...(owned ? [posixOrcadFenceGuard(owned)] : []), + // Its own process group, so the lock can name every process the mutation started. + 'orca_state_group() { if command -v setsid >/dev/null 2>&1; then ORCA_STATE_MUTATION_GROUP=1 setsid "$@";', + `elif command -v perl >/dev/null 2>&1; then ORCA_STATE_MUTATION_GROUP=1 perl -e ${shellEscape('setpgrp(0, 0); exec { $ARGV[0] } @ARGV or exit 127')} "$@";`, + 'else "$@"; fi; };', + // Why timeout inside the group: KILL then reaches tar and rm, not only the shell. + `if command -v timeout >/dev/null 2>&1; then orca_state_group timeout -s KILL ${ORCAD_STATE_MUTATION_DEADLINE_SECONDS} ${run}; else orca_state_group ${run}; fi;`, + `status=$?; ${owned ? `[ "$status" -eq ${ORCAD_FENCE_LOST_EXIT} ] && exit ${ORCAD_FENCE_LOST_EXIT}; ` : ''}if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then echo ${ORCAD_STATE_MUTATION_DEADLINE}; fi` + ].join(' ') +} + function noSymlinkedStateCommand(path: string): string { return `links=$(find ${path} -type l -print) && [ -z "$links" ]` } @@ -57,6 +142,11 @@ export function orcadSnapshotDirName(fullVersion: string, takenAtMs: number): st return `pre-${fullVersion}-${takenAtMs}` } +/** The newer build's state, kept so an interrupted rollback can put it back. */ +export function orcadRollbackRescueDirName(fullVersion: string, takenAtMs: number): string { + return `rollback-rescue-${fullVersion}-${takenAtMs}` +} + /** * Capture the snapshot, or report why there is nothing to capture. * @@ -68,9 +158,13 @@ export function orcadSnapshotDirName(fullVersion: string, takenAtMs: number): st export function captureOrcadStateSnapshotCommand( host: RemoteHostPlatform, userDataDir: string, - snapshotDir: string + snapshotDir: string, + baseDir: string ): string { - assertPosixHost(host) + const windows = windowsStateCommand(host, baseDir, 'snapshot-capture', [userDataDir, snapshotDir]) + if (windows) { + return windows + } const root = shellEscape(userDataDir) const dir = shellEscape(snapshotDir) const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) @@ -87,16 +181,20 @@ export function captureOrcadStateSnapshotCommand( ) } ).join(' ') - return [ - `members=;`, - memberTests, - 'if [ -z "$members" ]; then echo EMPTY; else', - `mkdir -p ${dir} && umask 077 &&`, - // Why a temp name then mv: a deploy killed mid-tar must not leave a truncated archive - // that a later rollback would happily restore. - `tar -C ${root} -cf ${archive}.partial $members && mv ${archive}.partial ${archive} &&`, - 'echo CAPTURED; fi' - ].join(' ') + return serializedStateMutationCommand( + baseDir, + [ + `members=;`, + memberTests, + 'if [ -z "$members" ]; then echo EMPTY; else', + // Umask first so the snapshot dir, not just the archive, is owner-only. + `umask 077 && mkdir -p ${dir} &&`, + // Why a temp name then mv: a deploy killed mid-tar must not leave a truncated archive + // that a later rollback would happily restore. + `tar -C ${root} -cf ${archive}.partial $members && mv ${archive}.partial ${archive} &&`, + 'echo CAPTURED; fi' + ].join(' ') + ) } export type OrcadSnapshotCapture = 'captured' | 'empty' | 'failed' @@ -111,19 +209,36 @@ export function parseOrcadSnapshotCapture(output: string): OrcadSnapshotCapture export function probeOrcadStateSnapshotCommand( host: RemoteHostPlatform, - snapshotDir: string + snapshotDir: string, + baseDir?: string ): string { - assertPosixHost(host) + const windows = windowsStateCommand(host, baseDir, 'snapshot-probe', [snapshotDir]) + if (windows) { + return windows + } const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) return `test -f ${archive} && echo PRESENT || echo ABSENT` } +export type OrcadSnapshotPresence = 'present' | 'absent' | 'unverifiable' + +/** A lost probe is `unverifiable`, never `absent`. */ +export function parseOrcadSnapshotPresence(output: string): OrcadSnapshotPresence { + const value = output.trim().split('\n').pop()?.trim() + if (value === 'PRESENT') { + return 'present' + } + return value === 'ABSENT' ? 'absent' : 'unverifiable' +} + /** * Restore the snapshot over the data root. * - * Two things make this safe to run: the members are removed before extraction (so a file the - * new version added is gone rather than half-shadowed), and neither the removal nor the - * extraction can reach `/daemon`, because the member list never names it. + * Three things make this safe to run: the archive is extracted into a stage first, so an + * unreadable archive fails before live state is touched; the members are then removed before + * the staged copies move in (so a file the new version added is gone rather than + * half-shadowed); and neither step can reach `/daemon`, because the member list never + * names it. * * The caller must have stopped orcad first. This does not check — it cannot, from a shell — * so `orcad-remote-deploy.ts` owns that ordering. @@ -131,20 +246,61 @@ export function probeOrcadStateSnapshotCommand( export function restoreOrcadStateSnapshotCommand( host: RemoteHostPlatform, userDataDir: string, - snapshotDir: string + snapshotDir: string, + baseDir: string ): string { - assertPosixHost(host) + const windows = windowsStateCommand(host, baseDir, 'snapshot-restore', [userDataDir, snapshotDir]) + if (windows) { + return windows + } const root = shellEscape(userDataDir) const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) - const removals = ORCAD_SNAPSHOT_MEMBERS.map( - (member) => `rm -rf ${root}/${shellEscape(member)};` - ).join(' ') - return [ - `test -f ${archive} || { echo MISSING; exit 0; };`, - `test -d ${root} || mkdir -p ${root};`, - removals, - `tar -C ${root} -xf ${archive} && echo RESTORED || echo FAILED` - ].join(' ') + const stage = shellEscape(joinRemotePath(host, userDataDir, ORCAD_STATE_RESTORE_STAGE_DIRNAME)) + const removals = removeMembersCommand(root) + const replacements = ORCAD_SNAPSHOT_MEMBERS.map((member) => { + const name = shellEscape(member) + return `if [ -e ${stage}/${name} ]; then mv ${stage}/${name} ${root}/${name}; fi` + }).join(' && ') + const stagedMemberChecks = ORCAD_SNAPSHOT_MEMBERS.map( + (member) => `[ -e ${stage}/${shellEscape(member)} ]` + ).join(' || ') + return serializedStateMutationCommand( + baseDir, + [ + `test -f ${archive} || { echo MISSING; exit 0; };`, + 'umask 077;', + `test -d ${root} || mkdir -p ${root};`, + // Re-extracting from the intact archive makes an interrupted restore safe to rerun. + `rm -rf ${stage}; mkdir -p ${stage} || { echo FAILED; exit 0; };`, + // Extraction proves every archived byte is readable before live state is removed. + `tar -C ${stage} -xf ${archive} 2>/dev/null || { rm -rf ${stage}; echo FAILED; exit 0; };`, + `${stagedMemberChecks} || { rm -rf ${stage}; echo FAILED; exit 0; };`, + `if ${removals} && ${replacements}; then rm -rf ${stage}; echo RESTORED; else echo FAILED; fi` + ].join(' ') + ) +} + +/** Restore an originally empty state root after a candidate populated it. */ +export function clearOrcadStateSnapshotMembersCommand( + host: RemoteHostPlatform, + userDataDir: string, + baseDir: string +): string { + const windows = windowsStateCommand(host, baseDir, 'snapshot-clear', [userDataDir]) + if (windows) { + return windows + } + const root = shellEscape(userDataDir) + return serializedStateMutationCommand( + baseDir, + `test -d ${root} || mkdir -p ${root}; if ${removeMembersCommand(root)}; then echo RESTORED; else echo FAILED; fi` + ) +} + +function removeMembersCommand(root: string): string { + return ORCAD_SNAPSHOT_MEMBERS.map((member) => `rm -rf ${root}/${shellEscape(member)}`).join( + ' && ' + ) } export type OrcadSnapshotRestore = 'restored' | 'missing' | 'failed' @@ -161,9 +317,13 @@ export function parseOrcadSnapshotRestore(output: string): OrcadSnapshotRestore export function compareOrcadStateSnapshotCommand( host: RemoteHostPlatform, userDataDir: string, - snapshotDir: string + snapshotDir: string, + baseDir?: string ): string { - assertPosixHost(host) + const windows = windowsStateCommand(host, baseDir, 'snapshot-compare', [userDataDir, snapshotDir]) + if (windows) { + return windows + } const root = shellEscape(userDataDir) const dir = shellEscape(snapshotDir) const archive = shellEscape(joinRemotePath(host, snapshotDir, 'state.tar')) @@ -200,8 +360,15 @@ export function orcadSnapshotIsUnchanged(output: string): boolean { * caller compares; an `UNKNOWN` becomes `null`, which `assessOrcadRollback` treats as "yes, * assume writes". */ -export function newestStateMtimeCommand(host: RemoteHostPlatform, userDataDir: string): string { - assertPosixHost(host) +export function newestStateMtimeCommand( + host: RemoteHostPlatform, + userDataDir: string, + baseDir?: string +): string { + const windows = windowsStateCommand(host, baseDir, 'state-newest-mtime', [userDataDir]) + if (windows) { + return windows + } const root = shellEscape(userDataDir) const paths = ORCAD_SNAPSHOT_MEMBERS.map((member) => `${root}/${shellEscape(member)}`).join(' ') return [ diff --git a/src/main/ssh/orcad-stdio-bridge-provider.test.ts b/src/main/ssh/orcad-stdio-bridge-provider.test.ts new file mode 100644 index 00000000000..e69620d2e8d --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge-provider.test.ts @@ -0,0 +1,187 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { connect, type Server, type Socket } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { PassThrough } from 'node:stream' +import type { ClientChannel } from 'ssh2' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { OrcadStdioBridgeUnavailableError } from './orcad-host-unavailable' +import type { OrcadStdioBridge } from './orcad-stdio-bridge' +import { + ORCAD_STDIO_BRIDGE_MAX_CHANNELS, + OrcadStdioBridgePortForwardProvider +} from './orcad-stdio-bridge-provider' +import { spawnLocalBridgeChannel, startEchoServer } from './orcad-stdio-bridge-test-channel' +import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script' +import type { SshConnection } from './ssh-connection' +import type { StartedPortForward } from './ssh-port-forward-provider' + +let echo: { server: Server; port: number } +let script: string +let scratch: string +const started: StartedPortForward[] = [] +const clients: Socket[] = [] + +beforeAll(async () => { + echo = await startEchoServer() + scratch = mkdtempSync(join(tmpdir(), 'orcad-stdio-provider-')) + script = join(scratch, 'orcad-host-script.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) +}) + +afterEach(async () => { + for (const client of clients.splice(0)) { + client.destroy() + } + await Promise.all(started.splice(0).map((forward) => forward.close())) +}) + +afterAll(() => { + echo.server.close() + rmSync(scratch, { recursive: true, force: true }) +}) + +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the provider reads only getClient; exec goes through the injected openChannel. +const conn = { getClient: () => ({}) } as unknown as SshConnection +const bridge: OrcadStdioBridge = { command: 'bridge', mode: 'base64', wrapCommand: false } + +/** The real bridge, run locally through the Windows host script's base64 framing. */ +const localBridge = (): ClientChannel => + spawnLocalBridgeChannel(process.execPath, [script, 'stdio-bridge', String(echo.port)]) + +async function startProvider( + overrides: Partial[0]> = {} +): Promise { + const provider = new OrcadStdioBridgePortForwardProvider({ + resolveBridge: async () => bridge, + checkBridge: async () => 'running', + openChannel: async () => localBridge(), + ...overrides + }) + const forward = await provider.start(conn, { + id: 'pf-1', + connectionId: 'ssh-1', + localHost: '127.0.0.1', + localPort: 0, + remoteHost: '127.0.0.1', + remotePort: echo.port + }) + started.push(forward) + return forward +} + +function dial(port: number): Promise { + return new Promise((resolve, reject) => { + const socket = connect(port, '127.0.0.1', () => resolve(socket)) + socket.once('error', reject) + clients.push(socket) + }) +} + +function echoOf(socket: Socket, message: string): Promise { + return new Promise((resolve) => { + let received = '' + socket.on('data', (chunk: Buffer) => { + received += chunk.toString('utf8') + if (received.length >= message.length) { + resolve(received) + } + }) + socket.write(message) + }) +} + +const closedOf = (socket: Socket): Promise => + new Promise((resolve) => (socket.destroyed ? resolve() : socket.once('close', () => resolve()))) + +describe('the managed tunnel over the stdio bridge', () => { + it('serves the same local port a forward would, one bridge per connection', async () => { + const openChannel = vi.fn(async () => localBridge()) + const forward = await startProvider({ openChannel }) + expect(forward.entry.localPort).toBeGreaterThan(0) + const [first, second] = await Promise.all([ + dial(forward.entry.localPort), + dial(forward.entry.localPort) + ]) + expect(await echoOf(first, 'first')).toBe('first') + expect(await echoOf(second, 'second')).toBe('second') + expect(openChannel).toHaveBeenCalledTimes(2) + }) + + it('refuses to start where the bridge cannot run, so a deploy never registers it', async () => { + const failure = new OrcadStdioBridgeUnavailableError('exit 127') + await expect( + startProvider({ + checkBridge: async () => { + throw failure + } + }) + ).rejects.toBe(failure) + }) + + it('starts when the check only lost contact, resolving the bridge again per connection', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const resolveBridge = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('platform probe timed out')) + .mockResolvedValue(bridge) + const forward = await startProvider({ resolveBridge }) + const socket = await dial(forward.entry.localPort) + expect(await echoOf(socket, 'after')).toBe('after') + expect(resolveBridge).toHaveBeenCalledTimes(2) + warn.mockRestore() + }) + + it('drops only the socket when its channel is lost, and keeps listening', async () => { + const channels: PassThrough[] = [] + const openChannel = vi.fn(async () => { + const channel = Object.assign(new PassThrough(), { stderr: new PassThrough(), close() {} }) + channels.push(channel) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the provider uses only the stream, `stderr`, `close`, and events. + return channel as unknown as ClientChannel + }) + const forward = await startProvider({ openChannel }) + const lost = await dial(forward.entry.localPort) + await vi.waitFor(() => expect(channels).toHaveLength(1)) + channels[0].emit('close') + await closedOf(lost) + await dial(forward.entry.localPort) + await vi.waitFor(() => expect(channels).toHaveLength(2)) + }) + + it('caps concurrent bridges under sshd’s session limit and admits the next as one closes', async () => { + const channels: PassThrough[] = [] + const openChannel = vi.fn(async () => { + const channel = Object.assign(new PassThrough(), { + stderr: new PassThrough(), + close: () => channel.emit('close') + }) + channels.push(channel) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the provider uses only the stream, `stderr`, `close`, and events. + return channel as unknown as ClientChannel + }) + const forward = await startProvider({ openChannel }) + const sockets = await Promise.all( + Array.from({ length: ORCAD_STDIO_BRIDGE_MAX_CHANNELS + 1 }, () => + dial(forward.entry.localPort) + ) + ) + await vi.waitFor(() => + expect(openChannel).toHaveBeenCalledTimes(ORCAD_STDIO_BRIDGE_MAX_CHANNELS) + ) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(openChannel).toHaveBeenCalledTimes(ORCAD_STDIO_BRIDGE_MAX_CHANNELS) + sockets[0].destroy() + await vi.waitFor(() => + expect(openChannel).toHaveBeenCalledTimes(ORCAD_STDIO_BRIDGE_MAX_CHANNELS + 1) + ) + }) + + it('closes every bridge and connection when the tunnel closes', async () => { + const forward = await startProvider() + const socket = await dial(forward.entry.localPort) + expect(await echoOf(socket, 'ping')).toBe('ping') + await forward.close() + await closedOf(socket) + }) +}) diff --git a/src/main/ssh/orcad-stdio-bridge-provider.ts b/src/main/ssh/orcad-stdio-bridge-provider.ts new file mode 100644 index 00000000000..4c9437bf137 --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge-provider.ts @@ -0,0 +1,217 @@ +/** + * The managed tunnel's second transport, for hosts whose sshd refuses port forwarding: the same + * local listener the forward gives, but each accepted socket rides its own exec channel running + * the stdio bridge. Nothing above the tunnel can tell the two apart. + * + * Disconnects match the forward: a lost channel drops its socket and the listener stays, so the + * tunnel's own reconnect logic decides, and a lost transport never reads as a dead server. + */ +import { createServer, type Socket } from 'node:net' +import type { ClientChannel } from 'ssh2' +import { OrcadStdioBridgeUnavailableError } from './orcad-host-unavailable' +import { + checkOrcadStdioBridge, + openOrcadStdioBridgeChannel, + resolveOrcadStdioBridge, + type OrcadStdioBridge +} from './orcad-stdio-bridge' +import { + OrcadStdioBridgeBase64Encoder, + OrcadStdioBridgeDecoder, + type OrcadStdioBridgeSignal +} from './orcad-stdio-bridge-stream' +import type { SshConnection } from './ssh-connection' +import type { + PortForwardStartOptions, + SshPortForwardProvider, + StartedPortForward +} from './ssh-port-forward-provider' +import { listenForPortForward } from './ssh2-port-forward-provider' + +/** + * Each bridge is one sshd session, and OpenSSH's MaxSessions defaults to 10 per connection. The + * client holds one shared control socket and one per stream (terminals, each browser), so 8 + * covers it and leaves room for the exec work sharing the connection. Past it, sockets queue. + */ +export const ORCAD_STDIO_BRIDGE_MAX_CHANNELS = 8 +export const ORCAD_STDIO_BRIDGE_QUEUE_TIMEOUT_MS = 30_000 + +type OrcadStdioBridgeDependencies = { + resolveBridge: (conn: SshConnection, port: number) => Promise + checkBridge: (conn: SshConnection, bridge: OrcadStdioBridge) => Promise + openChannel: (conn: SshConnection, bridge: OrcadStdioBridge) => Promise +} + +const defaultDependencies: OrcadStdioBridgeDependencies = { + resolveBridge: resolveOrcadStdioBridge, + checkBridge: checkOrcadStdioBridge, + openChannel: openOrcadStdioBridgeChannel +} + +export class OrcadStdioBridgePortForwardProvider implements SshPortForwardProvider { + constructor(private readonly dependencies: OrcadStdioBridgeDependencies = defaultDependencies) {} + + // Why not system SSH: each bridge socket would be its own `ssh` process (a fresh login, and + // perhaps a key prompt, when multiplexing is off), and its refusal can't be probed up front. + canHandle(conn: SshConnection): boolean { + return conn.getClient() !== null + } + + async start(conn: SshConnection, options: PortForwardStartOptions): Promise { + const deps = this.dependencies + let bridge: Promise | null = null + // Memoized once it resolves; a failed resolve is retried by the next socket. + const resolveBridge = (): Promise => { + bridge ??= deps.resolveBridge(conn, options.remotePort).catch((error: unknown) => { + bridge = null + throw error + }) + return bridge + } + try { + await deps.checkBridge(conn, await resolveBridge()) + } catch (error) { + if (error instanceof OrcadStdioBridgeUnavailableError) { + throw error + } + // A slow or dropped check is loss of contact; sockets resolve the bridge again. + console.warn('[ssh] Orca stdio bridge check was unverifiable:', error) + } + + const sockets = new Set() + const channels = new Set() + const waiting: { socket: Socket; timer: ReturnType }[] = [] + let open = 0 + let closed = false + + const admitNext = (): void => { + while (open < ORCAD_STDIO_BRIDGE_MAX_CHANNELS) { + const next = waiting.shift() + if (!next) { + return + } + clearTimeout(next.timer) + void bridgeSocket(next.socket) + } + } + const release = (): void => { + open -= 1 + admitNext() + } + + const bridgeSocket = async (socket: Socket): Promise => { + if (closed || socket.destroyed) { + return + } + open += 1 + let resolved: OrcadStdioBridge + let channel: ClientChannel + try { + resolved = await resolveBridge() + channel = await deps.openChannel(conn, resolved) + } catch { + socket.destroy() + release() + return + } + channel.once('close', release) + if (closed || socket.destroyed) { + closeChannel(channel) + return + } + channels.add(channel) + channel.once('close', () => channels.delete(channel)) + pipeSocketThroughBridge(socket, channel, resolved.mode) + } + + const server = createServer((socket) => { + sockets.add(socket) + socket.on('error', () => socket.destroy()) + const entry = { + socket, + timer: setTimeout(() => socket.destroy(), ORCAD_STDIO_BRIDGE_QUEUE_TIMEOUT_MS) + } + socket.on('close', () => { + sockets.delete(socket) + clearTimeout(entry.timer) + const index = waiting.indexOf(entry) + if (index !== -1) { + waiting.splice(index, 1) + } + }) + waiting.push(entry) + admitNext() + }) + + await listenForPortForward(server, options.localHost, options.localPort) + const address = server.address() + const localPort = typeof address === 'object' && address ? address.port : options.localPort + const entry = { + id: options.id, + connectionId: options.connectionId, + localPort, + remoteHost: options.remoteHost, + remotePort: options.remotePort, + label: options.label + } + + const close = (): Promise => { + if (closed) { + return Promise.resolve() + } + closed = true + for (const { timer } of waiting.splice(0)) { + clearTimeout(timer) + } + for (const channel of channels) { + closeChannel(channel) + } + for (const socket of sockets) { + socket.destroy() + } + return new Promise((resolve) => server.close(() => resolve())) + } + return { entry, close, dispose: () => void close() } + } +} + +function pipeSocketThroughBridge( + socket: Socket, + channel: ClientChannel, + mode: OrcadStdioBridge['mode'] +): void { + const decoder = new OrcadStdioBridgeDecoder(mode) + // A refused dial is what the forward's failed channel open is: the socket just closes. + decoder.once('signal', (signal: OrcadStdioBridgeSignal) => { + if (signal !== 'ready') { + socket.destroy() + closeChannel(channel) + } + }) + decoder.on('error', () => { + socket.destroy() + closeChannel(channel) + }) + channel.pipe(decoder).pipe(socket) + if (mode === 'base64') { + socket.pipe(new OrcadStdioBridgeBase64Encoder()).pipe(channel) + } else { + socket.pipe(channel) + } + channel.stderr.resume() + channel.on('error', () => socket.destroy()) + channel.on('close', () => { + if (!socket.writableEnded) { + socket.destroy() + } + }) + socket.on('close', () => closeChannel(channel)) +} + +function closeChannel(channel: ClientChannel): void { + try { + channel.close() + } catch { + // Late callbacks after the transport dropped. + } +} diff --git a/src/main/ssh/orcad-stdio-bridge-script.ts b/src/main/ssh/orcad-stdio-bridge-script.ts new file mode 100644 index 00000000000..55b6a084caf --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge-script.ts @@ -0,0 +1,87 @@ +/** + * The host side of the stdio tunnel to managed orcad: run on the host's pinned Node over one SSH + * exec channel, it dials orcad's loopback port and pipes that socket to its own stdin/stdout. + * Hosts whose sshd refuses port forwarding (`AllowTcpForwarding no`) still allow exec, which is + * how every orcad host op already runs. + * + * A sentinel line comes first, so the client can skip whatever a login profile prints and tell a + * running bridge from a host where it could not start. `base64` mode frames each chunk as one + * ASCII line: a PowerShell DefaultShell re-decodes native output as text, which corrupts bytes. + */ +import { + ORCAD_NODE_RUNTIME_DIR_PREFIX, + ORCAD_NODE_RUNTIME_POSIX_EXECUTABLE, + ORCAD_RUNTIMES_DIRNAME +} from '../../shared/orcad-artifacts' +import { shellEscape } from './ssh-connection-utils' +import { RELAY_REMOTE_DIR } from './relay-protocol' + +export type OrcadStdioBridgeMode = 'raw' | 'base64' + +export const ORCAD_STDIO_BRIDGE_READY = 'ORCA-STDIO-BRIDGE READY' +/** The bridge ran but orcad's port refused it: a dead server, not a missing bridge. */ +export const ORCAD_STDIO_BRIDGE_REFUSED = 'ORCA-STDIO-BRIDGE REFUSED' +/** No pinned Node in the host's runtime store. */ +export const ORCAD_STDIO_BRIDGE_NO_NODE = 'ORCA-STDIO-BRIDGE NO_NODE' +/** Bounds a half-closed bridge whose orcad side never closes after the client's EOF. */ +const HALF_CLOSE_GRACE_MS = 30_000 + +const text = JSON.stringify + +/** `orcadStdioBridge(port, mode)`: shared by the POSIX `-e` script and the Windows host script. */ +export const ORCAD_STDIO_BRIDGE_FUNCTION = `function orcadStdioBridge(port, mode) { + const net = require('net') + const out = process.stdout + const exit = () => out.write('', () => process.exit(0)) + out.on('error', () => process.exit(0)) + let connected = false + const socket = net.connect({ host: '127.0.0.1', port }) + socket.on('error', (error) => { + if (!connected) out.write(${text(`${ORCAD_STDIO_BRIDGE_REFUSED} `)} + String(error.code || 'ERROR') + '\\n', () => process.exit(0)) + }) + socket.on('close', () => { if (connected) exit() }) + socket.on('connect', () => { + connected = true + out.write(${text(`${ORCAD_STDIO_BRIDGE_READY}\n`)}) + const input = process.stdin + input.on('end', () => { + socket.end() + setTimeout(() => socket.destroy(), ${HALF_CLOSE_GRACE_MS}).unref() + }) + if (mode !== 'base64') { + input.pipe(socket, { end: false }) + socket.pipe(out, { end: false }) + return + } + let pending = '' + input.setEncoding('latin1') + input.on('data', (chunk) => { + const lines = (pending + chunk).split('\\n') + pending = lines.pop() + for (const line of lines) { + const encoded = line.trim() + if (encoded && !socket.write(Buffer.from(encoded, 'base64'))) { + input.pause() + socket.once('drain', () => input.resume()) + } + } + }) + socket.on('data', (chunk) => { + if (!out.write(chunk.toString('base64') + '\\n')) { + socket.pause() + out.once('drain', () => socket.resume()) + } + }) + }) +}` + +/** Any verified runtime in the store serves; orcad is running on one of them. */ +export function orcadPosixStdioBridgeCommand(port: number): string { + const script = `${ORCAD_STDIO_BRIDGE_FUNCTION}\norcadStdioBridge(Number(process.argv[1]), 'raw')` + const runtimes = `"$HOME"/${shellEscape(`${RELAY_REMOTE_DIR}/${ORCAD_RUNTIMES_DIRNAME}`)}` + return [ + `for runtime in ${runtimes}/${ORCAD_NODE_RUNTIME_DIR_PREFIX}*/${ORCAD_NODE_RUNTIME_POSIX_EXECUTABLE}; do`, + `if [ -x "$runtime" ]; then exec "$runtime" -e ${shellEscape(script)} ${String(port)}; fi;`, + `done; echo ${shellEscape(ORCAD_STDIO_BRIDGE_NO_NODE)}; exit 127` + ].join(' ') +} diff --git a/src/main/ssh/orcad-stdio-bridge-stream.ts b/src/main/ssh/orcad-stdio-bridge-stream.ts new file mode 100644 index 00000000000..1eeb098b3c5 --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge-stream.ts @@ -0,0 +1,100 @@ +/** The client side of one stdio bridge channel: its sentinel, then orcad's bytes. */ +import { Transform, type TransformCallback } from 'node:stream' +import { + ORCAD_STDIO_BRIDGE_NO_NODE, + ORCAD_STDIO_BRIDGE_READY, + ORCAD_STDIO_BRIDGE_REFUSED, + type OrcadStdioBridgeMode +} from './orcad-stdio-bridge-script' + +export type OrcadStdioBridgeSignal = 'ready' | 'refused' | 'no-node' + +// What a login profile may print before the bridge starts; past this, it is not a bridge. +const MAX_PRELUDE_BYTES = 64 * 1024 +const NEWLINE = 0x0a + +function sentinelOf(line: string): OrcadStdioBridgeSignal | null { + const trimmed = line.trim() + if (trimmed === ORCAD_STDIO_BRIDGE_READY) { + return 'ready' + } + if (trimmed.startsWith(ORCAD_STDIO_BRIDGE_REFUSED)) { + return 'refused' + } + return trimmed === ORCAD_STDIO_BRIDGE_NO_NODE ? 'no-node' : null +} + +/** + * Bridge stdout in, orcad's bytes out. Emits `signal` once with the sentinel it found; nothing + * passes before `ready`, and nothing at all after `refused` or `no-node`. + */ +export class OrcadStdioBridgeDecoder extends Transform { + private prelude: Buffer = Buffer.alloc(0) + private signal: OrcadStdioBridgeSignal | null = null + private pendingLine = '' + + constructor(private readonly mode: OrcadStdioBridgeMode) { + super() + } + + get bridgeSignal(): OrcadStdioBridgeSignal | null { + return this.signal + } + + override _transform(chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback): void { + if (this.signal === 'ready') { + this.forward(chunk) + done() + return + } + if (this.signal) { + done() + return + } + this.prelude = Buffer.concat([this.prelude, chunk]) + let newline = this.prelude.indexOf(NEWLINE) + while (newline !== -1) { + const signal = sentinelOf(this.prelude.subarray(0, newline).toString('latin1')) + this.prelude = this.prelude.subarray(newline + 1) + if (signal) { + this.signal = signal + this.emit('signal', signal) + const rest = this.prelude + this.prelude = Buffer.alloc(0) + if (signal === 'ready' && rest.length > 0) { + this.forward(rest) + } + done() + return + } + newline = this.prelude.indexOf(NEWLINE) + } + done( + this.prelude.length > MAX_PRELUDE_BYTES + ? new Error('The SSH host printed no stdio bridge sentinel.') + : undefined + ) + } + + private forward(chunk: Buffer): void { + if (this.mode === 'raw') { + this.push(chunk) + return + } + const lines = (this.pendingLine + chunk.toString('latin1')).split('\n') + this.pendingLine = lines.pop() ?? '' + for (const line of lines) { + const encoded = line.trim() + if (encoded) { + this.push(Buffer.from(encoded, 'base64')) + } + } + } +} + +/** Client bytes to one base64 line per chunk, the framing the Windows bridge reads. */ +export class OrcadStdioBridgeBase64Encoder extends Transform { + override _transform(chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback): void { + done(null, `${chunk.toString('base64')}\n`) + } +} diff --git a/src/main/ssh/orcad-stdio-bridge-test-channel.ts b/src/main/ssh/orcad-stdio-bridge-test-channel.ts new file mode 100644 index 00000000000..4fc4b5fe87b --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge-test-channel.ts @@ -0,0 +1,52 @@ +/** Test-only: a bridge run as a local child, shaped like the SSH exec channel that runs it. */ +import { createServer, type Server } from 'node:net' +import { Duplex } from 'node:stream' +import type { ClientChannel } from 'ssh2' +import { spawnProcess } from '../../shared/child-process/run-process' + +export function spawnLocalBridgeChannel( + program: string, + args: readonly string[], + env: NodeJS.ProcessEnv = process.env +): ClientChannel { + const child = spawnProcess({ program, args, env }) + // Why emitClose off: like ssh2, `close` comes once, after the exit status. + const channel = new Duplex({ + emitClose: false, + read() { + child.stdout.resume() + }, + write(chunk: Buffer, _encoding, done) { + child.stdin.write(chunk, done) + }, + final(done) { + child.stdin.end(done) + } + }) + child.stdout.on('data', (chunk: Buffer) => { + if (!channel.push(chunk)) { + child.stdout.pause() + } + }) + child.stdout.on('end', () => channel.push(null)) + child.stdin.on('error', () => {}) + child.on('exit', (code) => channel.emit('exit', code)) + child.on('close', () => channel.emit('close')) + Object.assign(channel, { stderr: child.stderr, close: () => child.kill() }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the bridge paths use only the stream, `stderr`, `close`, and the `exit`/`close` events assigned above. + return channel as unknown as ClientChannel +} + +/** A loopback echo server standing in for orcad. */ +export async function startEchoServer(): Promise<{ server: Server; port: number }> { + const server = createServer((socket) => { + socket.on('error', () => socket.destroy()) + socket.pipe(socket) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('echo server has no port') + } + return { server, port: address.port } +} diff --git a/src/main/ssh/orcad-stdio-bridge.test.ts b/src/main/ssh/orcad-stdio-bridge.test.ts new file mode 100644 index 00000000000..b3db5ef0d94 --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge.test.ts @@ -0,0 +1,233 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { Server } from 'node:net' +import type { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import type { ClientChannel } from 'ssh2' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { OrcadStdioBridgeUnavailableError } from './orcad-host-unavailable' +import { checkOrcadStdioBridge, type OrcadStdioBridge } from './orcad-stdio-bridge' +import { + orcadPosixStdioBridgeCommand, + type OrcadStdioBridgeMode +} from './orcad-stdio-bridge-script' +import { OrcadStdioBridgeBase64Encoder, OrcadStdioBridgeDecoder } from './orcad-stdio-bridge-stream' +import { spawnLocalBridgeChannel, startEchoServer } from './orcad-stdio-bridge-test-channel' +import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script' +import type { SshConnection } from './ssh-connection' + +const POSIX = process.platform !== 'win32' +let echo: { server: Server; port: number } +let scratch: string + +beforeAll(async () => { + echo = await startEchoServer() + scratch = mkdtempSync(join(tmpdir(), 'orcad-stdio-bridge-')) +}) + +afterAll(() => { + echo.server.close() + rmSync(scratch, { recursive: true, force: true }) +}) + +/** Every byte value, so a text layer anywhere in the path would show. */ +const PAYLOAD = Buffer.from(Array.from({ length: 4096 }, (_, index) => index % 256)) + +async function roundTrip(channel: ClientChannel, mode: OrcadStdioBridgeMode): Promise { + const decoder = new OrcadStdioBridgeDecoder(mode) + channel.pipe(decoder) + const input = mode === 'base64' ? new OrcadStdioBridgeBase64Encoder() : new PassThrough() + input.pipe(channel) + input.write(PAYLOAD) + const received: Buffer[] = [] + let length = 0 + await new Promise((resolve, reject) => { + decoder.on('error', reject) + decoder.on('data', (chunk: Buffer) => { + received.push(chunk) + length += chunk.length + if (length >= PAYLOAD.length) { + resolve() + } + }) + }) + input.end() + await new Promise((resolve) => channel.once('close', resolve)) + return Buffer.concat(received) +} + +function posixHome(withRuntime: boolean): string { + const home = mkdtempSync(join(scratch, 'home-')) + if (withRuntime) { + const bin = join(home, '.orca-remote', 'runtimes', 'node-abc', 'bin') + mkdirSync(bin, { recursive: true }) + symlinkSync(process.execPath, join(bin, 'node')) + } + return home +} + +function connectionRunning(channel: () => ClientChannel): SshConnection { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the check calls only exec. + return { exec: async () => channel() } as unknown as SshConnection +} + +const posixBridge = (port: number): OrcadStdioBridge => ({ + command: orcadPosixStdioBridgeCommand(port), + mode: 'raw', + wrapCommand: true +}) + +describe.runIf(POSIX)('the POSIX stdio bridge command', () => { + it('pipes every byte to orcad and back on the store’s pinned Node', async () => { + const home = posixHome(true) + const channel = spawnLocalBridgeChannel( + '/bin/sh', + ['-c', orcadPosixStdioBridgeCommand(echo.port)], + { + ...process.env, + HOME: home + } + ) + expect((await roundTrip(channel, 'raw')).equals(PAYLOAD)).toBe(true) + }) + + it('proves the bridge runs even when orcad refuses the dial', async () => { + const closed = await startEchoServer() + closed.server.close() + const home = posixHome(true) + const conn = connectionRunning(() => + spawnLocalBridgeChannel('/bin/sh', ['-c', posixBridge(closed.port).command], { + ...process.env, + HOME: home + }) + ) + await expect(checkOrcadStdioBridge(conn, posixBridge(closed.port))).resolves.toBe('running') + }) + + it('reports a host with no pinned Node as one the bridge cannot run on', async () => { + const home = posixHome(false) + const conn = connectionRunning(() => + spawnLocalBridgeChannel('/bin/sh', ['-c', posixBridge(echo.port).command], { + ...process.env, + HOME: home + }) + ) + await expect(checkOrcadStdioBridge(conn, posixBridge(echo.port))).rejects.toBeInstanceOf( + OrcadStdioBridgeUnavailableError + ) + }) +}) + +describe('the Windows host script’s stdio-bridge op', () => { + it('frames every byte as base64 lines both ways', async () => { + const script = join(scratch, 'orcad-host-script.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const channel = spawnLocalBridgeChannel(process.execPath, [ + script, + 'stdio-bridge', + String(echo.port) + ]) + expect((await roundTrip(channel, 'base64')).equals(PAYLOAD)).toBe(true) + }) +}) + +describe('reading a bridge channel', () => { + it('skips what a login profile prints before the sentinel, and keeps bytes after it', async () => { + const decoder = new OrcadStdioBridgeDecoder('raw') + const signals: string[] = [] + decoder.on('signal', (signal: string) => signals.push(signal)) + const out: Buffer[] = [] + decoder.on('data', (chunk: Buffer) => out.push(chunk)) + decoder.write(Buffer.from('Welcome to box\nORCA-STDIO-BRIDGE READY\n\x00\x01')) + decoder.end(Buffer.from([0xff])) + await new Promise((resolve) => decoder.on('end', resolve)) + expect(signals).toEqual(['ready']) + expect(Buffer.concat(out)).toEqual(Buffer.from([0, 1, 0xff])) + }) + + it('decodes base64 lines a PowerShell host rewrote with CRLF', async () => { + const decoder = new OrcadStdioBridgeDecoder('base64') + const out: Buffer[] = [] + decoder.on('data', (chunk: Buffer) => out.push(chunk)) + decoder.end(Buffer.from('ORCA-STDIO-BRIDGE READY\r\nAAE=\r\n\r\n/w==\r\n')) + await new Promise((resolve) => decoder.on('end', resolve)) + expect(Buffer.concat(out)).toEqual(Buffer.from([0, 1, 0xff])) + }) + + it('fails a channel that prints no sentinel within its bound', async () => { + const decoder = new OrcadStdioBridgeDecoder('raw') + const failed = new Promise((resolve) => decoder.on('error', resolve)) + decoder.write(Buffer.alloc(70 * 1024, 0x61)) + await expect(failed).resolves.toBeInstanceOf(Error) + }) +}) + +/** A channel the test drives by hand. */ +function scriptedChannel(): ClientChannel & EventEmitter & { stderr: PassThrough } { + const channel = Object.assign(new PassThrough(), { + stderr: new PassThrough(), + close: () => {} + }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the check uses only the stream, `stderr`, `close`, and events. + return channel as unknown as ClientChannel & EventEmitter & { stderr: PassThrough } +} + +describe('checking a bridge before the tunnel relies on it', () => { + it('reads a channel lost without an exit status as unverifiable, never as unavailable', async () => { + const channel = scriptedChannel() + const check = checkOrcadStdioBridge( + connectionRunning(() => channel), + posixBridge(1) + ) + await new Promise((resolve) => setImmediate(resolve)) + channel.emit('close') + await expect(check).resolves.toBe('unverifiable') + }) + + it('reads a silent bridge as unverifiable once the check times out', async () => { + const channel = scriptedChannel() + await expect( + checkOrcadStdioBridge( + connectionRunning(() => channel), + posixBridge(1), + 20 + ) + ).resolves.toBe('unverifiable') + }) + + it('reads a reported exit with no sentinel as a host the bridge cannot run on', async () => { + const channel = scriptedChannel() + const check = checkOrcadStdioBridge( + connectionRunning(() => channel), + posixBridge(1) + ) + await new Promise((resolve) => setImmediate(resolve)) + channel.stderr.write('node.exe is not recognized') + channel.emit('exit', 9009) + channel.emit('close') + await expect(check).rejects.toThrow(/exit 9009: node.exe is not recognized/u) + }) + + it('reads a script cut short with no sentinel as unverifiable, not a permanent refusal', async () => { + const channel = scriptedChannel() + const check = checkOrcadStdioBridge( + connectionRunning(() => channel), + posixBridge(1) + ) + await new Promise((resolve) => setImmediate(resolve)) + channel.emit('exit', 1) + channel.emit('close') + await expect(check).resolves.toBe('unverifiable') + }) + + it('reads an exec channel sshd would not open as unverifiable', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the check calls only exec. + const conn = { + exec: async () => { + throw new Error('open failed') + } + } as unknown as SshConnection + await expect(checkOrcadStdioBridge(conn, posixBridge(1))).resolves.toBe('unverifiable') + }) +}) diff --git a/src/main/ssh/orcad-stdio-bridge.ts b/src/main/ssh/orcad-stdio-bridge.ts new file mode 100644 index 00000000000..f68088121b5 --- /dev/null +++ b/src/main/ssh/orcad-stdio-bridge.ts @@ -0,0 +1,131 @@ +/** Which command runs the stdio bridge on a host, and whether it can run there at all. */ +import type { ClientChannel } from 'ssh2' +import { + OrcadHostUnsupportedError, + OrcadStdioBridgeUnavailableError +} from './orcad-host-unavailable' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { orcadRemoteBaseDir, orcadWindowsHostOpCommand } from './orcad-remote-windows-node' +import { + orcadPosixStdioBridgeCommand, + type OrcadStdioBridgeMode +} from './orcad-stdio-bridge-script' +import { OrcadStdioBridgeDecoder, type OrcadStdioBridgeSignal } from './orcad-stdio-bridge-stream' +import type { SshConnection } from './ssh-connection' +import { isWindowsRemoteHost } from './ssh-remote-platform' +import { detectRemoteHostPlatform } from './ssh-remote-platform-detection' + +export type OrcadStdioBridge = { + command: string + mode: OrcadStdioBridgeMode + wrapCommand: boolean +} + +export const ORCAD_STDIO_BRIDGE_CHECK_TIMEOUT_MS = 20_000 +const MAX_STDERR_TAIL_CHARS = 2_000 +// POSIX shells and cmd.exe report a command they cannot find with these. +const COMMAND_NOT_FOUND_EXITS: ReadonlySet = new Set([127, 9009]) + +export async function resolveOrcadStdioBridge( + conn: SshConnection, + port: number +): Promise { + const host = await detectRemoteHostPlatform(conn) + if (!host) { + throw new OrcadHostUnsupportedError('This SSH host platform is not supported by managed orcad.') + } + if (!isWindowsRemoteHost(host)) { + return { command: orcadPosixStdioBridgeCommand(port), mode: 'raw', wrapCommand: true } + } + // Stages this client's pinned node.exe and host script, which an app update may have changed. + const context = await resolveOrcadRemoteContext(conn.getTarget(), conn, undefined, host) + return { + command: orcadWindowsHostOpCommand( + host, + orcadRemoteBaseDir(host, context.remoteHome), + 'stdio-bridge', + [String(port)] + ), + mode: 'base64', + wrapCommand: host.commandDialect !== 'powershell' + } +} + +export function openOrcadStdioBridgeChannel( + conn: SshConnection, + bridge: OrcadStdioBridge +): Promise { + return conn.exec(bridge.command, { wrapCommand: bridge.wrapCommand }) +} + +function closeQuietly(channel: ClientChannel): void { + try { + channel.close() + } catch { + // Already closed. + } +} + +/** + * Runs the bridge once. Any sentinel proves it runs, even when orcad's port refused it. Only an + * exit the host reported without one is proof it can't run; anything else is unverifiable. + */ +export async function checkOrcadStdioBridge( + conn: SshConnection, + bridge: OrcadStdioBridge, + timeoutMs = ORCAD_STDIO_BRIDGE_CHECK_TIMEOUT_MS +): Promise<'running' | 'unverifiable'> { + let channel: ClientChannel + try { + channel = await openOrcadStdioBridgeChannel(conn, bridge) + } catch { + return 'unverifiable' + } + const decoder = new OrcadStdioBridgeDecoder(bridge.mode) + let stderr = '' + let exitCode: number | null = null + return new Promise((resolve, reject) => { + let settled = false + const timer = setTimeout(() => settle('unverifiable'), timeoutMs) + function settle(verdict: 'running' | 'unverifiable' | Error): void { + if (settled) { + return + } + settled = true + clearTimeout(timer) + closeQuietly(channel) + decoder.destroy() + if (verdict instanceof Error) { + reject(verdict) + } else { + resolve(verdict) + } + } + const unavailable = (detail: string): Error => + new OrcadStdioBridgeUnavailableError( + `The SSH host could not run the managed Orca server's stdio bridge: ${detail}` + ) + decoder.on('signal', (signal: OrcadStdioBridgeSignal) => { + settle(signal === 'no-node' ? unavailable('no pinned Node runtime') : 'running') + }) + decoder.on('error', () => settle('unverifiable')) + decoder.resume() + channel.pipe(decoder) + channel.stderr.on('data', (chunk: Buffer) => { + stderr = (stderr + chunk.toString('utf8')).slice(-MAX_STDERR_TAIL_CHARS) + }) + channel.on('exit', (code: unknown) => { + exitCode = typeof code === 'number' ? code : null + }) + channel.on('close', () => { + // Why only command-not-found: a lost channel, or a script cut short (a concurrent host-script + // write, a killed node.exe), proves nothing permanent about the host. + settle( + COMMAND_NOT_FOUND_EXITS.has(exitCode) + ? unavailable(`exit ${exitCode}${stderr.trim() ? `: ${stderr.trim()}` : ''}`) + : 'unverifiable' + ) + }) + channel.on('error', () => settle('unverifiable')) + }) +} diff --git a/src/main/ssh/orcad-terminal-census-client.test.ts b/src/main/ssh/orcad-terminal-census-client.test.ts new file mode 100644 index 00000000000..8ed563ad120 --- /dev/null +++ b/src/main/ssh/orcad-terminal-census-client.test.ts @@ -0,0 +1,131 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY } from '../../shared/orcad-runtime-capabilities' +import { + createEnvironmentFromPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { emptyOrcadActivationRecord, type OrcadActivationRecord } from './orcad-activation-record' + +const mocks = vi.hoisted(() => ({ send: vi.fn(), ensure: vi.fn(), verify: vi.fn() })) +vi.mock('../../shared/remote-runtime-client', () => ({ + sendRemoteRuntimeRequestWithStatusPreflight: mocks.send +})) +vi.mock('./orcad-managed-tunnel', () => ({ ensureOrcadManagedTunnel: mocks.ensure })) +vi.mock('./orcad-managed-serving-verify', () => ({ verifyOrcadManagedServing: mocks.verify })) + +const { collectManagedTerminalCensus } = await import('./orcad-terminal-census-client') +const collect = (record: OrcadActivationRecord) => + collectManagedTerminalCensus('/profile', environment, record) + +const environment: KnownRuntimeEnvironment = createEnvironmentFromPairingOffer({ + id: 'environment-1', + name: 'Managed', + now: 1, + offer: { v: 2, endpoint: 'ws://127.0.0.1:46768', deviceToken: 't', publicKeyB64: 'k' }, + connectionDependency: 'ssh-tunnel' +}) +const active = { + ...emptyOrcadActivationRecord(), + active: '1.0.0', + activatedAt: '2026-01-01T00:00:00.000Z' +} +const unverifiable = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null +} +const census = { liveSessions: 2, startedSinceActivation: 1, daemonProtocolVersion: 7 } + +function answerWith(capabilities: string[], response: unknown) { + mocks.send.mockImplementation(async (_pairing, _method, _params, _timeout, validate) => { + validate({ ok: true, result: { capabilities } }) + return response + }) +} + +describe('managed orcad terminal census client', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.ensure.mockResolvedValue(undefined) + mocks.verify.mockResolvedValue({ state: 'serving' }) + }) + + it('reads an idle census without contacting a host that has nothing active', async () => { + await expect(collect({ ...active, active: null })).resolves.toEqual({ + liveSessions: 0, + startedSinceActivation: 0, + daemonProtocolVersion: null + }) + expect(mocks.send).not.toHaveBeenCalled() + }) + + it('starts a server that idled out behind a live forward before asking it', async () => { + const order: string[] = [] + mocks.verify.mockImplementation(async () => { + order.push('wake') + return { state: 'started', boundPort: null } + }) + mocks.send.mockImplementation(async (_pairing, _method, _params, _timeout, validate) => { + order.push('census') + validate({ ok: true, result: { capabilities: [ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY] } }) + return { ok: true, result: census } + }) + await expect(collect(active)).resolves.toEqual(census) + expect(order).toEqual(['wake', 'census']) + }) + + it('asks an advertising host with the activation time', async () => { + answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { ok: true, result: census }) + await expect(collect(active)).resolves.toEqual(census) + expect(mocks.send.mock.calls[0]?.slice(1, 3)).toEqual([ + 'orcad.terminalCensus', + { activatedAt: Date.parse(active.activatedAt) } + ]) + }) + + it.each([ + ['an older host without the capability', () => answerWith([], { ok: true, result: census })], + [ + 'method not found', + () => + answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { + ok: false, + error: { code: 'method_not_found', message: 'no' } + }) + ], + [ + 'a malformed answer', + () => answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { ok: true, result: {} }) + ], + ['loss of contact', () => mocks.send.mockRejectedValue(new Error('socket closed'))] + ])('reads %s as unverifiable, never as zero', async (_label, arrange) => { + arrange() + await expect(collect(active)).resolves.toEqual(unverifiable) + }) + + it('reads an unparseable activation time as unverifiable', async () => { + await expect(collect({ ...active, activatedAt: 'later' })).resolves.toEqual(unverifiable) + expect(mocks.send).not.toHaveBeenCalled() + }) + + it('reads a tunnel that cannot open as unverifiable', async () => { + mocks.ensure.mockRejectedValue(new Error('SSH unavailable')) + await expect(collectManagedTerminalCensus('/profile', environment, active)).resolves.toEqual( + unverifiable + ) + }) + + it('asks the server to release finished automation shells only when an update needs it', async () => { + answerWith([ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY], { ok: true, result: census }) + await collect(active) + expect(mocks.send.mock.calls[0]?.[2]).toEqual({ activatedAt: Date.parse(active.activatedAt) }) + + await collectManagedTerminalCensus('/profile', environment, active, undefined, { + releaseFinishedAutomationTerminals: true + }) + expect(mocks.send.mock.calls[1]?.[2]).toEqual({ + activatedAt: Date.parse(active.activatedAt), + releaseFinishedAutomationTerminals: true + }) + }) +}) diff --git a/src/main/ssh/orcad-terminal-census-client.ts b/src/main/ssh/orcad-terminal-census-client.ts new file mode 100644 index 00000000000..6b8f9f6cb61 --- /dev/null +++ b/src/main/ssh/orcad-terminal-census-client.ts @@ -0,0 +1,74 @@ +/** + * Asks a managed orcad, through its tunnel, how many terminals its daemon runs. Every failure, + * including an older host without the method, reads as an unverifiable census, never as zero. + */ +import { ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY } from '../../shared/orcad-runtime-capabilities' +import { ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES } from '../../shared/electron-remote-runtime-client-capabilities' +import { + getPreferredPairingOffer, + type KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import { + ORCAD_TERMINAL_CENSUS_METHOD, + OrcadTerminalCensusSchema, + type OrcadTerminalCensus +} from '../../shared/orcad-terminal-census' +import { sendRemoteRuntimeRequestWithStatusPreflight } from '../../shared/remote-runtime-client' +import type { OrcadActivationRecord } from './orcad-activation-record' +import { ensureOrcadManagedTunnel } from './orcad-managed-tunnel' +import { verifyOrcadManagedServing } from './orcad-managed-serving-verify' + +const UNVERIFIABLE: OrcadTerminalCensus = { + liveSessions: null, + startedSinceActivation: null, + daemonProtocolVersion: null +} + +/** + * The census through the server's ensured tunnel, after starting a server that idled out; a + * tunnel that cannot open, or a server that cannot start, is unverifiable. + */ +export async function collectManagedTerminalCensus( + userDataPath: string, + environment: KnownRuntimeEnvironment, + record: OrcadActivationRecord, + timeoutMs = 15_000, + options: { releaseFinishedAutomationTerminals?: boolean } = {} +): Promise { + if (!record.active) { + return { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null } + } + const activatedAt = record.activatedAt ? Date.parse(record.activatedAt) : Number.NaN + if (!Number.isFinite(activatedAt) || activatedAt < 0) { + return UNVERIFIABLE + } + try { + await ensureOrcadManagedTunnel(userDataPath, environment.id) + // A server that idled out behind a forward still up is started first, or it never answers. + await verifyOrcadManagedServing(userDataPath, environment.id) + const response = await sendRemoteRuntimeRequestWithStatusPreflight( + getPreferredPairingOffer(environment), + ORCAD_TERMINAL_CENSUS_METHOD, + { + activatedAt, + ...(options.releaseFinishedAutomationTerminals + ? { releaseFinishedAutomationTerminals: true } + : {}) + }, + timeoutMs, + (status) => { + if ( + !status.ok || + !status.result.capabilities?.includes(ORCAD_TERMINAL_CENSUS_RUNTIME_CAPABILITY) + ) { + throw new Error('The managed Orca server does not report a terminal census.') + } + }, + undefined, + ELECTRON_REMOTE_RUNTIME_CLIENT_CAPABILITIES + ) + return response.ok ? OrcadTerminalCensusSchema.parse(response.result) : UNVERIFIABLE + } catch { + return UNVERIFIABLE + } +} diff --git a/src/main/ssh/orcad-transaction-incumbent.ts b/src/main/ssh/orcad-transaction-incumbent.ts new file mode 100644 index 00000000000..58de1500a79 --- /dev/null +++ b/src/main/ssh/orcad-transaction-incumbent.ts @@ -0,0 +1,114 @@ +/** The incumbent slot across a journaled activation or rollback: stop it, then put it back. */ +import type { OrcadActivationLockControl } from './orcad-activation-lock' +import type { OrcadSnapshotVerdict } from './orcad-activation-transaction' +import { orcadStopFreedTheHost } from './orcad-remote-process-control' +import { withoutAbortSignal } from './orcad-remote-runtime-control' +import { + launchOrcadSlot, + stopOrcadSlot, + ORCAD_SLOT_STOP_WAIT_SECONDS, + type OrcadSlotIdentity, + type OrcadSlotOptions +} from './orcad-recovery-slot' +import { recoverOrcadIncumbent } from './orcad-incumbent-recovery' +import { errorMessage } from '../../shared/error-message' + +/** Null once the incumbent provably exited; otherwise why not, with the fence kept if it may still change. */ +export async function stopTransactionIncumbent( + options: OrcadSlotOptions, + incumbent: OrcadSlotIdentity, + lock: OrcadActivationLockControl +): Promise { + const stopped = await stopOrcadSlot(options, incumbent.remoteDir, false) + if (orcadStopFreedTheHost(stopped)) { + return null + } + // Only a stop that may have been delivered can still change the host; otherwise nothing happened. + if (stopped === 'still-running' || stopped === 'unconfirmed') { + lock.retain() + } + return ( + `Could not verify that orcad ${incumbent.version} exited within ` + + `${ORCAD_SLOT_STOP_WAIT_SECONDS}s (${stopped}).` + ) +} + +export async function restartAfterSnapshotFailure( + options: OrcadSlotOptions, + incumbent: OrcadSlotIdentity, + lock: OrcadActivationLockControl +): Promise { + try { + await launchOrcadSlot(withoutAbortSignal(options), incumbent) + lock.recovered() + return `orcad ${incumbent.version} was restarted and is serving again.` + } catch (error) { + lock.retain() + return `restarting orcad ${incumbent.version} failed: ${errorMessage(error)} This host requires recovery.` + } +} + +/** + * Restores `restoreState` when it was replaced, then restarts the incumbent; both slots are + * already proven exited. Null on success; otherwise why not, with the fence kept. + */ +export async function putTransactionIncumbentBack( + options: OrcadSlotOptions, + lock: OrcadActivationLockControl, + input: { + transactionStartedAt: string + launchedVersion: string | null + incumbent: OrcadSlotIdentity | null + restoreState: OrcadSnapshotVerdict | null + launchedFromState?: OrcadSnapshotVerdict | null + } +): Promise { + try { + const recovery = await recoverOrcadIncumbent(withoutAbortSignal(options), { + ...input, + slotsProvenExited: true + }) + if (recovery.outcome === 'refused') { + lock.retain() + return recovery.reason + } + lock.recovered() + return null + } catch (error) { + lock.retain() + return `Restoring the previous version failed: ${errorMessage(error)} This host requires recovery.` + } +} + +/** Stop the build this run launched, then put the incumbent back only on safe state. */ +export async function restoreAfterRejectedCandidate( + options: OrcadSlotOptions, + lock: OrcadActivationLockControl, + input: { + launchedDir: string + launchedVersion: string + transactionStartedAt: string + incumbent: OrcadSlotIdentity | null + restoreState: OrcadSnapshotVerdict | null + launchedFromState?: OrcadSnapshotVerdict | null + } +): Promise { + const { launchedDir, ...restore } = input + const stopped = await stopOrcadSlot(withoutAbortSignal(options), launchedDir, true).catch( + (error: unknown) => `unverifiable: ${errorMessage(error)}` + ) + if (stopped !== 'stopped' && stopped !== 'already-exited') { + lock.retain() + return ( + `orcad ${input.launchedVersion} could not be confirmed stopped (${stopped}), so nothing ` + + 'was restored over state it may still own. This host requires recovery.' + ) + } + const failure = await putTransactionIncumbentBack(options, lock, restore) + return ( + failure ?? + (input.incumbent + ? `orcad ${input.incumbent.version} was restarted and is serving again.` + : 'No previous version was active, so this host is now serving nothing.') + ) +} diff --git a/src/main/ssh/orcad-tunnel-transport-memo.ts b/src/main/ssh/orcad-tunnel-transport-memo.ts new file mode 100644 index 00000000000..529146c9f59 --- /dev/null +++ b/src/main/ssh/orcad-tunnel-transport-memo.ts @@ -0,0 +1,20 @@ +/** Which transport each host's managed tunnel last started on, for connect telemetry. */ +export type OrcadTunnelTransport = 'tcp_forward' | 'stdio_bridge' + +// Keyed by the local target id, which never leaves this process. +const transports = new Map() + +export function rememberOrcadTunnelTransport( + targetId: string, + transport: OrcadTunnelTransport +): void { + transports.set(targetId, transport) +} + +export function knownOrcadTunnelTransport(targetId: string): OrcadTunnelTransport | null { + return transports.get(targetId) ?? null +} + +export function resetOrcadTunnelTransportMemoForTests(): void { + transports.clear() +} diff --git a/src/main/ssh/orcad-tunneled-pairing.test.ts b/src/main/ssh/orcad-tunneled-pairing.test.ts new file mode 100644 index 00000000000..ee49783ab62 --- /dev/null +++ b/src/main/ssh/orcad-tunneled-pairing.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { encodePairingOffer, parsePairingCode, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import type { ServeReadiness } from '../server/serve-readiness' +import { tunneledOrcadPairingCode } from './orcad-tunneled-pairing' + +function readiness(endpoint = 'ws://[::1]:6768/runtime?mode=paired#fragment'): ServeReadiness { + return { + runtimeId: 'runtime-1', + boundEndpoint: 'ws://127.0.0.1:6768', + advertisedEndpoint: null, + managedWslCliReconciliation: 'settled', + pairing: { + available: true, + url: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint, + deviceToken: 'device-token', + publicKeyB64: 'public-key', + pairedDeviceId: 'device-1' + }), + endpoint, + deviceId: 'device-1', + webClientUrl: null, + scope: 'runtime', + qr: null + } + } +} + +describe('tunneledOrcadPairingCode', () => { + it('rewrites only the endpoint authority for the local tunnel', () => { + const rewritten = parsePairingCode(tunneledOrcadPairingCode(readiness(), 46_768)) + + expect(rewritten).toEqual({ + v: PAIRING_OFFER_VERSION, + endpoint: 'ws://127.0.0.1:46768/runtime?mode=paired#fragment', + deviceToken: 'device-token', + publicKeyB64: 'public-key', + pairedDeviceId: 'device-1' + }) + }) + + it('refuses a non-loopback offer', () => { + expect(() => tunneledOrcadPairingCode(readiness('wss://runtime.example.com'), 46_768)).toThrow( + 'not loopback-only' + ) + }) +}) diff --git a/src/main/ssh/orcad-tunneled-pairing.ts b/src/main/ssh/orcad-tunneled-pairing.ts new file mode 100644 index 00000000000..53432181884 --- /dev/null +++ b/src/main/ssh/orcad-tunneled-pairing.ts @@ -0,0 +1,25 @@ +import { encodePairingOffer, parsePairingCode } from '../../shared/pairing' +import { classifyRemotePairingHostname } from '../../shared/remote-pairing-address' +import type { ServeReadiness } from '../server/serve-readiness' + +export function tunneledOrcadPairingCode(readiness: ServeReadiness, localPort: number): string { + if (!readiness.pairing.available) { + throw new Error( + `The managed Orca server did not publish a pairing offer: ${readiness.pairing.guidance}` + ) + } + const offer = parsePairingCode(readiness.pairing.url) + if (!offer) { + throw new Error('The managed Orca server published an invalid pairing offer.') + } + const endpoint = new URL(offer.endpoint) + if ( + (endpoint.protocol !== 'ws:' && endpoint.protocol !== 'wss:') || + classifyRemotePairingHostname(endpoint.hostname) !== 'loopback' + ) { + throw new Error('The managed Orca server pairing endpoint is not loopback-only.') + } + endpoint.hostname = '127.0.0.1' + endpoint.port = String(localPort) + return encodePairingOffer({ ...offer, endpoint: endpoint.toString() }) +} diff --git a/src/main/ssh/orcad-unreachable-setup-release.test.ts b/src/main/ssh/orcad-unreachable-setup-release.test.ts new file mode 100644 index 00000000000..b4aedd774a0 --- /dev/null +++ b/src/main/ssh/orcad-unreachable-setup-release.test.ts @@ -0,0 +1,110 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { getManagedOrcadFenceEnvironmentId } from '../../shared/managed-orcad-ssh-owner' +import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../../shared/pairing' +import { addManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { listEnvironments } from '../../shared/runtime-environment-store' +import type { SshTarget } from '../../shared/ssh-types' +import { closeTestStores, createSqliteTestStore } from '../persistence-test-harness' +import { Store } from '../persistence/loading-store/store' +import { stageOrcadMigrationDestination } from './orcad-migration-cutover-coordinator' +import { listOrcadMigrationSourceCutovers } from './orcad-migration-cutover-journal' +import { fakeOrcadMigrationDestination } from './orcad-migration-destination-fake' +import { fenceOrcadMigrationSource } from './orcad-migration-source-fence' +import { releaseUnreachableOrcadSetup } from './orcad-unreachable-setup-release' +import { SshTargetOrcadClaims } from './ssh-target-orcad-claims' + +const TARGET: SshTarget = { + id: 'ssh-locked', + label: 'Locked down', + host: 'locked.example.com', + port: 22, + username: 'dev', + generation: 2 +} +const LOCAL_PORT = 46_900 + +const directories: string[] = [] +afterEach(async () => { + await closeTestStores() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +/** A conversion that fenced the host and registered its server, then could not reach it. */ +async function strandedConversion() { + const userDataPath = mkdtempSync(join(tmpdir(), 'orcad-unreachable-setup-')) + directories.push(userDataPath) + const store = createSqliteTestStore(Store, { dataFile: join(userDataPath, 'orca-data.json') }) + store.addSshTarget(TARGET) + store.addRepo({ + id: 'repo-1', + path: '/srv/app', + displayName: 'App', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: TARGET.id + }) + const claims = new SshTargetOrcadClaims(store) + const fenced = await fenceOrcadMigrationSource({ + userDataPath, + store, + claims, + targetId: TARGET.id, + destinationEnvironmentId: 'env-1', + destinationName: 'Managed', + terminalProof: { verdict: 'exited', provenPtyIds: [] }, + hasDirectSshAuthority: () => false + }) + if (fenced.outcome !== 'fenced') { + throw new Error('expected a fence') + } + addManagedOrcadEnvironment(userDataPath, { + id: 'env-1', + name: 'Managed', + pairingCode: encodePairingOffer({ + v: PAIRING_OFFER_VERSION, + endpoint: `ws://127.0.0.1:${LOCAL_PORT}/`, + deviceToken: 'device-token', + publicKeyB64: 'public-key' + }), + orcadDeployment: { + sshTargetId: TARGET.id, + sshTargetGeneration: store.getSshTarget(TARGET.id)!.generation!, + localPort: LOCAL_PORT, + remotePort: 6_768 + } + }) + return { userDataPath, store, claims, migrationId: fenced.cutover.migrationId } +} + +describe('releasing a setup the host never let this client reach', () => { + it('unregisters the server and releases the fence, keeping every source row', async () => { + const h = await strandedConversion() + + await releaseUnreachableOrcadSetup({ ...h, targetId: TARGET.id }) + + expect(getManagedOrcadFenceEnvironmentId(h.store.getSshTarget(TARGET.id))).toBeNull() + expect(listOrcadMigrationSourceCutovers(h.userDataPath)).toEqual([]) + expect(listEnvironments(h.userDataPath)).toEqual([]) + expect(h.store.getRepos()).toEqual([expect.objectContaining({ id: 'repo-1' })]) + }) + + it('leaves a server that already holds staged state alone', async () => { + const h = await strandedConversion() + const destination = fakeOrcadMigrationDestination() + await stageOrcadMigrationDestination( + { userDataPath: h.userDataPath, store: h.store, claims: h.claims, destination }, + h.migrationId + ) + + await releaseUnreachableOrcadSetup({ ...h, targetId: TARGET.id }) + + expect(getManagedOrcadFenceEnvironmentId(h.store.getSshTarget(TARGET.id))).toBe('env-1') + expect(listEnvironments(h.userDataPath)).toHaveLength(1) + }) +}) diff --git a/src/main/ssh/orcad-unreachable-setup-release.ts b/src/main/ssh/orcad-unreachable-setup-release.ts new file mode 100644 index 00000000000..7d68e2067b3 --- /dev/null +++ b/src/main/ssh/orcad-unreachable-setup-release.ts @@ -0,0 +1,89 @@ +/** + * Recovers a host an earlier build stranded: a conversion fenced it and registered its managed + * server, but the host's sshd refuses the forward that server is reached by, so nothing was ever + * staged there. The source still holds every row, so the server is unregistered and the fence + * released through the undeployed-fence path, and the relay serves the host again. An orcad the + * setup did activate is stopped first, so a later conversion starts fresh rather than deferring. + */ +import { listEnvironments } from '../../shared/runtime-environment-store' +import { removeManagedOrcadEnvironment } from '../../shared/runtime-environment-managed-orcad-store' +import { findOrcadMigrationSourceCutoverForTarget } from './orcad-migration-cutover-journal' +import { releaseUndeployedMigrationFence } from './orcad-migration-source-fence' +import { closeOrcadManagedTunnel } from './orcad-managed-tunnel' +import type { SshTargetOrcadClaims } from './ssh-target-orcad-claims' +import { withOrcadActivationLock } from './orcad-activation-lock' +import { withDeactivatedVersion } from './orcad-activation-record' +import { + readOrcadActivationRecord, + writeOrcadActivationRecord +} from './orcad-activation-record-store' +import { managedOrcadSlot } from './orcad-managed-runtime-context' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { orcadStopFreedTheHost } from './orcad-remote-process-control' +import { orcadSlotDir, stopOrcadSlot } from './orcad-recovery-slot' +import { getSshConnectionManager, getSshTargetRegistryStore } from './ssh-target-registry' +import { errorMessage } from '../../shared/error-message' + +export async function releaseUnreachableOrcadSetup(args: { + userDataPath: string + claims: SshTargetOrcadClaims + targetId: string + signal?: AbortSignal +}): Promise { + const cutover = findOrcadMigrationSourceCutoverForTarget(args.userDataPath, args.targetId) + // Why only this phase: a staged or committed destination holds state only it can account for. + if (cutover?.phase !== 'source-fenced') { + return + } + const environmentId = cutover.destinationEnvironmentId + await closeOrcadManagedTunnel(environmentId).catch(() => undefined) + await stopStrandedOrcad(args.targetId, args.signal).catch((error: unknown) => { + console.warn(`[orcad] Could not stop the unreachable setup's server: ${errorMessage(error)}`) + }) + const isRegistered = (id: string): boolean => + listEnvironments(args.userDataPath).some((entry) => entry.id === id) + // Unregister first: a crash after it leaves an undeployed fence, which the same path releases. + if (isRegistered(environmentId)) { + removeManagedOrcadEnvironment(args.userDataPath, environmentId) + } + await releaseUndeployedMigrationFence({ + userDataPath: args.userDataPath, + claims: args.claims, + targetId: args.targetId, + isDestinationRegistered: isRegistered, + signal: args.signal + }) +} + +/** Clears `active` only once the slot is proven exited; its daemon and terminals outlive it. */ +async function stopStrandedOrcad(targetId: string, signal?: AbortSignal): Promise { + const target = getSshTargetRegistryStore()?.getTarget(targetId) + const connectionManager = getSshConnectionManager() + if (!target || !connectionManager) { + return + } + const context = await resolveOrcadRemoteContext( + target, + await connectionManager.connect(target), + signal + ) + if (!context.activationRecord.active) { + return + } + // The port only matters for a launch, which this never does. + const slot = managedOrcadSlot(context, 0, signal) + await withOrcadActivationLock( + slot, + async () => { + const record = await readOrcadActivationRecord(slot) + if (!record.active) { + return + } + const stopped = await stopOrcadSlot(slot, orcadSlotDir(slot, record.active), false) + if (orcadStopFreedTheHost(stopped)) { + await writeOrcadActivationRecord(slot, withDeactivatedVersion(record)) + } + }, + () => undefined + ) +} diff --git a/src/main/ssh/orcad-unreachable-setup-stop.test.ts b/src/main/ssh/orcad-unreachable-setup-stop.test.ts new file mode 100644 index 00000000000..4d38ca4a315 --- /dev/null +++ b/src/main/ssh/orcad-unreachable-setup-stop.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { emptyOrcadActivationRecord } from './orcad-activation-record' + +const mocks = vi.hoisted(() => { + const record: { active: string | null } = { active: '1.0.0+a' } + return { stop: vi.fn(), write: vi.fn(), release: vi.fn(), record } +}) +vi.mock('../../shared/runtime-environment-store', () => ({ listEnvironments: () => [] })) +vi.mock('../../shared/runtime-environment-managed-orcad-store', () => ({ + removeManagedOrcadEnvironment: vi.fn() +})) +vi.mock('./orcad-migration-cutover-journal', () => ({ + findOrcadMigrationSourceCutoverForTarget: () => ({ + phase: 'source-fenced', + destinationEnvironmentId: 'env-1' + }) +})) +vi.mock('./orcad-migration-source-fence', () => ({ + releaseUndeployedMigrationFence: mocks.release +})) +vi.mock('./orcad-managed-tunnel', () => ({ closeOrcadManagedTunnel: async () => undefined })) +vi.mock('./ssh-target-registry', () => ({ + getSshTargetRegistryStore: () => ({ getTarget: () => ({ id: 'ssh-1' }) }), + getSshConnectionManager: () => ({ connect: async () => ({}) }) +})) +vi.mock('./orcad-remote-context', () => ({ + resolveOrcadRemoteContext: async () => ({ + activationRecord: { ...emptyOrcadActivationRecord(), active: mocks.record.active }, + connection: {}, + host: { os: 'linux' }, + remoteHome: '/home/u', + userDataDir: '/home/u/.orca' + }) +})) +vi.mock('./orcad-activation-lock', () => ({ + withOrcadActivationLock: async (_options: unknown, run: () => Promise) => run() +})) +vi.mock('./orcad-activation-record-store', () => ({ + readOrcadActivationRecord: async () => ({ + ...emptyOrcadActivationRecord(), + active: mocks.record.active + }), + writeOrcadActivationRecord: mocks.write +})) +vi.mock('./orcad-recovery-slot', () => ({ + orcadSlotDir: (_slot: unknown, version: string) => `/slots/${version}`, + stopOrcadSlot: mocks.stop +})) + +const { releaseUnreachableOrcadSetup } = await import('./orcad-unreachable-setup-release') +const release = () => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: claims pass through to the mocked fence release only. + releaseUnreachableOrcadSetup({ userDataPath: '/u', claims: {} as never, targetId: 'ssh-1' }) + +beforeEach(() => { + vi.clearAllMocks() + mocks.record.active = '1.0.0+a' +}) + +describe('releasing an unreachable setup', () => { + it('stops the orcad it activated and clears active only on proven exit', async () => { + mocks.stop.mockResolvedValueOnce('stopped') + await release() + expect(mocks.stop).toHaveBeenCalledWith(expect.anything(), '/slots/1.0.0+a', false) + expect(mocks.write.mock.calls[0]?.[1]).toMatchObject({ active: null, previous: '1.0.0+a' }) + expect(mocks.release).toHaveBeenCalledOnce() + }) + + it('keeps the record when the stop is not proven, and still releases the fence', async () => { + mocks.stop.mockResolvedValueOnce('unconfirmed') + await release() + expect(mocks.write).not.toHaveBeenCalled() + expect(mocks.release).toHaveBeenCalledOnce() + }) + + it('touches nothing on a host whose setup never activated', async () => { + mocks.record.active = null + await release() + expect(mocks.stop).not.toHaveBeenCalled() + expect(mocks.release).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ssh/orcad-update-plan.test.ts b/src/main/ssh/orcad-update-plan.test.ts index 5d90c597e3c..daa2014a016 100644 --- a/src/main/ssh/orcad-update-plan.test.ts +++ b/src/main/ssh/orcad-update-plan.test.ts @@ -1,12 +1,16 @@ import { describe, expect, it } from 'vitest' import { assessOrcadRollback, planOrcadUpdate } from './orcad-update-plan' +import { collectOrcadTerminalCensus } from '../orcad/orcad-terminal-census' +import type { DaemonSessionInfo } from '../daemon/types' import { emptyOrcadActivationRecord, type OrcadActivationRecord, type OrcadStateSnapshot } from './orcad-activation-record' +const PROTOCOL = { protocolVersion: 3, previousProtocolVersions: [1, 2] } + const SNAPSHOT: OrcadStateSnapshot = { dirName: 'pre-0.2.0+bb01-1000', takenBeforeVersion: '0.2.0+bb01', @@ -28,18 +32,20 @@ function record(overrides: Partial = {}): OrcadActivation describe('planOrcadUpdate', () => { it('does nothing when the candidate is already active', () => { const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.2.0+bb01', - census: { liveSessions: 0, startedSinceActivation: 0 } + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 } }) expect(plan).toMatchObject({ action: 'noop' }) }) it('defers rather than restarting a host with live terminals', () => { const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.3.0+cc01', - census: { liveSessions: 3, startedSinceActivation: 1 } + census: { liveSessions: 3, startedSinceActivation: 1, daemonProtocolVersion: 3 } }) expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_terminals_running' }) expect(plan.action === 'defer' && plan.reason).toContain('would not kill them') @@ -47,9 +53,10 @@ describe('planOrcadUpdate', () => { it('defers when the session count cannot be established', () => { const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.3.0+cc01', - census: { liveSessions: null, startedSinceActivation: null } + census: { liveSessions: null, startedSinceActivation: null, daemonProtocolVersion: 3 } }) expect(plan).toMatchObject({ action: 'defer', @@ -59,9 +66,10 @@ describe('planOrcadUpdate', () => { it('plans a forced update with an unknown census as if terminals were live', () => { const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.3.0+cc01', - census: { liveSessions: null, startedSinceActivation: null }, + census: { liveSessions: null, startedSinceActivation: null, daemonProtocolVersion: 3 }, force: true }) expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) @@ -69,19 +77,68 @@ describe('planOrcadUpdate', () => { it('carries the daemon across a forced update with live terminals', () => { const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.3.0+cc01', - census: { liveSessions: 2, startedSinceActivation: 0 }, + census: { liveSessions: 2, startedSinceActivation: 0, daemonProtocolVersion: 3 }, force: true }) expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: true }) }) - it('replaces the daemon only when nothing is running under it', () => { + it.each([false, true])( + "never claims a degraded host's in-process terminals survive (force: %s)", + async (force) => { + // One daemon session on a reachable protocol plus two terminals inside orcad itself. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the census reads only createdAt and protocolVersion. + const daemonSession = { + sessionId: 's', + createdAt: 5, + protocolVersion: 3 + } as DaemonSessionInfo + const census = await collectOrcadTerminalCensus( + 1, + async () => [daemonSession], + async () => 2 + ) + expect(census).toMatchObject({ liveSessions: 3, inProcessSessions: 2 }) + const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, + record: record(), + candidateVersion: '0.3.0+cc01', + census, + force + }) + expect(plan).toMatchObject({ + action: 'defer', + code: 'orcad_update_ends_in_process_terminals' + }) + expect(plan.action === 'defer' && plan.reason).toContain('Any restart ends them') + } + ) + + it('names in-process terminals, not an unreported protocol, when the daemon is empty', async () => { + const census = await collectOrcadTerminalCensus( + 1, + async () => [], + async () => 2 + ) const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, record: record(), candidateVersion: '0.3.0+cc01', - census: { liveSessions: 0, startedSinceActivation: 0 } + census, + force: true + }) + expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_ends_in_process_terminals' }) + }) + + it('replaces the daemon only when nothing is running under it', () => { + const plan = planOrcadUpdate({ + candidateDaemonProtocol: PROTOCOL, + record: record(), + candidateVersion: '0.3.0+cc01', + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 } }) expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: false }) }) @@ -90,9 +147,10 @@ describe('planOrcadUpdate', () => { describe('assessOrcadRollback', () => { it('is clean when the snapshot is intact and nothing happened since activation', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: true, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: false }) expect(safety).toMatchObject({ safety: 'clean', target: '0.1.0+aa01' }) @@ -100,9 +158,10 @@ describe('assessOrcadRollback', () => { it('is lossy, and names what goes, once the store has been written since activation', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: true, - census: { liveSessions: 1, startedSinceActivation: 0 }, + census: { liveSessions: 1, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: true }) expect(safety).toMatchObject({ safety: 'lossy', target: '0.1.0+aa01' }) @@ -111,9 +170,10 @@ describe('assessOrcadRollback', () => { it('treats an unreadable store mtime as writes, not as a clean rollback', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: true, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: null }) expect(safety).toMatchObject({ safety: 'lossy' }) @@ -122,9 +182,10 @@ describe('assessOrcadRollback', () => { // The point past which rollback is unsafe: the first terminal created after activation. it('refuses once a terminal started after activation, because restoring would orphan it', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: true, - census: { liveSessions: 4, startedSinceActivation: 1 }, + census: { liveSessions: 4, startedSinceActivation: 1, daemonProtocolVersion: 3 }, stateWritesSinceActivation: true }) expect(safety).toMatchObject({ @@ -136,9 +197,10 @@ describe('assessOrcadRollback', () => { it('refuses when the snapshot the record names is gone from the host', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: false, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: false }) expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) @@ -147,9 +209,10 @@ describe('assessOrcadRollback', () => { it('refuses when no snapshot was ever recorded', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record({ snapshot: null }), snapshotPresent: true, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: false }) expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_missing' }) @@ -157,9 +220,10 @@ describe('assessOrcadRollback', () => { it('refuses when the post-activation session count is unverifiable', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record(), snapshotPresent: true, - census: { liveSessions: 2, startedSinceActivation: null }, + census: { liveSessions: 2, startedSinceActivation: null, daemonProtocolVersion: 3 }, stateWritesSinceActivation: false }) expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_census_unavailable' }) @@ -167,11 +231,83 @@ describe('assessOrcadRollback', () => { it('refuses when there is no previous version to go back to', () => { const safety = assessOrcadRollback({ + targetDaemonProtocol: PROTOCOL, record: record({ previous: null }), snapshotPresent: true, - census: { liveSessions: 0, startedSinceActivation: 0 }, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: 3 }, stateWritesSinceActivation: false }) expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_no_target' }) }) }) + +describe('D7 daemon protocol crossing', () => { + const census = (daemonProtocolVersion: number | null, liveSessions: number | null = 2) => ({ + liveSessions, + startedSinceActivation: 0, + daemonProtocolVersion + }) + + it.each([false, true])( + 'defers an update that would strand live terminals (force %s)', + (force) => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + candidateDaemonProtocol: PROTOCOL, + census: census(4), + force + }) + expect(plan).toMatchObject({ action: 'defer', code: 'orcad_update_strands_live_terminals' }) + } + ) + + it('will not force past live terminals whose daemon protocol is unknown', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + candidateDaemonProtocol: PROTOCOL, + census: census(null), + force: true + }) + expect(plan).toMatchObject({ + action: 'defer', + code: 'orcad_update_daemon_protocol_unverifiable' + }) + }) + + it('needs no protocol answer when no terminals are running', () => { + const plan = planOrcadUpdate({ + record: record(), + candidateVersion: '0.3.0+cc01', + candidateDaemonProtocol: PROTOCOL, + census: census(null, 0) + }) + expect(plan).toMatchObject({ action: 'proceed', preservesLiveDaemon: false }) + }) + + it.each([ + [4, 'orcad_rollback_strands_live_terminals'], + [null, 'orcad_rollback_daemon_protocol_unverifiable'] + ])('refuses a rollback when the daemon speaks %s', (daemonProtocolVersion, code) => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: true, + census: census(daemonProtocolVersion), + targetDaemonProtocol: PROTOCOL, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code }) + }) + + it('does not read an unverifiable snapshot probe as a missing snapshot', () => { + const safety = assessOrcadRollback({ + record: record(), + snapshotPresent: null, + census: census(3, 0), + targetDaemonProtocol: PROTOCOL, + stateWritesSinceActivation: false + }) + expect(safety).toMatchObject({ safety: 'unsafe', code: 'orcad_rollback_snapshot_unverifiable' }) + }) +}) diff --git a/src/main/ssh/orcad-update-plan.ts b/src/main/ssh/orcad-update-plan.ts index 2653956ebe5..f634e7395b5 100644 --- a/src/main/ssh/orcad-update-plan.ts +++ b/src/main/ssh/orcad-update-plan.ts @@ -19,20 +19,13 @@ * pre-activation snapshot rather than against a version comparison. */ import type { OrcadActivationRecord } from './orcad-activation-record' +import type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' +import { + assessOrcadLiveDaemonCrossing, + type OrcadDaemonProtocolFacts +} from './orcad-daemon-protocol-crossing' -export type OrcadTerminalCensus = { - /** - * Sessions the live daemon owns right now. `null` means the probe could not answer — - * never treated as zero, because loss of contact is not evidence of process death - * (docs/reference/ssh-execution-boundary.md). - */ - liveSessions: number | null - /** - * Of those, how many started at or after `record.activatedAt`. These are the sessions the - * pre-activation snapshot does not describe. - */ - startedSinceActivation: number | null -} +export type { OrcadTerminalCensus } from '../../shared/orcad-terminal-census' export type OrcadUpdateDecision = | { action: 'noop'; reason: string } @@ -47,6 +40,9 @@ export type OrcadUpdateDecision = export type OrcadUpdateDeferCode = | 'orcad_update_terminals_running' | 'orcad_update_terminal_census_unavailable' + | 'orcad_update_strands_live_terminals' + | 'orcad_update_daemon_protocol_unverifiable' + | 'orcad_update_ends_in_process_terminals' /** * Decide whether to restart orcad onto `candidateVersion`. @@ -60,6 +56,8 @@ export function planOrcadUpdate(input: { record: OrcadActivationRecord candidateVersion: string census: OrcadTerminalCensus + /** The candidate's daemon protocol and the older ones it can still attach to. */ + candidateDaemonProtocol: OrcadDaemonProtocolFacts force?: boolean }): OrcadUpdateDecision { if (input.record.active === input.candidateVersion) { @@ -68,7 +66,44 @@ export function planOrcadUpdate(input: { reason: `${input.candidateVersion} is already the active version; nothing to restart.` } } + const inProcess = input.census.inProcessSessions ?? 0 + // First, and force cannot override it: these run inside orcad, so no daemon carries them + // across, and an empty daemon list would otherwise read as an unreported protocol. + if (inProcess > 0) { + return { + action: 'defer', + code: 'orcad_update_ends_in_process_terminals', + reason: + `${inProcess} terminal${inProcess === 1 ? ' runs' : 's run'} inside the orcad process ` + + 'itself because its terminal daemon is degraded. Any restart ends ' + + `${inProcess === 1 ? 'it' : 'them'}, and forcing the update cannot keep ` + + `${inProcess === 1 ? 'it' : 'them'} alive. Close ${inProcess === 1 ? 'it' : 'them'}, ` + + 'then update.' + } + } + const crossing = assessOrcadLiveDaemonCrossing(input.census, input.candidateDaemonProtocol) + // Why force cannot override: the operator can accept a mixed pair, not unreachable terminals. + if (crossing === 'strands-live-terminals') { + return { + action: 'defer', + code: 'orcad_update_strands_live_terminals', + reason: + `The live terminal daemon speaks protocol ${String(input.census.daemonProtocolVersion)}, ` + + `which orcad ${input.candidateVersion} cannot attach to. Restarting now would leave ` + + 'every running terminal unreachable. Update when no terminals are running.' + } + } const { liveSessions } = input.census + if (input.force && crossing === 'unverifiable') { + return { + action: 'defer', + code: 'orcad_update_daemon_protocol_unverifiable', + reason: + 'The terminal daemon did not report its protocol, so this update cannot show that ' + + `orcad ${input.candidateVersion} will reach the terminals it is forced past. Retry ` + + 'when the daemon answers.' + } + } if (liveSessions === null) { if (!input.force) { return { @@ -136,6 +171,9 @@ export type OrcadRollbackUnsafeCode = | 'orcad_rollback_snapshot_missing' | 'orcad_rollback_orphans_live_terminals' | 'orcad_rollback_census_unavailable' + | 'orcad_rollback_snapshot_unverifiable' + | 'orcad_rollback_strands_live_terminals' + | 'orcad_rollback_daemon_protocol_unverifiable' /** * How safe it is to switch back to `record.previous`. @@ -156,9 +194,11 @@ export type OrcadRollbackUnsafeCode = */ export function assessOrcadRollback(input: { record: OrcadActivationRecord - /** Whether the snapshot named by the record is actually still on the host. */ - snapshotPresent: boolean + /** Whether the snapshot is still on the host; `null` means the probe was unverifiable. */ + snapshotPresent: boolean | null census: OrcadTerminalCensus + /** The rollback target's daemon protocol facts, from the client's copy of its bytes. */ + targetDaemonProtocol: OrcadDaemonProtocolFacts /** * Whether the shared store has been written since activation, from its mtime against * `record.activatedAt`. `null` means unknown, which is treated as "yes" — claiming a @@ -176,6 +216,15 @@ export function assessOrcadRollback(input: { 'to. Deploy a known-good build instead.' } } + if (input.record.snapshot && input.snapshotPresent === null) { + return { + safety: 'unsafe', + code: 'orcad_rollback_snapshot_unverifiable', + reason: + 'The host did not give a trustworthy answer about the pre-activation snapshot. Retry ' + + 'when the host is reachable; loss of contact is not evidence the snapshot is gone.' + } + } if (!input.record.snapshot || !input.snapshotPresent) { return { safety: 'unsafe', @@ -209,6 +258,26 @@ export function assessOrcadRollback(input: { 'reattach. Close them (or let them exit) and roll back then.' } } + const crossing = assessOrcadLiveDaemonCrossing(input.census, input.targetDaemonProtocol) + if (crossing === 'unverifiable') { + return { + safety: 'unsafe', + code: 'orcad_rollback_daemon_protocol_unverifiable', + reason: + 'The terminal daemon did not report its protocol, so this rollback cannot show that ' + + `${target} would reach the terminals still running. Retry when the daemon answers.` + } + } + if (crossing === 'strands-live-terminals') { + return { + safety: 'unsafe', + code: 'orcad_rollback_strands_live_terminals', + reason: + `The live terminal daemon speaks protocol ${String(input.census.daemonProtocolVersion)}, ` + + `which ${target} does not list. Rolling back now would leave every running terminal ` + + 'unreachable. Roll back when no terminals are running, or deploy forward.' + } + } if (input.stateWritesSinceActivation === false) { return { safety: 'clean', diff --git a/src/main/ssh/orcad-windows-host-fence-ops.test.ts b/src/main/ssh/orcad-windows-host-fence-ops.test.ts new file mode 100644 index 00000000000..7993ac76ca7 --- /dev/null +++ b/src/main/ssh/orcad-windows-host-fence-ops.test.ts @@ -0,0 +1,221 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runProcess, spawnProcess } from '../../shared/child-process/run-process' +import { ORCAD_FENCE_LOST_EXIT, ORCAD_FENCE_LOST_MARKER } from './orcad-activation-fence-scope' +import { ORCAD_WINDOWS_HOST_SCRIPT } from './orcad-windows-host-script' + +const dirs: string[] = [] +afterEach(() => { + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +function hostWithFence(token: string) { + const dir = mkdtempSync(join(tmpdir(), 'orcad-win-fence-')) + dirs.push(dir) + const script = join(dir, 'host.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const root = join(dir, '.orcad-activation-transaction') + const lock = join(root, '.install-lock') + mkdirSync(lock, { recursive: true }) + writeFileSync(join(lock, '.orca-fence-owner'), token) + const journal = join(root, 'transaction.json') + writeFileSync(journal, JSON.stringify({ fenceToken: token }, null, 2)) + const op = (...args: string[]) => + runProcess({ program: process.execPath, args: [script, ...args], timeoutMs: 15_000 }) + return { lock, journal, op } +} + +// The Windows host script enforces the same generation check as the POSIX guard. +describe('the Windows host script under an activation fence', () => { + it('runs an op only for the fence’s current holder', async () => { + const host = hostWithFence('successor') + const stale = await host.op('--fence', host.lock, 'stale', 'fence-check') + expect(stale.code).toBe(ORCAD_FENCE_LOST_EXIT) + expect(stale.stdout).toContain(ORCAD_FENCE_LOST_MARKER) + const owner = await host.op('--fence', host.lock, 'successor', 'fence-check') + expect(owner).toMatchObject({ code: 0, stdout: 'OK' }) + }) + + it('releases only its own generation, leaving a successor’s fence and journal', async () => { + const host = hostWithFence('successor') + expect((await host.op('fence-release', host.lock, host.journal, 'stale')).stdout).toBe( + 'SUPERSEDED' + ) + expect(existsSync(host.lock)).toBe(true) + expect(existsSync(host.journal)).toBe(true) + expect((await host.op('fence-release', host.lock, host.journal, 'successor')).stdout).toBe( + 'RELEASED' + ) + expect(existsSync(host.lock)).toBe(false) + expect(existsSync(host.journal)).toBe(false) + }) + + // Astra pass 9, the Windows state-mutation race: a successor's restore must not run beside a + // paused clear whose holder the script cannot identify. + it('keeps a successor restore busy while a paused clear still holds the mutation lock', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orcad-win-mutation-')) + dirs.push(dir) + const script = join(dir, 'host.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const root = join(dir, 'root') + const snapshot = join(dir, 'snapshot') + mkdirSync(join(root, 'profiles'), { recursive: true }) + writeFileSync(join(root, 'profiles', 'state.json'), 'snapshot-state') + const fence = join(dir, '.orcad-activation-transaction', '.install-lock') + mkdirSync(fence, { recursive: true }) + writeFileSync(join(fence, '.orca-fence-owner'), 'first-token') + const fenced = (token: string, ...args: string[]) => [script, '--fence', fence, token, ...args] + const capture = await runProcess({ + program: process.execPath, + args: fenced('first-token', 'snapshot-capture', root, snapshot) + }) + expect(capture.stdout.trim()).toBe('CAPTURED') + const ready = join(dir, 'ready') + const resume = join(dir, 'resume') + const preload = join(dir, 'pause-rm.cjs') + writeFileSync( + preload, + `const fs=require('fs'); const rm=fs.promises.rm; fs.promises.rm=async function(p,...a){ if(p===${JSON.stringify(join(root, 'profiles'))}){ fs.writeFileSync(${JSON.stringify(ready)},''); while(!fs.existsSync(${JSON.stringify(resume)})) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,20) } return rm.call(this,p,...a) }` + ) + const first = spawnProcess({ + program: process.execPath, + args: ['--require', preload, ...fenced('first-token', 'snapshot-clear', root)] + }) + try { + await vi.waitFor(() => expect(existsSync(ready)).toBe(true)) + utimesSync(join(dir, 'orcad-state-mutation.lock'), new Date(0), new Date(0)) + writeFileSync(join(fence, '.orca-fence-owner'), 'successor-token') + const restore = await runProcess({ + program: process.execPath, + args: fenced('successor-token', 'snapshot-restore', root, snapshot) + }) + expect(restore.stdout.trim()).toBe('STATE_MUTATION_BUSY') + expect(readFileSync(join(root, 'profiles', 'state.json'), 'utf8')).toBe('snapshot-state') + } finally { + writeFileSync(resume, '') + first.kill('SIGKILL') + } + }) +}) + +// BUG-23 on Windows: the host names the lock an exited predecessor left, never a live run's, and +// writes nothing, so the steal's own arbitration is the only thing that takes it (Astra 26087). +describe('the Windows host script proving a lock an exited client left', () => { + function quietHost(owner: string) { + const dir = mkdtempSync(join(tmpdir(), 'orcad-win-exited-')) + dirs.push(dir) + const script = join(dir, 'host.js') + writeFileSync(script, ORCAD_WINDOWS_HOST_SCRIPT) + const lock = join(dir, '.orcad-activation-transaction', '.install-lock') + mkdirSync(lock, { recursive: true }) + writeFileSync(join(lock, '.orca-fence-owner'), owner) + const quietSince = new Date(Date.now() - 10 * 60_000) + utimesSync(lock, quietSince, quietSince) + const mutation = join(dir, 'orcad-state-mutation.lock') + const check = async (guard: '0' | '1', ...tokens: string[]) => { + const before = statSync(lock).mtimeMs + const result = await runProcess({ + program: process.execPath, + args: [script, 'fence-exited-owner', lock, guard, ...tokens], + timeoutMs: 15_000 + }) + expect(statSync(lock).mtimeMs).toBe(before) + return result.stdout.trim() + } + return { lock, mutation, check } + } + + it('names a quiet lock an exited client holds, and nothing else', async () => { + const host = quietHost('t-exited') + expect(await host.check('0', 't-other')).toBe('KEPT') + expect(await host.check('0', 't-other', 't-exited')).toBe('EXITED_OWNER t-exited') + }) + + it('keeps a lock that is not yet quiet', async () => { + const host = quietHost('t-exited') + utimesSync(host.lock, new Date(), new Date()) + expect(await host.check('0', 't-exited')).toBe('KEPT') + }) + + it('keeps the fence while its state mutation holder may still run', async () => { + const host = quietHost('t-exited') + mkdirSync(host.mutation) + // A live pid whose creation time is unreadable is unverifiable, never exited. + writeFileSync( + join(host.mutation, 'owner.json'), + JSON.stringify({ pid: process.pid, creationTimeMs: 1234 }) + ) + expect(await host.check('1', 't-exited')).toBe('KEPT') + // The install lock guards no state mutation. + expect(await host.check('0', 't-exited')).toBe('EXITED_OWNER t-exited') + }) + + // Astra 26087 r2: the steal can only hold an absent mutation lock across the takeover. + it('keeps the fence while any mutation lock exists, even one whose holder exited', async () => { + const host = quietHost('t-exited') + mkdirSync(host.mutation) + utimesSync(host.mutation, new Date(0), new Date(0)) + expect(await host.check('1', 't-exited')).toBe('KEPT') + const exited = await runProcess({ program: process.execPath, args: ['-p', 'process.pid'] }) + writeFileSync( + join(host.mutation, 'owner.json'), + JSON.stringify({ pid: Number(exited.stdout.trim()), creationTimeMs: 1234 }) + ) + expect(await host.check('1', 't-exited')).toBe('KEPT') + rmSync(host.mutation, { recursive: true }) + expect(await host.check('1', 't-exited')).toBe('EXITED_OWNER t-exited') + }) +}) + +// Astra 26087 r2: the exited-owner steal holds the mutation lock across the takeover, so a +// mutation that takes the lock afterwards must find its fence gone and stop before any work. +describe('a Windows state mutation admitted after its fence was replaced', () => { + it('stops before touching state and frees the mutation lock', async () => { + const host = hostWithFence('old-token') + const dir = join(host.lock, '..', '..') + const root = join(dir, 'root') + mkdirSync(join(root, 'profiles'), { recursive: true }) + writeFileSync(join(root, 'profiles', 'state.json'), 'kept') + const preload = join(dir, 'steal-before-mutation-lock.cjs') + // The fence check passes, then the fence is replaced right before the mutation lock is taken. + writeFileSync( + preload, + `const fs = require('fs'); const mkdir = fs.mkdirSync; +fs.mkdirSync = function (p, ...rest) { + if (String(p).endsWith('orcad-state-mutation.lock')) fs.writeFileSync(${JSON.stringify(join(host.lock, '.orca-fence-owner'))}, 'new-token') + return mkdir.call(this, p, ...rest) +}` + ) + const result = await runProcess({ + program: process.execPath, + args: [ + '--require', + preload, + join(dir, 'host.js'), + '--fence', + host.lock, + 'old-token', + 'snapshot-clear', + root + ], + timeoutMs: 15_000 + }) + expect(result.code).toBe(ORCAD_FENCE_LOST_EXIT) + expect(result.stdout.trim()).toBe(ORCAD_FENCE_LOST_MARKER) + expect(readFileSync(join(root, 'profiles', 'state.json'), 'utf8')).toBe('kept') + expect(existsSync(join(dir, 'orcad-state-mutation.lock'))).toBe(false) + }) +}) diff --git a/src/main/ssh/orcad-windows-host-fence-ops.ts b/src/main/ssh/orcad-windows-host-fence-ops.ts new file mode 100644 index 00000000000..c794daced6e --- /dev/null +++ b/src/main/ssh/orcad-windows-host-fence-ops.ts @@ -0,0 +1,74 @@ +/** + * The activation fence's ownership check and conditional release inside the Windows host script, + * matching the POSIX guard in `orcad-activation-fence-scope.ts`, and the exited-own-lock check of + * `orcad-exited-own-lock.ts`. + */ +import { + ORCAD_FENCE_LOST_EXIT, + ORCAD_FENCE_LOST_MARKER, + ORCAD_FENCE_OWNER_FILENAME +} from './orcad-activation-fence-scope' +import { ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS } from './orcad-state-snapshot-members' + +export const ORCAD_WINDOWS_FENCE_ARG = '--fence' + +const text = JSON.stringify + +/** Strips `--fence ` and exits before the op unless the token is still ours. */ +export const ORCAD_WINDOWS_FENCE_PRELUDE = ` +function fenceOwner(lockDir) { + try { return fs.readFileSync(path.join(lockDir, ${text(ORCAD_FENCE_OWNER_FILENAME)}), 'utf8') } catch { return null } +} +// The token this op runs under; null outside a fence, so nothing refreshes a fence it does not own. +let FENCE_TOKEN = null +let FENCE_DIR = null +function fencedArgv(argv) { + if (argv[0] !== ${text(ORCAD_WINDOWS_FENCE_ARG)}) return argv + FENCE_DIR = argv[1] + FENCE_TOKEN = argv[2] + if (fenceOwner(argv[1]) !== argv[2]) { + process.stdout.write(${text(`${ORCAD_FENCE_LOST_MARKER}\n`)}, () => process.exit(${ORCAD_FENCE_LOST_EXIT})) + return ['fence-lost'] + } + return argv.slice(3) +} +` + +export const ORCAD_WINDOWS_HOST_FENCE_OPS = ` +Object.assign(ops, { + 'fence-lost'() {}, + 'fence-check'() { + answer('OK') + }, + // Each piece is renamed aside and kept only if it is ours, else put straight back, so a release + // that stalls after its token check never deletes a successor's journal or lock. + 'fence-release'(lockDir, journal, token) { + if (fenceOwner(lockDir) !== token) return answer('SUPERSEDED') + const journalAside = journal + '.release.' + process.pid + let moved = false + try { fs.renameSync(journal, journalAside); moved = true } catch (error) { if (error.code !== 'ENOENT') throw error } + if (moved) { + const ours = fs.readFileSync(journalAside, 'utf8').includes(${text(`"fenceToken": `)} + JSON.stringify(token)) + if (!ours && !fs.existsSync(journal)) fs.renameSync(journalAside, journal) + fs.rmSync(journalAside, { force: true }) + } + const lockAside = lockDir + '.released.' + process.pid + fs.renameSync(lockDir, lockAside) + if (fenceOwner(lockAside) === token) fs.rmSync(lockAside, { recursive: true, force: true, maxRetries: 5 }) + else if (!fs.existsSync(lockDir)) fs.renameSync(lockAside, lockDir) + try { fs.rmdirSync(path.dirname(lockDir)) } catch {} + answer('RELEASED') + }, + // Read-only: names which of \`tokens\` (clients proven exited) the lock holds, once it is quiet + // and, with guardArg '1', no state-mutation lock exists. + // The steal then takes the lock under its own arbitration, so nothing here writes. + 'fence-exited-owner'(lockDir, guardArg, ...tokens) { + const quiet = (target) => Date.now() - (lstatOrNull(target)?.mtimeMs ?? Date.now()) > ${ORCAD_EXITED_OWN_LOCK_QUIET_SECONDS * 1000} + const token = fenceOwner(lockDir) + if (!tokens.includes(token) || !quiet(lockDir)) return answer('KEPT') + // Any mutation lock refuses, as the steal does: it can only take an absent one. + if (guardArg === '1' && lstatOrNull(MUTATION_LOCK)) return answer('KEPT') + answer('EXITED_OWNER ' + token) + } +}) +` diff --git a/src/main/ssh/orcad-windows-host-lane.test.ts b/src/main/ssh/orcad-windows-host-lane.test.ts new file mode 100644 index 00000000000..d7949316df3 --- /dev/null +++ b/src/main/ssh/orcad-windows-host-lane.test.ts @@ -0,0 +1,148 @@ +// Managed orcad on a real Win32-OpenSSH host, one cell per DefaultShell: prove the conversion's +// terminal gate against the pinned relay already serving the host, then resolve the context +// (pinned node.exe, host script), deploy and activate, prove readiness and liveness, reach orcad +// through the stdio bridge (this account's sshd refuses forwarding), decommission +// through the instance-bound stop request, prove exit, and run a GC pass. config/ci/windows-ssh-provider/invoke-pinned-relay-cells.ps1 +// provisions the account and runs this file for `orcad-*` cells; ssh-windows-hosts.yml runs that. +// +// Run: ORCA_RUN_SSH_WINDOWS_HOST=1 ORCA_SSH_WINDOWS_HOST_CELL= pnpm test +import { randomUUID } from 'node:crypto' +import { writeFileSync } from 'node:fs' +import { afterAll, beforeAll, describe, expect, it, vi, type MockInstance } from 'vitest' + +vi.mock('electron', () => ({ app: { getAppPath: () => process.cwd() } })) + +import { SshConnection } from './ssh-connection' +import { + connectHostileHost, + installHostileHostAppEnvironment +} from './ssh-hostile-host-test-harness' +import { resolveOrcadRemoteContext } from './orcad-remote-context' +import { proveWindowsRelayTerminalGate } from './orcad-windows-relay-terminal-gate-test-cell' +import { proveWindowsStdioBridge } from './orcad-windows-stdio-bridge-test-cell' +import { deployOrcad } from './orcad-remote-deploy' +import { orcadLivenessProbeCommand, parseOrcadLiveness } from './orcad-remote-launch' +import { orcadSlotDir, type OrcadSlotOptions } from './orcad-recovery-slot' +import { decommissionRemoteOrcad } from './orcad-remote-stop' +import { readOrcadActivationRecord } from './orcad-activation-record-store' +import { gcOldOrcadVersions } from './orcad-remote-gc' +import { execOrcadRemote } from './orcad-remote-runtime-control' +import { + isWindowsOrcadCellId, + readWindowsHostCellDescriptor, + windowsHostSshTarget +} from './ssh-windows-host-cells' + +const RUN = process.env.ORCA_RUN_SSH_WINDOWS_HOST === '1' +const CELL_TIMEOUT_MS = 20 * 60_000 + +/** Orcad's own host ops and launches: these must never take a PowerShell hop. */ +function isOrcadHostCommand(command: string): boolean { + return /orcad-host-script-[0-9a-f]{16}\.js|[\\/]orcad\.js /u.test(command) +} + +describe.runIf(RUN)('managed orcad on a Windows OpenSSH host', () => { + let cleanupAppEnvironment: (() => void) | null = null + + beforeAll(() => { + cleanupAppEnvironment = installHostileHostAppEnvironment() + }) + + afterAll(() => { + cleanupAppEnvironment?.() + }) + + it( + 'deploys, serves, decommissions by request and exits', + async () => { + const descriptor = readWindowsHostCellDescriptor(process.env.ORCA_SSH_WINDOWS_HOST_CELL ?? '') + if (!isWindowsOrcadCellId(descriptor.cell)) { + throw new Error(`${descriptor.cell} runs in ssh-relay-windows-host-lane.test.ts`) + } + const sshTarget = windowsHostSshTarget( + descriptor, + { id: descriptor.cell, remoteRuntime: 'pinned-node' }, + randomUUID() + ) + let exec: MockInstance | null = null + const receipt: Record = { cell: descriptor.cell, target: descriptor.target } + let conn: SshConnection | null = null + try { + conn = await connectHostileHost(sshTarget) + // A relay-hosted source first: conversion may proceed only once its terminals exited. + Object.assign(receipt, await proveWindowsRelayTerminalGate(conn, sshTarget.id)) + // After the prelude: its relay deploy re-spies `exec`, which is the same spy, and restores it. + exec = vi.spyOn(SshConnection.prototype, 'exec') + const context = await resolveOrcadRemoteContext(sshTarget, conn) + expect(context.host.os).toBe('win32') + const options: OrcadSlotOptions = { + conn, + host: context.host, + remoteHome: context.remoteHome, + nodePath: 'node', + userDataDir: context.userDataDir, + bindHost: '127.0.0.1', + port: 0 + } + const deployed = await deployOrcad({ + ...options, + target: context.serverTarget, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null } + }) + receipt.deploy = deployed + expect(deployed.outcome).toBe('installed-and-activated') + const slotDir = orcadSlotDir(options, deployed.fullVersion) + const liveness = async () => + parseOrcadLiveness( + await execOrcadRemote(options, orcadLivenessProbeCommand(options.host, slotDir)) + ) + expect(await liveness()).toBe('LIVE') + // The managed tunnel picks the stdio bridge on this account and reaches orcad through it. + receipt.stdioBridge = await proveWindowsStdioBridge(conn, options.host, slotDir) + expect(receipt.stdioBridge).toMatchObject({ + forwarding: 'refused', + response: expect.stringMatching(/^HTTP\/1\.1 101/u) + }) + // Decommission stops it by the instance-bound request orcad itself completes and proves. + const record = await readOrcadActivationRecord(options) + receipt.decommission = await decommissionRemoteOrcad({ + ...options, + record, + census: { liveSessions: 0, startedSinceActivation: 0, daemonProtocolVersion: null } + }) + // The message carries the refusal code and reason, which toMatchObject's diff omits. + expect(receipt.decommission, JSON.stringify(receipt.decommission)).toMatchObject({ + outcome: 'decommissioned', + version: deployed.fullVersion + }) + expect(await liveness()).toBe('DEAD') + // A GC pass after decommission must leave the recorded previous slot in place. + await gcOldOrcadVersions({ + conn, + host: options.host, + remoteHome: options.remoteHome, + currentDirAbsPath: slotDir, + record: await readOrcadActivationRecord(options) + }) + expect(await liveness()).toBe('DEAD') + + const commands = exec.mock.calls.map(([command]) => String(command)) + const orcadCommands = commands.filter(isOrcadHostCommand) + receipt.orcadCommands = orcadCommands.length + receipt.powershellCommands = commands.filter((command) => + /^powershell\.exe /iu.test(command) + ).length + expect(orcadCommands.length).toBeGreaterThan(0) + for (const command of orcadCommands) { + expect(command).not.toMatch(/EncodedCommand/u) + } + receipt.passed = true + } finally { + exec?.mockRestore() + await conn?.disconnect().catch(() => {}) + writeFileSync(descriptor.receipt, `${JSON.stringify(receipt, null, 2)}\n`) + } + }, + CELL_TIMEOUT_MS + ) +}) diff --git a/src/main/ssh/orcad-windows-host-preparation.ts b/src/main/ssh/orcad-windows-host-preparation.ts new file mode 100644 index 00000000000..e18f3d80dc6 --- /dev/null +++ b/src/main/ssh/orcad-windows-host-preparation.ts @@ -0,0 +1,40 @@ +/** + * What every managed-orcad operation on a Windows host needs before its first host op: this + * client's pinned node.exe in the runtime store, and the host script it runs. + * + * Both are idempotent: a present runtime costs one probe and nothing is uploaded, and the host + * script is content-addressed, so rewriting it changes nothing a running orcad depends on. + */ +import { join } from 'node:path' +import { getAppEnvironment } from '../../shared/app-environment' +import type { NodeRuntimeTarget } from '../../shared/node-runtime-pin' +import { ensureRemoteOrcadNodeRuntime } from './orcad-remote-node-runtime' +import { installOrcadWindowsHostScript, orcadRemoteBaseDir } from './orcad-remote-windows-node' +import { materializeNodeRuntimeArchive } from './pinned-runtime-materializer' +import type { SshConnection } from './ssh-connection' +import { joinRemotePath, type RemoteHostPlatform } from './ssh-remote-platform' + +export async function prepareWindowsOrcadHost(options: { + conn: SshConnection + host: RemoteHostPlatform + remoteHome: string + serverTarget: NodeRuntimeTarget + signal?: AbortSignal +}): Promise { + const baseDir = orcadRemoteBaseDir(options.host, options.remoteHome) + await ensureRemoteOrcadNodeRuntime({ + conn: options.conn, + host: options.host, + // Only its parent is read: the runtime store sits beside the slots. + slotDir: joinRemotePath(options.host, baseDir, 'orcad-host'), + target: options.serverTarget, + archivePath: () => + materializeNodeRuntimeArchive( + options.serverTarget, + join(getAppEnvironment().getPath('userData'), 'orcad-artifacts'), + { signal: options.signal } + ), + signal: options.signal + }) + await installOrcadWindowsHostScript(options, baseDir) +} diff --git a/src/main/ssh/orcad-windows-host-script.ts b/src/main/ssh/orcad-windows-host-script.ts new file mode 100644 index 00000000000..9b6957c3d84 --- /dev/null +++ b/src/main/ssh/orcad-windows-host-script.ts @@ -0,0 +1,299 @@ +/** + * The one fixed script Windows orcad hosts run, as a file beside the slots, under the pinned + * node.exe with plain path and number arguments: `node.exe