test(updater): capture Windows auto-relaunch boundary

2026-10-13 00:02:54 +00:00 · 2026-09-04 15:59:50 -07:00
parent 1b3077382d
commit 3eac7b05fd
6 changed files with 1395 additions and 0 deletions
@@ -0,0 +1,241 @@
+name: PR 18505 disposable packaged update QA
+
+on:
+  push:
+    branches:
+      - 'qa/pr18505-hosted-update-final'
+    paths:
+      - '.github/workflows/pr18505-hosted-update.yml'
+      - 'tests/tools/pr18505-hosted-update/**'
+
+permissions:
+  contents: read
+
+jobs:
+  windows-in-app-update:
+    name: windows-control-${{ matrix.traffic-mode }}
+    runs-on: windows-2022
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        traffic-mode: [none, until-old-exit]
+    env:
+      SOURCE_TAG: v1.4.198-adhoc.20260904205337
+      FINAL_TAG: v1.4.198-adhoc.20260904215141
+      FINAL_VERSION: 1.4.198-adhoc.20260904215141
+      HIGHER_TAG: v1.4.198-adhoc.20260904215158
+      HIGHER_VERSION: 1.4.198-adhoc.20260904215158
+      EXPECTED_HEAD: 1b3077382d1d77bb459ec6c937940d64f5fde812
+      ORCA_QA_SINGLE_HOP: '1'
+      ORCA_QA_TRAFFIC_MODE: ${{ matrix.traffic-mode }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Verify QA commit parent
+        shell: bash
+        run: test "$(git rev-parse HEAD^)" = "$EXPECTED_HEAD"
+      - uses: ./.github/actions/install-node-dependencies
+      - name: Verify releases and download older source installer
+        shell: pwsh
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          New-Item -ItemType Directory -Force artifacts/source | Out-Null
+          $source = gh release view $env:SOURCE_TAG --repo stablyai/orca-adhoc --json isDraft,isPrerelease,assets | ConvertFrom-Json
+          if ($source.isDraft -or -not $source.isPrerelease) { throw "$env:SOURCE_TAG is not a published prerelease" }
+          if (-not ($source.assets.name -contains 'orca-windows-setup.exe')) { throw "$env:SOURCE_TAG has no Windows installer" }
+          foreach ($tag in @($env:FINAL_TAG, $env:HIGHER_TAG)) {
+            $release = gh release view $tag --repo stablyai/orca-adhoc --json body,isDraft,isPrerelease,assets | ConvertFrom-Json
+            if ($release.isDraft -or -not $release.isPrerelease) { throw "$tag is not a published prerelease" }
+            if (-not $release.body.Contains($env:EXPECTED_HEAD)) { throw "$tag was not built from $env:EXPECTED_HEAD" }
+            if (-not ($release.assets.name -contains 'orca-windows-setup.exe')) { throw "$tag has no Windows installer" }
+          }
+          gh release download $env:SOURCE_TAG --repo stablyai/orca-adhoc --pattern orca-windows-setup.exe --dir artifacts/source
+      - name: Exercise real in-app update at traffic boundary
+        shell: pwsh
+        run: node tests/tools/pr18505-hosted-update/windows-live-update.mjs artifacts/source/orca-windows-setup.exe $env:FINAL_TAG $env:FINAL_VERSION $env:HIGHER_TAG $env:HIGHER_VERSION "artifacts/windows-$env:ORCA_QA_TRAFFIC_MODE"
+      - name: Prove manual recovery opens installed target idle
+        if: failure()
+        shell: pwsh
+        run: node tests/tools/pr18505-hosted-update/windows-manual-recovery.mjs $env:FINAL_VERSION "artifacts/windows-$env:ORCA_QA_TRAFFIC_MODE/profile" "artifacts/windows-$env:ORCA_QA_TRAFFIC_MODE"
+      - name: Record exact-artifact NSIS evidence boundary
+        if: always()
+        shell: pwsh
+        run: |
+          $output = "artifacts/windows-$env:ORCA_QA_TRAFFIC_MODE"
+          New-Item -ItemType Directory -Force $output | Out-Null
+          @(
+            "The exact release installer uses stock electron-builder NSIS without nsis.debugLogging.",
+            "There is no native NSIS LogSet file to preserve without rebuilding a different artifact.",
+            "app.log preserves electron-updater's exact installer invocation and arguments; the JSONL timeline preserves install-tree changes."
+          ) | Set-Content "$output/nsis-evidence-boundary.txt"
+          $roots = @(
+            "$env:LOCALAPPDATA/orca-updater",
+            "$env:LOCALAPPDATA/Programs"
+          )
+          $manifest = foreach ($root in $roots) {
+            if (Test-Path $root) {
+              Get-ChildItem -LiteralPath $root -File -Recurse -ErrorAction SilentlyContinue |
+                Select-Object FullName, Length, LastWriteTimeUtc, VersionInfo
+            }
+          }
+          $manifest | ConvertTo-Json -Depth 4 | Set-Content "$output/installed-file-manifest.json"
+      - name: Upload Windows proof
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: pr18505-windows-control-${{ matrix.traffic-mode }}
+          path: artifacts/windows-${{ matrix.traffic-mode }}/**
+          retention-days: 14
+          if-no-files-found: error
+
+  macos-in-app-update:
+    if: false
+    runs-on: macos-15
+    timeout-minutes: 45
+    env:
+      SOURCE_TAG: v1.4.198-adhoc.20260904205337
+      FINAL_TAG: v1.4.198-adhoc.20260904215141
+      FINAL_VERSION: 1.4.198-adhoc.20260904215141
+      HIGHER_TAG: v1.4.198-adhoc.20260904215158
+      HIGHER_VERSION: 1.4.198-adhoc.20260904215158
+      EXPECTED_HEAD: 1b3077382d1d77bb459ec6c937940d64f5fde812
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Verify QA commit parent
+        run: test "$(git rev-parse HEAD^)" = "$EXPECTED_HEAD"
+      - uses: ./.github/actions/install-node-dependencies
+      - name: Resolve hosted runner architecture
+        run: |
+          set -euo pipefail
+          case "$(uname -m)" in
+            arm64) echo 'MAC_ARCH=arm64' >> "$GITHUB_ENV" ;;
+            x86_64) echo 'MAC_ARCH=x64' >> "$GITHUB_ENV" ;;
+            *) echo "unsupported macOS architecture: $(uname -m)" >&2; exit 1 ;;
+          esac
+      - name: Verify releases and download older source DMG
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts/source artifacts/macos
+          source_json="$(gh release view "$SOURCE_TAG" --repo stablyai/orca-adhoc --json isDraft,isPrerelease,assets)"
+          jq -e '.isDraft == false and .isPrerelease == true' <<<"$source_json" >/dev/null
+          jq -e --arg asset "orca-macos-${MAC_ARCH}.dmg" '.assets | map(.name) | index($asset) != null' <<<"$source_json" >/dev/null
+          for tag in "$FINAL_TAG" "$HIGHER_TAG"; do
+            release_json="$(gh release view "$tag" --repo stablyai/orca-adhoc --json body,isDraft,isPrerelease,assets)"
+            jq -e --arg head "$EXPECTED_HEAD" '.isDraft == false and .isPrerelease == true and (.body | contains($head))' <<<"$release_json" >/dev/null
+            jq -e --arg asset "orca-macos-${MAC_ARCH}.dmg" '.assets | map(.name) | index($asset) != null' <<<"$release_json" >/dev/null
+          done
+          gh release download "$SOURCE_TAG" --repo stablyai/orca-adhoc --pattern "orca-macos-${MAC_ARCH}.dmg" --dir artifacts/source
+      - name: Install signed older source app in disposable login
+        run: |
+          set -euo pipefail
+          mount="$RUNNER_TEMP/orca-source-dmg"
+          mkdir -p "$mount"
+          hdiutil attach "artifacts/source/orca-macos-${MAC_ARCH}.dmg" -nobrowse -readonly -mountpoint "$mount"
+          trap 'hdiutil detach "$mount" >/dev/null 2>&1 || true' EXIT
+          test -d "$mount/Orca.app"
+          sudo ditto "$mount/Orca.app" /Applications/Orca.app
+          sudo chown -R "$(id -u):$(id -g)" /Applications/Orca.app
+          codesign --verify --deep --strict /Applications/Orca.app
+          xattr -dr com.apple.quarantine /Applications/Orca.app || true
+          hdiutil detach "$mount"
+          trap - EXIT
+          swiftc tests/tools/pr18505-hosted-update/mac-visible-window.swift -o artifacts/macos/mac-visible-window
+      - name: Exercise two real ShipIt updates under bounded CLI traffic
+        env:
+          ORCA_QA_MAC_WINDOW_PROBE: ${{ github.workspace }}/artifacts/macos/mac-visible-window
+        run: node tests/tools/pr18505-hosted-update/windows-live-update.mjs /Applications/Orca.app/Contents/MacOS/Orca "$FINAL_TAG" "$FINAL_VERSION" "$HIGHER_TAG" "$HIGHER_VERSION" artifacts/macos
+      - name: Upload macOS proof
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: pr18505-macos-packaged-update
+          path: artifacts/macos/**
+          retention-days: 14
+          if-no-files-found: error
+
+  linux-package-manager-upgrade:
+    if: false
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      EXPECTED_HEAD: 1b3077382d1d77bb459ec6c937940d64f5fde812
+      FINAL_VERSION: 1.4.198-qa.18505.1
+      HIGHER_VERSION: 1.4.198-qa.18505.2
+      FINAL_DPKG_VERSION: 1.4.198~qa.18505.1
+      HIGHER_DPKG_VERSION: 1.4.198~qa.18505.2
+      DPKG_VERSION_FORMAT: '${Version}'
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Verify QA commit parent
+        run: test "$(git rev-parse HEAD^)" = "$EXPECTED_HEAD"
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: electron
+      - name: Build exact-head package inputs
+        run: |
+          set -euo pipefail
+          status=0
+          pnpm run build:cli || status=1
+          scripts=(build:relay build:electron-vite:parallel)
+          pids=()
+          for script in "${scripts[@]}"; do pnpm run "$script" & pids+=("$!"); done
+          for pid in "${pids[@]}"; do wait "$pid" || status=1; done
+          test "$status" -eq 0
+          pnpm run build:web-from-renderer
+          pnpm run build:native
+      - name: Build final-head and strictly-higher deb packages
+        env:
+          ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1'
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts/linux packages
+          ORCA_LOCAL_BUILD_VERSION="$FINAL_VERSION" pnpm exec electron-builder --config config/electron-builder.config.cjs --linux deb --x64 --publish never
+          mv "dist/orca-ide_${FINAL_VERSION}_amd64.deb" packages/final.deb
+          ORCA_LOCAL_BUILD_VERSION="$HIGHER_VERSION" pnpm exec electron-builder --config config/electron-builder.config.cjs --linux deb --x64 --publish never
+          mv "dist/orca-ide_${HIGHER_VERSION}_amd64.deb" packages/higher.deb
+          dpkg-deb --info packages/final.deb > artifacts/linux/final-package-info.txt
+          dpkg-deb --info packages/higher.deb > artifacts/linux/higher-package-info.txt
+          sha256sum packages/*.deb > artifacts/linux/package-sha256.txt
+          dpkg --compare-versions "$HIGHER_DPKG_VERSION" gt "$FINAL_DPKG_VERSION"
+      - name: Install and visibly launch exact final-head package
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y ./packages/final.deb
+          test "$(dpkg-query -W -f="$DPKG_VERSION_FORMAT" orca-ide)" = "$FINAL_DPKG_VERSION"
+          Xvfb :99 -screen 0 1440x900x24 >artifacts/linux/xvfb.log 2>&1 &
+          echo "$!" > artifacts/linux/xvfb-pid.txt
+          echo 'DISPLAY=:99' >> "$GITHUB_ENV"
+          DISPLAY=:99 node tests/tools/pr18505-hosted-update/linux-packaged-upgrade.mjs /opt/Orca/orca-ide "$FINAL_VERSION" "$RUNNER_TEMP/profile-linux" artifacts/linux final seed
+      - name: Upgrade through apt and visibly launch strictly higher package
+        run: |
+          set -euo pipefail
+          sudo apt-get install -y ./packages/higher.deb
+          test "$(dpkg-query -W -f="$DPKG_VERSION_FORMAT" orca-ide)" = "$HIGHER_DPKG_VERSION"
+          DISPLAY=:99 node tests/tools/pr18505-hosted-update/linux-packaged-upgrade.mjs /opt/Orca/orca-ide "$HIGHER_VERSION" "$RUNNER_TEMP/profile-linux" artifacts/linux higher reuse
+          sudo dpkg -r orca-ide
+      - name: Stop exact Xvfb PID
+        if: always()
+        run: |
+          pid="$(cat artifacts/linux/xvfb-pid.txt 2>/dev/null || true)"
+          if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then kill -TERM "$pid"; fi
+      - name: Upload Linux proof
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: pr18505-linux-packaged-upgrade
+          path: artifacts/linux/**
+          retention-days: 14
+          if-no-files-found: error
@@ -0,0 +1,173 @@
+import { chromium } from 'playwright'
+import { spawn } from 'node:child_process'
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import {
+  seedFreshProfile,
+  buildFreshProfile,
+  createSeededRepo
+} from '../win-update-e2e/onboarding-profile.mjs'
+
+const [exePathArg, expectedVersion, profileDirArg, outputDirArg, label, profileMode] =
+  process.argv.slice(2)
+if (
+  !exePathArg ||
+  !expectedVersion ||
+  !profileDirArg ||
+  !outputDirArg ||
+  !label ||
+  !['seed', 'reuse'].includes(profileMode)
+) {
+  throw new Error(
+    'usage: linux-packaged-upgrade.mjs <exe> <version> <profile> <output> <label> <seed|reuse>'
+  )
+}
+const exePath = path.resolve(exePathArg)
+const profileDir = path.resolve(profileDirArg)
+const outputDir = path.resolve(outputDirArg)
+mkdirSync(outputDir, { recursive: true })
+const isolatedHome = path.join(profileDir, 'home')
+mkdirSync(isolatedHome, { recursive: true })
+const profileDataPath = path.join(profileDir, 'orca-data.json')
+if (profileMode === 'seed') {
+  seedFreshProfile(
+    profileDir,
+    buildFreshProfile({ repo: createSeededRepo(path.join(profileDir, 'fixture-repo')) })
+  )
+} else if (!existsSync(profileDataPath)) {
+  throw new Error(`upgraded package did not preserve the existing profile at ${profileDataPath}`)
+}
+const env = {
+  ...process.env,
+  ORCA_E2E_USER_DATA_DIR: profileDir,
+  ORCA_E2E_HOME_DIR: isolatedHome,
+  ORCA_USER_DATA_PATH: profileDir,
+  ORCA_BACKGROUND_LAUNCH: '1',
+  HOME: isolatedHome,
+  ELECTRON_ENABLE_LOGGING: '1'
+}
+delete env.ELECTRON_RUN_AS_NODE
+const app = spawn(exePath, ['--no-sandbox', '--remote-debugging-port=9333'], {
+  env,
+  stdio: ['ignore', 'pipe', 'pipe']
+})
+if (!app.pid) {
+  throw new Error('packaged Linux app launch returned no PID')
+}
+const appPid = app.pid
+const stopOwnedApp = () => {
+  try {
+    process.kill(appPid, 'SIGTERM')
+  } catch {}
+}
+const isOwnedAppAlive = () => {
+  try {
+    process.kill(appPid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+process.once('exit', stopOwnedApp)
+const logs = []
+app.stdout.on('data', (chunk) => logs.push(chunk))
+app.stderr.on('data', (chunk) => logs.push(chunk))
+writeFileSync(path.join(outputDir, `${label}-pid.txt`), `${app.pid}\n`)
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+const deadline = Date.now() + 120_000
+let cdpReady = false
+while (Date.now() < deadline) {
+  try {
+    const response = await fetch('http://127.0.0.1:9333/json')
+    const targets = await response.json()
+    if (targets.some((target) => target.type === 'page')) {
+      cdpReady = true
+      break
+    }
+  } catch {}
+  await sleep(500)
+}
+if (!cdpReady) {
+  throw new Error('timed out waiting for packaged Linux CDP page')
+}
+const browser = await chromium.connectOverCDP('http://127.0.0.1:9333')
+const page = browser
+  .contexts()
+  .flatMap((context) => context.pages())
+  .find((candidate) => candidate.url().startsWith('file:'))
+if (!page) {
+  throw new Error('no packaged Orca renderer page found')
+}
+await page.waitForLoadState('domcontentloaded')
+await page.locator('body').waitFor({ state: 'visible', timeout: 60_000 })
+const runtimeVersion = await page.evaluate(() => window.api.updater.getVersion())
+if (runtimeVersion !== expectedVersion) {
+  throw new Error(`runtime version ${runtimeVersion} != ${expectedVersion}`)
+}
+let settingsSurfaceVisible = false
+try {
+  const settingsButton = page.getByRole('button', { name: 'Settings', exact: true })
+  await settingsButton.waitFor({ state: 'visible', timeout: 20_000 })
+  await settingsButton.click({ timeout: 20_000 })
+  await page
+    .getByText('Updates', { exact: true })
+    .first()
+    .waitFor({ state: 'visible', timeout: 60_000 })
+  await page
+    .getByText(`Current version: ${expectedVersion}`, { exact: true })
+    .waitFor({ state: 'visible', timeout: 30_000 })
+  settingsSurfaceVisible = true
+} catch (error) {
+  writeFileSync(
+    path.join(outputDir, `${label}-settings-unavailable.json`),
+    JSON.stringify(
+      {
+        error: String(error),
+        bodyText: (await page.locator('body').innerText()).slice(0, 8_000),
+        buttonLabels: await page.locator('button').evaluateAll((buttons) =>
+          buttons.map(
+            (button) => button.getAttribute('aria-label') ?? button.textContent?.trim() ?? ''
+          )
+        )
+      },
+      null,
+      2
+    )
+  )
+}
+await page.screenshot({ path: path.join(outputDir, `${label}-packaged.png`), fullPage: false })
+writeFileSync(
+  path.join(outputDir, `${label}-visible-state.json`),
+  JSON.stringify(
+    {
+      pid: app.pid,
+      profileMode,
+      runtimeVersion,
+      settingsSurfaceVisible,
+      title: await page.title(),
+      url: page.url(),
+      bodyText: (await page.locator('body').innerText()).slice(0, 8000)
+    },
+    null,
+    2
+  )
+)
+await browser.close()
+stopOwnedApp()
+let cleanupDeadline = Date.now() + 15_000
+while (isOwnedAppAlive() && Date.now() < cleanupDeadline) {
+  await sleep(250)
+}
+if (isOwnedAppAlive()) {
+  process.kill(appPid, 'SIGKILL')
+  cleanupDeadline = Date.now() + 5_000
+  while (isOwnedAppAlive() && Date.now() < cleanupDeadline) {
+    await sleep(250)
+  }
+}
+if (isOwnedAppAlive()) {
+  throw new Error(`packaged Linux app PID ${appPid} did not exit after exact-PID cleanup`)
+}
+process.removeListener('exit', stopOwnedApp)
+writeFileSync(path.join(outputDir, `${label}-app.log`), Buffer.concat(logs).toString('utf8'))
@@ -0,0 +1,35 @@
+import CoreGraphics
+import Foundation
+
+guard CommandLine.arguments.count == 2, let expectedPid = Int32(CommandLine.arguments[1]) else {
+  fputs("usage: mac-visible-window <pid>\n", stderr)
+  exit(2)
+}
+
+let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements]
+guard let rows = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else {
+  fputs("CGWindowListCopyWindowInfo failed\n", stderr)
+  exit(3)
+}
+
+let visible = rows.contains { row in
+  guard
+    let ownerPid = row[kCGWindowOwnerPID as String] as? NSNumber,
+    let layer = row[kCGWindowLayer as String] as? NSNumber,
+    let alpha = row[kCGWindowAlpha as String] as? NSNumber,
+    let bounds = row[kCGWindowBounds as String] as? [String: Any],
+    let width = bounds["Width"] as? NSNumber,
+    let height = bounds["Height"] as? NSNumber
+  else {
+    return false
+  }
+  return ownerPid.int32Value == expectedPid && layer.intValue == 0 && alpha.doubleValue > 0 &&
+    width.doubleValue > 0 && height.doubleValue > 0
+}
+
+if visible {
+  print("visible")
+  exit(0)
+}
+print("not-visible")
+exit(1)
@@ -0,0 +1,634 @@
+import { chromium } from 'playwright'
+import { spawn, spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import {
+  seedFreshProfile,
+  buildFreshProfile,
+  createSeededRepo
+} from '../win-update-e2e/onboarding-profile.mjs'
+import { silentInstall } from '../win-update-e2e/installer-steps.mjs'
+import { runScriptFileJson } from '../win-update-e2e/powershell-runner.mjs'
+import {
+  captureOptionalSettingsSurface,
+  createInstallLifecycleRecorder,
+  sendRuntimeRequest,
+  startInstallFileTimeline
+} from './windows-update-evidence.mjs'
+
+// Shared by disposable Windows NSIS and macOS ShipIt jobs; source is an installer or app executable respectively.
+
+const [sourceInstaller, finalTag, finalVersion, higherTag, higherVersion, outputDirArg] =
+  process.argv.slice(2)
+if (
+  !sourceInstaller ||
+  !finalTag ||
+  !finalVersion ||
+  !higherTag ||
+  !higherVersion ||
+  !outputDirArg
+) {
+  throw new Error(
+    'usage: windows-live-update.mjs <source-installer> <final-tag> <final-version> <higher-tag> <higher-version> <output-dir>'
+  )
+}
+
+const outputDir = path.resolve(outputDirArg)
+mkdirSync(outputDir, { recursive: true })
+if (process.platform !== 'win32' && process.platform !== 'darwin') {
+  throw new Error(`packaged in-app updater harness does not support ${process.platform}`)
+}
+const profileDir = path.join(outputDir, 'profile')
+const isolatedHome = path.join(profileDir, 'home')
+mkdirSync(isolatedHome, { recursive: true })
+const freshProfile = buildFreshProfile({
+  repo: createSeededRepo(path.join(profileDir, 'fixture-repo'))
+})
+freshProfile.ui = { lastUpdateCheckAt: Date.now() }
+seedFreshProfile(profileDir, freshProfile)
+
+const exePath =
+  process.platform === 'win32'
+    ? silentInstall(path.resolve(sourceInstaller)).exePath
+    : path.resolve(sourceInstaller)
+const cliPath =
+  process.platform === 'win32'
+    ? path.join(path.dirname(exePath), 'resources', 'bin', 'orca.exe')
+    : path.resolve(path.dirname(exePath), '..', 'Resources', 'bin', 'orca')
+if (!existsSync(cliPath)) {
+  throw new Error(`packaged CLI missing: ${cliPath}`)
+}
+
+const appEnv = {
+  ...process.env,
+  ORCA_E2E_USER_DATA_DIR: profileDir,
+  ORCA_E2E_HOME_DIR: isolatedHome,
+  ORCA_USER_DATA_PATH: profileDir,
+  ORCA_BACKGROUND_LAUNCH: '1',
+  HOME: isolatedHome,
+  USERPROFILE: isolatedHome,
+  ELECTRON_ENABLE_LOGGING: '1'
+}
+delete appEnv.ELECTRON_RUN_AS_NODE
+
+const appLog = path.join(outputDir, 'app.log')
+const appOut = await import('node:fs').then(({ openSync }) => openSync(appLog, 'a'))
+const runtimePath = path.join(profileDir, 'orca-runtime.json')
+const ownedPids = new Set()
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+const trafficMode = process.env.ORCA_QA_TRAFFIC_MODE ?? 'continuous'
+if (!['continuous', 'until-old-exit', 'none'].includes(trafficMode)) {
+  throw new Error(`invalid ORCA_QA_TRAFFIC_MODE: ${trafficMode}`)
+}
+
+function isPidAlive(pid) {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function stopOwnedPids() {
+  for (const pid of ownedPids) {
+    if (!isPidAlive(pid)) {
+      continue
+    }
+    try {
+      process.kill(pid, 'SIGTERM')
+    } catch {}
+  }
+}
+process.once('exit', stopOwnedPids)
+
+async function waitFor(predicate, timeoutMs, label, intervalMs = 500) {
+  const deadline = Date.now() + timeoutMs
+  let last
+  while (Date.now() < deadline) {
+    try {
+      last = await predicate()
+      if (last) {
+        return last
+      }
+    } catch (error) {
+      last = error
+    }
+    await sleep(intervalMs)
+  }
+  throw new Error(`timeout waiting for ${label}; last=${String(last)}`)
+}
+
+async function waitForRuntime(excludedRuntimeIds, label) {
+  return waitFor(
+    () => {
+      if (!existsSync(runtimePath)) {
+        return false
+      }
+      const value = JSON.parse(readFileSync(runtimePath, 'utf8'))
+      if (!Number.isInteger(value.pid) || value.pid <= 0 || typeof value.runtimeId !== 'string') {
+        return false
+      }
+      if (excludedRuntimeIds.has(value.runtimeId) || !isPidAlive(value.pid)) {
+        return false
+      }
+      return value
+    },
+    300_000,
+    label,
+    1_000
+  )
+}
+
+async function waitForCdpPage() {
+  await waitFor(
+    async () => {
+      const response = await fetch('http://127.0.0.1:9333/json')
+      const targets = await response.json()
+      return targets.some((target) => target.type === 'page')
+    },
+    120_000,
+    'CDP page'
+  )
+  const browser = await chromium.connectOverCDP('http://127.0.0.1:9333')
+  const page = await waitFor(
+    async () => {
+      return browser
+        .contexts()
+        .flatMap((context) => context.pages())
+        .find((candidate) => {
+          try {
+            return new URL(candidate.url()).pathname.endsWith('/index.html')
+          } catch {
+            return false
+          }
+        })
+    },
+    60_000,
+    'Orca renderer page'
+  )
+  await page.waitForLoadState('domcontentloaded')
+  await waitFor(
+    () =>
+      page
+        .evaluate(() => window.api.updater.getVersion())
+        .then((version) => (typeof version === 'string' && version.length > 0 ? version : false)),
+    120_000,
+    'packaged updater renderer bridge'
+  )
+  return { browser, page }
+}
+
+async function launchDriven(label, excludedRuntimeIds) {
+  await waitFor(
+    async () => {
+      try {
+        await fetch('http://127.0.0.1:9333/json')
+        return false
+      } catch {
+        return true
+      }
+    },
+    30_000,
+    'previous CDP listener exit'
+  )
+
+  const launcher = spawn(exePath, ['--remote-debugging-port=9333'], {
+    env: appEnv,
+    stdio: ['ignore', appOut, appOut]
+  })
+  if (!launcher.pid) {
+    throw new Error(`${label} packaged app launch returned no PID`)
+  }
+  ownedPids.add(launcher.pid)
+  launcher.once('exit', () => ownedPids.delete(launcher.pid))
+  writeFileSync(path.join(outputDir, `${label}-launcher-pid.txt`), `${launcher.pid}\n`)
+
+  const [{ browser, page }, runtime] = await Promise.all([
+    waitForCdpPage(),
+    waitForRuntime(excludedRuntimeIds, `${label} authoritative runtime`)
+  ])
+  await waitFor(() => hasVisibleWindow(runtime.pid), 60_000, `${label} visible top-level window`)
+  const settingsVisible = await captureOptionalSettingsSurface(page, outputDir, label)
+  ownedPids.add(runtime.pid)
+  writeFileSync(path.join(outputDir, `${label}-pid.txt`), `${runtime.pid}\n`)
+  return { browser, page, runtime, settingsVisible }
+}
+
+async function terminateExactRuntime(session, label) {
+  const disconnected = session.browser?.isConnected()
+    ? new Promise((resolve) => session.browser.once('disconnected', resolve))
+    : Promise.resolve()
+  if (isPidAlive(session.runtime.pid)) {
+    process.kill(session.runtime.pid, 'SIGTERM')
+  }
+  await waitFor(() => !isPidAlive(session.runtime.pid), 30_000, `${label} exact PID exit`)
+  ownedPids.delete(session.runtime.pid)
+  await Promise.race([disconnected, sleep(10_000)])
+}
+
+function readVisibleWindows() {
+  const snapshot = runScriptFileJson(path.resolve('tests/tools/win-update-e2e/window-enum.ps1'))
+  if (Array.isArray(snapshot?.windows)) {
+    return snapshot.windows
+  }
+  return snapshot?.windows ? [snapshot.windows] : []
+}
+
+function hasVisibleWindow(pid) {
+  if (process.platform === 'win32') {
+    return readVisibleWindows().some((entry) => Number(entry.pid) === pid)
+  }
+  const probe = process.env.ORCA_QA_MAC_WINDOW_PROBE
+  if (!probe) {
+    throw new Error('ORCA_QA_MAC_WINDOW_PROBE is required on macOS')
+  }
+  const result = spawnSync(probe, [String(pid)], { encoding: 'utf8', timeout: 30_000 })
+  if (result.error) {
+    throw result.error
+  }
+  return result.status === 0 && result.stdout.trim() === 'visible'
+}
+
+function invokeBounded(args, lane, trafficLog, trafficResults) {
+  return new Promise((resolve) => {
+    const startedAt = Date.now()
+    const child = spawn(cliPath, args, { env: appEnv, stdio: ['ignore', 'pipe', 'pipe'] })
+    if (child.pid) {
+      ownedPids.add(child.pid)
+    }
+    const chunks = []
+    child.stdout.on('data', (chunk) => chunks.push(chunk))
+    child.stderr.on('data', (chunk) => chunks.push(chunk))
+    let settled = false
+    let timer
+    const finish = (result) => {
+      if (settled) {
+        return
+      }
+      settled = true
+      clearTimeout(timer)
+      if (child.pid) {
+        ownedPids.delete(child.pid)
+      }
+      trafficResults[lane].calls += 1
+      if (result.code === 0) {
+        trafficResults[lane].successes += 1
+      }
+      writeFileSync(
+        trafficLog,
+        `${JSON.stringify({ lane, args, pid: child.pid, startedAt, finishedAt: Date.now(), durationMs: Date.now() - startedAt, ...result, output: Buffer.concat(chunks).toString('utf8').slice(0, 2000) })}\n`,
+        { flag: 'a' }
+      )
+      resolve(result)
+    }
+    timer = setTimeout(() => {
+      try {
+        child.kill()
+      } catch {}
+    }, 10_000)
+    child.once('exit', (code, signal) => finish({ code, signal }))
+    child.once('error', (error) => finish({ error: error.message }))
+  })
+}
+
+async function startTraffic(label) {
+  const trafficLog = path.join(outputDir, `${label}-cli-traffic.jsonl`)
+  const trafficResults = {
+    status: { calls: 0, successes: 0 },
+    orchestration: { calls: 0, successes: 0 }
+  }
+  if (trafficMode === 'none') {
+    writeFileSync(
+      trafficLog,
+      `${JSON.stringify({ at: Date.now(), trafficMode, event: 'traffic-disabled' })}\n`
+    )
+    return {
+      trafficResults,
+      async stop() {}
+    }
+  }
+  const commands = [
+    ['status', ['status', '--json']],
+    ['orchestration', ['orchestration', 'run-list', '--limit', '5', '--json']]
+  ]
+  const warmup = await Promise.all(
+    commands.map(([lane, args]) => invokeBounded(args, lane, trafficLog, trafficResults))
+  )
+  if (warmup.some((result) => result.code !== 0)) {
+    throw new Error(`${label} CLI traffic warmup failed: ${JSON.stringify(warmup)}`)
+  }
+
+  let stop = false
+  const lanes = commands.map(async ([lane, args]) => {
+    const deadline = Date.now() + 120_000
+    while (!stop && Date.now() < deadline) {
+      await invokeBounded(args, lane, trafficLog, trafficResults)
+      if (!stop) {
+        await sleep(750)
+      }
+    }
+  })
+  let stopPromise
+  return {
+    trafficResults,
+    async stop() {
+      stopPromise ??= (async () => {
+        stop = true
+        await Promise.all(lanes)
+        if (
+          trafficResults.status.calls < 2 ||
+          trafficResults.orchestration.calls < 2 ||
+          trafficResults.status.successes < 1 ||
+          trafficResults.orchestration.successes < 1
+        ) {
+          throw new Error(`${label} bounded CLI traffic failed: ${JSON.stringify(trafficResults)}`)
+        }
+      })()
+      await stopPromise
+    }
+  }
+}
+
+async function assertRuntimeVersion(runtime, expectedVersion, label) {
+  const statusResult = await waitFor(
+    () => {
+      const result = spawnSync(cliPath, ['status', '--json'], {
+        env: appEnv,
+        encoding: 'utf8',
+        timeout: 10_000
+      })
+      if (result.status !== 0 || !result.stdout.trim()) {
+        return false
+      }
+      const parsed = JSON.parse(result.stdout)
+      return parsed?.result?.runtime?.appVersion === expectedVersion &&
+        parsed?.result?.app?.pid === runtime.pid &&
+        parsed?.result?.app?.desktopWindowStatus === 'available'
+        ? parsed
+        : false
+    },
+    120_000,
+    `${label} target runtime CLI status`,
+    1_000
+  )
+
+  const cliVersion = spawnSync(cliPath, ['--version'], {
+    env: appEnv,
+    encoding: 'utf8',
+    timeout: 10_000
+  })
+  if (cliVersion.status !== 0 || cliVersion.stdout.trim() !== expectedVersion) {
+    throw new Error(
+      `${label} packaged CLI version mismatch: ${cliVersion.stdout} ${cliVersion.stderr}`
+    )
+  }
+  const replacementVisible = hasVisibleWindow(runtime.pid)
+  if (!replacementVisible) {
+    throw new Error(
+      `${label} automatic replacement PID ${runtime.pid} has no visible top-level window`
+    )
+  }
+  return { statusResult, cliVersion: cliVersion.stdout.trim(), replacementVisible }
+}
+
+async function runHop(session, targetTag, targetVersion, label, seenRuntimeIds) {
+  const beforeVersion = await session.page.evaluate(() => window.api.updater.getVersion())
+  if (beforeVersion === targetVersion) {
+    throw new Error(`${label} source and target versions are identical: ${targetVersion}`)
+  }
+  const beforeStatus = await session.page.evaluate(() => window.api.updater.getStatus())
+  if (beforeStatus.state !== 'idle') {
+    throw new Error(`${label} started in non-idle updater state: ${JSON.stringify(beforeStatus)}`)
+  }
+  await session.page.screenshot({
+    path: path.join(
+      outputDir,
+      `${label}-01-source-${session.settingsVisible ? 'idle' : 'visible-app'}.png`
+    ),
+    fullPage: false
+  })
+
+  await session.page.evaluate(
+    ({ targetTag: tag }) => window.api.updater.check({ channel: 'adhoc', targetTag: tag }),
+    { targetTag }
+  )
+  const available = await waitFor(
+    () =>
+      session.page
+        .evaluate(() => window.api.updater.getStatus())
+        .then((status) => (status.state === 'available' ? status : false)),
+    120_000,
+    `${label} pinned update available`,
+    1_000
+  )
+  if (available.version !== targetVersion || available.source !== 'adhoc') {
+    throw new Error(`${label} unexpected available status: ${JSON.stringify(available)}`)
+  }
+  await session.page.screenshot({
+    path: path.join(outputDir, `${label}-02-update-available.png`),
+    fullPage: false
+  })
+
+  await session.page.evaluate(() => window.api.updater.download())
+  const downloaded = await waitFor(
+    () =>
+      session.page
+        .evaluate(() => window.api.updater.getStatus())
+        .then((status) => (status.state === 'downloaded' ? status : false)),
+    300_000,
+    `${label} pinned update downloaded`,
+    1_000
+  )
+  if (downloaded.version !== targetVersion) {
+    throw new Error(`${label} wrong downloaded version: ${JSON.stringify(downloaded)}`)
+  }
+  await session.page.screenshot({
+    path: path.join(outputDir, `${label}-03-update-downloaded.png`),
+    fullPage: false
+  })
+
+  const timeline = startInstallFileTimeline({
+    label,
+    outputDir,
+    sourcePid: session.runtime.pid,
+    exePath,
+    cliPath,
+    runtimePath
+  })
+  const traffic = await startTraffic(label)
+  const recordLifecycle = createInstallLifecycleRecorder({ label, outputDir, trafficMode })
+  let trafficStoppedAt = null
+  const stopTraffic = async () => {
+    if (trafficStoppedAt !== null) {
+      return
+    }
+    await traffic.stop()
+    trafficStoppedAt = Date.now()
+    recordLifecycle('traffic-stopped', { trafficStoppedAt })
+  }
+  let completed = false
+  try {
+    const disconnected = new Promise((resolve) => session.browser.once('disconnected', resolve))
+    const quitRequestedAt = Date.now()
+    recordLifecycle('quit-requested', { quitRequestedAt, sourcePid: session.runtime.pid })
+    await session.page.evaluate(() => window.api.updater.quitAndInstall())
+    const oldExitTask = (async () => {
+      await waitFor(
+        () => !isPidAlive(session.runtime.pid),
+        120_000,
+        `${label} old authoritative PID exit`
+      )
+      const exitedAt = Date.now()
+      recordLifecycle('old-pid-exited', { oldExitedAt: exitedAt, sourcePid: session.runtime.pid })
+      ownedPids.delete(session.runtime.pid)
+      if (trafficMode === 'until-old-exit') {
+        await stopTraffic()
+      }
+      return exitedAt
+    })()
+    await Promise.race([
+      disconnected,
+      sleep(120_000).then(() => {
+        throw new Error(`${label} initial CDP session did not disconnect for update`)
+      })
+    ])
+    const cdpDisconnectedAt = Date.now()
+    recordLifecycle('cdp-disconnected', { cdpDisconnectedAt })
+    const oldExitedAt = await oldExitTask
+
+    const replacement = await waitForRuntime(seenRuntimeIds, `${label} automatic replacement runtime`)
+    const replacementObservedAt = Date.now()
+    recordLifecycle('replacement-observed', {
+      replacementObservedAt,
+      replacementPid: replacement.pid,
+      replacementRuntimeId: replacement.runtimeId
+    })
+    ownedPids.add(replacement.pid)
+    seenRuntimeIds.add(replacement.runtimeId)
+    writeFileSync(path.join(outputDir, `${label}-replacement-pid.txt`), `${replacement.pid}\n`)
+    const runtimeEvidence = await assertRuntimeVersion(replacement, targetVersion, label)
+    const automaticUpdaterStatus = await sendRuntimeRequest(
+      replacement,
+      'updater.getStatus',
+      null
+    )
+    if (automaticUpdaterStatus?.state !== 'idle') {
+      throw new Error(
+        `${label} automatic replacement updater not idle: ${JSON.stringify(automaticUpdaterStatus)}`
+      )
+    }
+    await stopTraffic()
+    completed = true
+
+    return {
+      replacementRuntime: replacement,
+      evidence: {
+        beforeVersion,
+        settingsSurfaceVisible: session.settingsVisible,
+        targetTag,
+        targetVersion,
+        trafficMode,
+        quitRequestedAt,
+        cdpDisconnectedAt,
+        oldExitedAt,
+        trafficStoppedAt,
+        replacementObservedAt,
+        automaticUpdaterStatus,
+        replacement: {
+          pid: replacement.pid,
+          runtimeId: replacement.runtimeId,
+          startedAt: replacement.startedAt
+        },
+        ...runtimeEvidence,
+        trafficResults: traffic.trafficResults
+      }
+    }
+  } finally {
+    await timeline.stop()
+    if (!completed) {
+      await stopTraffic()
+    }
+  }
+}
+
+const seenRuntimeIds = new Set()
+let session = await launchDriven('source', seenRuntimeIds)
+seenRuntimeIds.add(session.runtime.runtimeId)
+const sourceVersion = await session.page.evaluate(() => window.api.updater.getVersion())
+
+const firstHopResult = await runHop(session, finalTag, finalVersion, 'hop-1', seenRuntimeIds)
+if (process.env.ORCA_QA_SINGLE_HOP === '1') {
+  writeFileSync(
+    path.join(outputDir, 'result.json'),
+    JSON.stringify({ sourceVersion, firstHop: firstHopResult.evidence }, null, 2)
+  )
+  await terminateExactRuntime(
+    { browser: session.browser, runtime: firstHopResult.replacementRuntime },
+    'hop-1 automatic replacement'
+  )
+} else {
+  await terminateExactRuntime(
+    { browser: session.browser, runtime: firstHopResult.replacementRuntime },
+    'hop-1 replacement'
+  )
+  session = await launchDriven('final-head', seenRuntimeIds)
+  seenRuntimeIds.add(session.runtime.runtimeId)
+  const finalHeadVersion = await session.page.evaluate(() => window.api.updater.getVersion())
+  if (finalHeadVersion !== finalVersion) {
+    throw new Error(`final-head relaunch version ${finalHeadVersion} != ${finalVersion}`)
+  }
+  const finalHeadIdleStatus = await session.page.evaluate(() => window.api.updater.getStatus())
+  if (finalHeadIdleStatus.state !== 'idle') {
+    throw new Error(`final-head relaunch updater not idle: ${JSON.stringify(finalHeadIdleStatus)}`)
+  }
+  await session.page.screenshot({
+    path: path.join(outputDir, 'hop-1-04-final-head-idle.png'),
+    fullPage: false
+  })
+
+  const secondHopResult = await runHop(session, higherTag, higherVersion, 'hop-2', seenRuntimeIds)
+  await terminateExactRuntime(
+    { browser: session.browser, runtime: secondHopResult.replacementRuntime },
+    'hop-2 replacement'
+  )
+  session = await launchDriven('higher-final-head', seenRuntimeIds)
+  seenRuntimeIds.add(session.runtime.runtimeId)
+  const higherRuntimeVersion = await session.page.evaluate(() => window.api.updater.getVersion())
+  if (higherRuntimeVersion !== higherVersion) {
+    throw new Error(`higher-version relaunch version ${higherRuntimeVersion} != ${higherVersion}`)
+  }
+  const higherIdleStatus = await session.page.evaluate(() => window.api.updater.getStatus())
+  if (higherIdleStatus.state !== 'idle') {
+    throw new Error(`higher-version relaunch updater not idle: ${JSON.stringify(higherIdleStatus)}`)
+  }
+  if (session.settingsVisible) {
+    await session.page.getByText(`Current version: ${higherVersion}`, { exact: true }).waitFor({
+      state: 'visible',
+      timeout: 30_000
+    })
+  }
+  await session.page.screenshot({
+    path: path.join(outputDir, 'hop-2-04-higher-final-head-idle.png'),
+    fullPage: false
+  })
+
+  writeFileSync(
+    path.join(outputDir, 'result.json'),
+    JSON.stringify(
+      {
+        sourceVersion,
+        finalHeadVersion,
+        higherRuntimeVersion,
+        finalHeadIdleStatus,
+        higherIdleStatus,
+        firstHop: firstHopResult.evidence,
+        secondHop: secondHopResult.evidence
+      },
+      null,
+      2
+    )
+  )
+
+  await terminateExactRuntime(session, 'final driven runtime')
+}
@@ -0,0 +1,152 @@
+import { spawn, spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { chromium } from 'playwright'
+import { locateInstalledExe } from '../win-update-e2e/installer-steps.mjs'
+import { runScriptFileJson } from '../win-update-e2e/powershell-runner.mjs'
+
+const [expectedVersion, profileDirArg, outputDirArg] = process.argv.slice(2)
+if (!expectedVersion || !profileDirArg || !outputDirArg || process.platform !== 'win32') {
+  throw new Error('usage: windows-manual-recovery.mjs <version> <profile> <output> (Windows only)')
+}
+const profileDir = path.resolve(profileDirArg)
+const outputDir = path.resolve(outputDirArg)
+mkdirSync(outputDir, { recursive: true })
+const exePath = locateInstalledExe()
+if (!exePath) {
+  throw new Error('updated Orca.exe is not installed')
+}
+const cliPath = path.join(path.dirname(exePath), 'resources', 'bin', 'orca.exe')
+const runtimePath = path.join(profileDir, 'orca-runtime.json')
+const isolatedHome = path.join(profileDir, 'home')
+const appEnv = {
+  ...process.env,
+  ORCA_E2E_USER_DATA_DIR: profileDir,
+  ORCA_E2E_HOME_DIR: isolatedHome,
+  ORCA_USER_DATA_PATH: profileDir,
+  ORCA_BACKGROUND_LAUNCH: '1',
+  HOME: isolatedHome,
+  USERPROFILE: isolatedHome,
+  ELECTRON_ENABLE_LOGGING: '1'
+}
+delete appEnv.ELECTRON_RUN_AS_NODE
+const ownedPids = new Set()
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+function isPidAlive(pid) {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function stopOwnedPids() {
+  for (const pid of ownedPids) {
+    if (!isPidAlive(pid)) {
+      continue
+    }
+    try {
+      process.kill(pid, 'SIGTERM')
+    } catch {}
+  }
+}
+process.once('exit', stopOwnedPids)
+
+async function waitFor(predicate, timeoutMs, label) {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    try {
+      const value = await predicate()
+      if (value) {
+        return value
+      }
+    } catch {}
+    await sleep(500)
+  }
+  throw new Error(`timeout waiting for ${label}`)
+}
+
+const recoveryLog = path.join(outputDir, 'manual-recovery-app.log')
+const appOut = await import('node:fs').then(({ openSync }) => openSync(recoveryLog, 'a'))
+const launcher = spawn(exePath, ['--remote-debugging-port=9333'], {
+  env: appEnv,
+  stdio: ['ignore', appOut, appOut]
+})
+if (!launcher.pid) {
+  throw new Error('manual recovery launch returned no PID')
+}
+ownedPids.add(launcher.pid)
+launcher.once('exit', () => ownedPids.delete(launcher.pid))
+writeFileSync(path.join(outputDir, 'manual-recovery-launcher-pid.txt'), `${launcher.pid}\n`)
+
+const runtime = await waitFor(() => {
+  if (!existsSync(runtimePath)) {
+    return false
+  }
+  const value = JSON.parse(readFileSync(runtimePath, 'utf8'))
+  return Number.isInteger(value.pid) && isPidAlive(value.pid) ? value : false
+}, 120_000, 'manual recovery runtime')
+ownedPids.add(runtime.pid)
+
+await waitFor(async () => {
+  const response = await fetch('http://127.0.0.1:9333/json')
+  const targets = await response.json()
+  return targets.some((target) => target.type === 'page')
+}, 120_000, 'manual recovery CDP page')
+const browser = await chromium.connectOverCDP('http://127.0.0.1:9333')
+const page = await waitFor(() => {
+  return browser
+    .contexts()
+    .flatMap((context) => context.pages())
+    .find((candidate) => candidate.url().startsWith('file:'))
+}, 60_000, 'manual recovery renderer')
+await page.waitForLoadState('domcontentloaded')
+const [runtimeVersion, updaterStatus] = await Promise.all([
+  page.evaluate(() => window.api.updater.getVersion()),
+  page.evaluate(() => window.api.updater.getStatus())
+])
+if (runtimeVersion !== expectedVersion || updaterStatus.state !== 'idle') {
+  throw new Error(
+    `manual recovery state mismatch: ${JSON.stringify({ runtimeVersion, updaterStatus })}`
+  )
+}
+const cliVersion = spawnSync(cliPath, ['--version'], {
+  env: appEnv,
+  encoding: 'utf8',
+  timeout: 10_000
+})
+if (cliVersion.status !== 0 || cliVersion.stdout.trim() !== expectedVersion) {
+  throw new Error(`manual recovery CLI mismatch: ${cliVersion.stdout} ${cliVersion.stderr}`)
+}
+const windows = runScriptFileJson(path.resolve('tests/tools/win-update-e2e/window-enum.ps1'))
+const visible = (Array.isArray(windows?.windows) ? windows.windows : [windows?.windows]).some(
+  (entry) => Number(entry?.pid) === runtime.pid
+)
+if (!visible) {
+  throw new Error(`manual recovery PID ${runtime.pid} has no visible top-level window`)
+}
+await page.screenshot({ path: path.join(outputDir, 'manual-recovery-idle.png'), fullPage: false })
+writeFileSync(
+  path.join(outputDir, 'manual-recovery-result.json'),
+  JSON.stringify(
+    {
+      expectedVersion,
+      runtimeVersion,
+      updaterStatus,
+      cliVersion: cliVersion.stdout.trim(),
+      exePath,
+      launcherPid: launcher.pid,
+      runtimePid: runtime.pid,
+      runtimeId: runtime.runtimeId,
+      visible
+    },
+    null,
+    2
+  )
+)
+await browser.close()
+stopOwnedPids()
+await waitFor(() => [...ownedPids].every((pid) => !isPidAlive(pid)), 30_000, 'owned PID exit')
+process.removeListener('exit', stopOwnedPids)
@@ -0,0 +1,160 @@
+import { randomUUID } from 'node:crypto'
+import { statSync, writeFileSync } from 'node:fs'
+import { createConnection } from 'node:net'
+import path from 'node:path'
+
+export function sendRuntimeRequest(metadata, method, params) {
+  const transport =
+    metadata.transports?.find((entry) => ['unix', 'named-pipe'].includes(entry.kind)) ??
+    metadata.transport
+  if (!transport?.endpoint || !metadata.authToken) {
+    throw new Error('runtime metadata has no compatible local transport')
+  }
+  return new Promise((resolve, reject) => {
+    const id = randomUUID()
+    const socket = createConnection(transport.endpoint)
+    let buffer = ''
+    let settled = false
+    const timer = setTimeout(() => finish(new Error(`runtime RPC ${method} timed out`)), 30_000)
+    function finish(error, value) {
+      if (settled) {
+        return
+      }
+      settled = true
+      clearTimeout(timer)
+      socket.end()
+      if (error) {
+        reject(error)
+      } else {
+        resolve(value)
+      }
+    }
+    socket.setEncoding('utf8')
+    socket.once('error', (error) => finish(error))
+    socket.once('close', () => finish(new Error(`runtime RPC ${method} closed before response`)))
+    socket.on('data', (chunk) => {
+      buffer += chunk
+      let newline = buffer.indexOf('\n')
+      while (newline !== -1 && !settled) {
+        const line = buffer.slice(0, newline)
+        buffer = buffer.slice(newline + 1)
+        newline = buffer.indexOf('\n')
+        if (!line.trim()) {
+          continue
+        }
+        const frame = JSON.parse(line)
+        if (frame._keepalive === true) {
+          continue
+        }
+        if (frame.id !== id) {
+          return finish(new Error(`runtime RPC ${method} response ID mismatch`))
+        }
+        if (frame.ok !== true) {
+          return finish(new Error(`runtime RPC ${method} failed: ${JSON.stringify(frame.error)}`))
+        }
+        return finish(null, frame.result)
+      }
+    })
+    socket.once('connect', () =>
+      socket.write(`${JSON.stringify({ id, authToken: metadata.authToken, method, params })}\n`)
+    )
+  })
+}
+
+function snapshotFile(filePath) {
+  try {
+    const stat = statSync(filePath)
+    return { exists: true, size: stat.size, mtimeMs: stat.mtimeMs }
+  } catch {
+    return { exists: false }
+  }
+}
+
+function isPidAlive(pid) {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch {
+    return false
+  }
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+export function createInstallLifecycleRecorder({ label, outputDir, trafficMode }) {
+  const lifecyclePath = path.join(outputDir, `${label}-install-lifecycle.jsonl`)
+  return (event, extra = {}) => {
+    writeFileSync(
+      lifecyclePath,
+      `${JSON.stringify({ event, at: Date.now(), trafficMode, ...extra })}\n`,
+      { flag: 'a' }
+    )
+  }
+}
+
+export async function captureOptionalSettingsSurface(page, outputDir, label) {
+  const settingsButton = page.getByRole('button', { name: 'Settings', exact: true })
+  try {
+    await settingsButton.waitFor({ state: 'visible', timeout: 20_000 })
+    await settingsButton.click({ timeout: 20_000 })
+    await page.getByText('Updates', { exact: true }).first().waitFor({
+      state: 'visible',
+      timeout: 60_000
+    })
+    return true
+  } catch (error) {
+    const diagnostics = await page.evaluate(() => ({
+      bodyText: document.body?.innerText.slice(0, 8_000),
+      buttonLabels: Array.from(document.querySelectorAll('button')).map(
+        (button) => button.getAttribute('aria-label') ?? button.textContent?.trim() ?? ''
+      ),
+      title: document.title,
+      url: location.href,
+      visibilityState: document.visibilityState,
+      viewport: { width: innerWidth, height: innerHeight }
+    }))
+    writeFileSync(
+      path.join(outputDir, `${label}-settings-unavailable.json`),
+      JSON.stringify({ error: String(error), diagnostics }, null, 2)
+    )
+    await page.screenshot({
+      path: path.join(outputDir, `${label}-visible-app.png`),
+      fullPage: false
+    })
+    return false
+  }
+}
+
+export function startInstallFileTimeline({
+  label,
+  outputDir,
+  sourcePid,
+  exePath,
+  cliPath,
+  runtimePath
+}) {
+  const timelinePath = path.join(outputDir, `${label}-install-file-timeline.jsonl`)
+  let stopped = false
+  const task = (async () => {
+    while (!stopped) {
+      writeFileSync(
+        timelinePath,
+        `${JSON.stringify({
+          at: Date.now(),
+          sourcePidAlive: isPidAlive(sourcePid),
+          appExecutable: snapshotFile(exePath),
+          cliExecutable: snapshotFile(cliPath),
+          runtimeMetadata: snapshotFile(runtimePath)
+        })}\n`,
+        { flag: 'a' }
+      )
+      await sleep(250)
+    }
+  })()
+  return {
+    async stop() {
+      stopped = true
+      await task
+    }
+  }
+}