mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 00:03:46 +00:00
Merge remote-tracking branch 'origin/main' into brennanb2025/i18n-catalog-decoupling
This commit is contained in:
Brennan Benson
@@ -6,6 +6,10 @@ inputs:
|
||||
description: Native runtime to prepare after the script-free install (none, node, or electron).
|
||||
required: false
|
||||
default: none
|
||||
node-version:
|
||||
description: Node.js version override; defaults to the version declared in package.json.
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
@@ -17,11 +21,19 @@ runs:
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node.js
|
||||
if: inputs.node-version == ''
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: package.json
|
||||
cache: pnpm
|
||||
|
||||
- name: Setup requested Node.js
|
||||
if: inputs.node-version != ''
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ inputs.node-version }}
|
||||
cache: pnpm
|
||||
|
||||
- name: Validate native runtime
|
||||
shell: bash
|
||||
env:
|
||||
|
||||
@@ -184,12 +184,14 @@ jobs:
|
||||
src/shared/posix-command-path-lookup.test.ts
|
||||
|
||||
test:
|
||||
name: tests ${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
name: tests node ${{ matrix.node }} ${{ matrix.shard }}/${{ matrix.shard_total }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: ['24', '26']
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
|
||||
shard_total: [16]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -200,6 +202,7 @@ jobs:
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
node-version: ${{ matrix.node }}
|
||||
|
||||
- name: Install Electron package binary for tests
|
||||
run: node config/scripts/install-electron-package-binary.mjs
|
||||
@@ -213,7 +216,7 @@ jobs:
|
||||
--exclude=src/main/providers/__tests__/shell-ready-framework-example.test.ts \
|
||||
--exclude=src/main/pty/omp-shell-wrapper.node-pty.test.ts \
|
||||
--exclude=src/shared/posix-command-path-lookup.test.ts \
|
||||
--shard=${{ matrix.shard }}/${{ strategy.job-total }}
|
||||
--shard=${{ matrix.shard }}/${{ matrix.shard_total }}
|
||||
|
||||
package:
|
||||
name: package
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
name: Terminal IME E2E
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/terminal-ime-e2e.yml'
|
||||
- 'config/patches/@xterm__xterm@6.1.0-beta.287.patch'
|
||||
- 'config/scripts/run-terminal-ibus-hangul-e2e.mjs'
|
||||
- 'config/scripts/terminal-ime-e2e-workflow.test.mjs'
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
- 'src/renderer/src/components/terminal-pane/keyboard-handlers.ts'
|
||||
- 'src/renderer/src/components/terminal-pane/keyboard-handlers-ime.test.tsx'
|
||||
- 'src/renderer/src/components/terminal-pane/pty-connection.ts'
|
||||
- 'src/renderer/src/components/terminal-pane/pty-connection.test.ts'
|
||||
- 'src/renderer/src/components/terminal-pane/terminal-ime-*'
|
||||
- 'src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts'
|
||||
- 'src/renderer/src/components/terminal-pane/xterm-bypass-policy.ts'
|
||||
- 'src/renderer/src/components/terminal-pane/xterm-bypass-policy.test.ts'
|
||||
- 'tests/e2e/chinese-ime-chat-input-repro.spec.ts'
|
||||
- 'tests/e2e/korean-ime-terminal-shift-enter-commit.spec.ts'
|
||||
- 'tests/e2e/terminal-ibus-hangul-native.spec.ts'
|
||||
- 'tests/e2e/terminal-ime-*.ts'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '30 9 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux-x11:
|
||||
name: Linux X11 terminal IME
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install native build and IME tools
|
||||
run: >-
|
||||
sudo apt-get update &&
|
||||
sudo apt-get install -y
|
||||
build-essential
|
||||
dbus-x11
|
||||
dconf-gsettings-backend
|
||||
ibus
|
||||
ibus-hangul
|
||||
libglib2.0-bin
|
||||
python3
|
||||
xdotool
|
||||
xfwm4
|
||||
xvfb
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: package.json
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
run_install: false
|
||||
|
||||
- name: Use external node-gyp to avoid pnpm bundled copy
|
||||
run: |
|
||||
npm install -g node-gyp@11.5.0
|
||||
echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build Electron app for E2E
|
||||
run: pnpm exec electron-vite build --mode e2e
|
||||
|
||||
- name: Run deterministic terminal IME boundary tests
|
||||
run: >-
|
||||
xvfb-run --auto-servernum
|
||||
env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
|
||||
pnpm run test:e2e --
|
||||
tests/e2e/terminal-ime-exact-byte.spec.ts
|
||||
--workers=1
|
||||
|
||||
- name: Run native IBus Hangul exact-byte tests
|
||||
env:
|
||||
SKIP_BUILD: '1'
|
||||
run: pnpm run test:e2e:terminal-ime-native
|
||||
|
||||
- name: Upload terminal IME evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: terminal-ime-evidence
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
@@ -128,7 +128,6 @@ jobs:
|
||||
node tools/win-update-e2e/run.mjs `
|
||||
--from "$env:FROM_EXE" `
|
||||
--to "$env:TO_EXE" `
|
||||
--require-distinct-artifacts `
|
||||
--expect "$env:EXPECT" `
|
||||
--soak-seconds "$env:SOAK_SECONDS" 2>&1 | Tee-Object -FilePath $log
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
|
||||
Monitor and steer your agents from your phone — get notified when an agent finishes and send follow-ups from anywhere.
|
||||
|
||||
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.32](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.32/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
|
||||
[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.36](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.36/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
|
||||
|
||||
</td>
|
||||
<td width="50%">
|
||||
@@ -230,7 +230,7 @@ yay -S stably-orca-bin
|
||||
Pair with your desktop app to monitor and steer your agents from your phone.
|
||||
|
||||
- **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) or [join TestFlight](https://testflight.apple.com/join/YjeGMQBA)
|
||||
- **Android:** [Download APK 0.0.32](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.32/app-release.apk)
|
||||
- **Android:** [Download APK 0.0.36](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.36/app-release.apk)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -21,7 +21,6 @@ type OutputChunk = Rollup.OutputChunk
|
||||
const PLAIN_NODE_ENTRY_NAMES = [
|
||||
'daemon-entry',
|
||||
'parcel-watcher-process-entry',
|
||||
'main-thread-hang-watchdog-entry',
|
||||
'computer-sidecar',
|
||||
'agent-hooks/managed-agent-hook-controls',
|
||||
'codex/codex-app-server-grant-entry'
|
||||
|
||||
@@ -153,7 +153,6 @@ module.exports = {
|
||||
'out/main/plugin-host-entry.js',
|
||||
'out/main/computer-sidecar.js',
|
||||
'out/main/parcel-watcher-process-entry.js',
|
||||
'out/main/main-thread-hang-watchdog-entry.js',
|
||||
'out/main/chunks/**',
|
||||
'resources/**',
|
||||
'node_modules/ws/**',
|
||||
|
||||
@@ -13,26 +13,6 @@
|
||||
; The image name and the LOCALAPPDATA folder name must stay in sync with
|
||||
; DAEMON_HOST_EXE_NAME and LOCAL_HOST_ROOT_NAME in
|
||||
; src/main/daemon/daemon-host-relocation.ts.
|
||||
|
||||
!include "getProcessInfo.nsh"
|
||||
Var pid
|
||||
|
||||
!macro customCheckAppRunning
|
||||
${if} ${isUpdated}
|
||||
; Why: an A-era packaged CLI may still hold Orca.exe while its exact native launcher waits.
|
||||
nsExec::Exec `"$PowerShellPath" -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "& { $$ErrorActionPreference='Stop'; $$app=[IO.Path]::GetFullPath($$args[0]); $$launcher=[IO.Path]::GetFullPath($$args[1]); $$comparison=[StringComparison]::OrdinalIgnoreCase; $$children=@(Get-CimInstance Win32_Process | Where-Object { $$_.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$_.ExecutablePath), $$app, $$comparison) }); $$launcherProofs=@(); foreach ($$child in $$children) { $$parent=Get-CimInstance Win32_Process -Filter ('ProcessId=' + $$child.ParentProcessId) -ErrorAction SilentlyContinue; if ($$parent -and $$parent.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$parent.ExecutablePath), $$launcher, $$comparison)) { $$launcherProofs += [pscustomobject]@{ ProcessId=[int]$$parent.ProcessId; CreationDate=[string]$$parent.CreationDate } }; Stop-Process -Id $$child.ProcessId -Force -ErrorAction SilentlyContinue }; $$launcherDeadline=[DateTime]::UtcNow.AddSeconds(30); do { $$remaining=@($$launcherProofs | Where-Object { $$proof=$$_; $$current=Get-CimInstance Win32_Process -Filter ('ProcessId=' + $$proof.ProcessId) -ErrorAction SilentlyContinue; $$current -and $$current.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$current.ExecutablePath), $$launcher, $$comparison) -and [String]::Equals([string]$$current.CreationDate, $$proof.CreationDate, [StringComparison]::Ordinal) }); if ($$remaining.Count -eq 0) { break }; Start-Sleep -Milliseconds 100 } while ([DateTime]::UtcNow -lt $$launcherDeadline); if ($$remaining.Count -ne 0) { exit 20 }; $$lockDeadline=[DateTime]::UtcNow.AddSeconds(30); $$released=$$false; do { try { $$stream=[IO.File]::Open($$app, [IO.FileMode]::Open, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None); $$stream.Dispose(); $$released=$$true } catch { Start-Sleep -Milliseconds 100 } } while (-not $$released -and [DateTime]::UtcNow -lt $$lockDeadline); if (-not $$released) { exit 21 } }" "$INSTDIR\Orca.exe" "$INSTDIR\resources\bin\orca.exe"`
|
||||
Pop $0
|
||||
${if} $0 != 0
|
||||
DetailPrint "Orca update release proof failed with status $0."
|
||||
SetErrorLevel 2
|
||||
Quit
|
||||
${endIf}
|
||||
${else}
|
||||
!insertmacro IS_POWERSHELL_AVAILABLE
|
||||
!insertmacro _CHECK_APP_RUNNING
|
||||
${endIf}
|
||||
!macroend
|
||||
|
||||
!macro customUnInstall
|
||||
${ifNot} ${isUpdated}
|
||||
nsExec::Exec 'taskkill /F /IM orca-terminal-daemon.exe'
|
||||
|
||||
File diff suppressed because one or more lines are too long
+712
-872
File diff suppressed because it is too large
Load Diff
@@ -166,11 +166,9 @@ describe('electron-builder config', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Why: the watchdog only arms in packaged builds, and its ELECTRON_RUN_AS_NODE
|
||||
// fork resolves the entry from app.asar.unpacked — inside the asar it never runs.
|
||||
it('unpacks the forked main-thread hang-watchdog entry', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).toEqual(
|
||||
expect.arrayContaining(['out/main/main-thread-hang-watchdog-entry.js'])
|
||||
it('keeps the worker-thread hang watchdog inside app.asar', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).not.toContain(
|
||||
'out/main/main-thread-hang-watchdog-entry.js'
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
@@ -37,13 +37,19 @@ describe('PR workflow parallelism', () => {
|
||||
expect(workflow.permissions).toEqual({ contents: 'read' })
|
||||
})
|
||||
|
||||
it('shards the general test suite across sixteen runners', () => {
|
||||
it('shards the general test suite across Node 24 and Node 26', () => {
|
||||
expect(workflow.jobs.test.strategy.matrix.node).toEqual(['24', '26'])
|
||||
expect(workflow.jobs.test.strategy.matrix.shard).toEqual(
|
||||
Array.from({ length: 16 }, (_, index) => index + 1)
|
||||
)
|
||||
expect(workflow.jobs.test.strategy.matrix.shard_total).toEqual([16])
|
||||
const testStep = workflow.jobs.test.steps.find((step) => step.name === 'Test shard')
|
||||
const installStep = workflow.jobs.test.steps.find(
|
||||
(step) => step.uses === './.github/actions/install-node-dependencies'
|
||||
)
|
||||
|
||||
expect(testStep.run).toContain('--shard=${{ matrix.shard }}/${{ strategy.job-total }}')
|
||||
expect(installStep.with['node-version']).toBe('${{ matrix.node }}')
|
||||
expect(testStep.run).toContain('--shard=${{ matrix.shard }}/${{ matrix.shard_total }}')
|
||||
for (const testFile of nativeShellContractFiles) {
|
||||
expect(testStep.run).toContain(`--exclude=${testFile}`)
|
||||
}
|
||||
@@ -95,9 +101,15 @@ describe('PR workflow parallelism', () => {
|
||||
const steps = dependencyAction.runs.steps
|
||||
const pnpmIndex = steps.findIndex((step) => step.name === 'Setup pnpm')
|
||||
const nodeIndex = steps.findIndex((step) => step.name === 'Setup Node.js')
|
||||
const requestedNodeIndex = steps.findIndex((step) => step.name === 'Setup requested Node.js')
|
||||
|
||||
expect(pnpmIndex).toBeLessThan(nodeIndex)
|
||||
expect(pnpmIndex).toBeLessThan(requestedNodeIndex)
|
||||
expect(steps[nodeIndex].with.cache).toBe('pnpm')
|
||||
expect(steps[nodeIndex].if).toBe("inputs.node-version == ''")
|
||||
expect(steps[requestedNodeIndex].if).toBe("inputs.node-version != ''")
|
||||
expect(steps[requestedNodeIndex].with['node-version']).toBe('${{ inputs.node-version }}')
|
||||
expect(steps[requestedNodeIndex].with.cache).toBe('pnpm')
|
||||
})
|
||||
|
||||
it('restores Electron downloads before preparing the package runtime', () => {
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
import { spawn, spawnSync } from 'node:child_process'
|
||||
import { closeSync, copyFileSync, mkdirSync, mkdtempSync, openSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
const projectDir = path.resolve(import.meta.dirname, '../..')
|
||||
const scriptPath = import.meta.filename
|
||||
const insideSessionFlag = '--inside-session'
|
||||
const processStopTimeoutMs = 5_000
|
||||
const processKillTimeoutMs = 1_000
|
||||
|
||||
function delay(milliseconds) {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds))
|
||||
}
|
||||
|
||||
function waitForExit(child) {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code, signal) => resolve(code ?? (signal ? 1 : 0)))
|
||||
})
|
||||
}
|
||||
|
||||
function processGroupMembers(processGroupId) {
|
||||
const result = spawnSync('ps', ['-o', 'pid=,ppid=,pgid=,comm=', '-g', String(processGroupId)], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
return []
|
||||
}
|
||||
return result.stdout
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
async function stopOwnedProcessGroup(processGroupId) {
|
||||
let members = processGroupMembers(processGroupId)
|
||||
if (members.length === 0) {
|
||||
return []
|
||||
}
|
||||
console.error(
|
||||
`[terminal-ime] stopping owned process group ${processGroupId}: ${members.join('; ')}`
|
||||
)
|
||||
try {
|
||||
process.kill(-processGroupId, 'SIGTERM')
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ESRCH') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const deadline = Date.now() + processStopTimeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
members = processGroupMembers(processGroupId)
|
||||
if (members.length === 0) {
|
||||
return []
|
||||
}
|
||||
await delay(100)
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(-processGroupId, 'SIGKILL')
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ESRCH') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const killDeadline = Date.now() + processKillTimeoutMs
|
||||
do {
|
||||
members = processGroupMembers(processGroupId)
|
||||
if (members.length === 0) {
|
||||
return []
|
||||
}
|
||||
await delay(100)
|
||||
} while (Date.now() < killDeadline)
|
||||
return members
|
||||
}
|
||||
|
||||
function commandOutput(command, args) {
|
||||
const result = spawnSync(command, args, { encoding: 'utf8' })
|
||||
return result.status === 0 ? result.stdout.trim() : result.stderr.trim()
|
||||
}
|
||||
|
||||
function configureHangulEngine() {
|
||||
for (const [key, value] of [
|
||||
['initial-input-mode', 'hangul'],
|
||||
['hangul-keyboard', '2']
|
||||
]) {
|
||||
const result = spawnSync(
|
||||
'gsettings',
|
||||
['set', 'org.freedesktop.ibus.engine.hangul', key, value],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`Failed to configure IBus Hangul ${key}: ${result.stderr.trim()}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForHangulEngine(ibusProcess) {
|
||||
const deadline = Date.now() + 15_000
|
||||
while (Date.now() < deadline) {
|
||||
if (ibusProcess.exitCode !== null) {
|
||||
throw new Error(`ibus-daemon exited early with code ${ibusProcess.exitCode}`)
|
||||
}
|
||||
const result = spawnSync('ibus', ['engine', 'hangul'], { stdio: 'pipe' })
|
||||
if (result.status === 0) {
|
||||
return
|
||||
}
|
||||
await delay(100)
|
||||
}
|
||||
throw new Error('Timed out while selecting the IBus Hangul engine')
|
||||
}
|
||||
|
||||
async function runInsideSession(evidenceDir) {
|
||||
const ibusLogPath = path.join(evidenceDir, 'ibus-daemon.log')
|
||||
const ibusLogFd = openSync(ibusLogPath, 'w')
|
||||
const windowManagerLogPath = path.join(evidenceDir, 'xfwm4.log')
|
||||
const windowManagerLogFd = openSync(windowManagerLogPath, 'w')
|
||||
const evidence = {
|
||||
display: process.env.DISPLAY ?? null,
|
||||
ibusDaemonPid: null,
|
||||
ibusGroupBeforeCleanup: [],
|
||||
ibusGroupAfterCleanup: [],
|
||||
playwrightPid: null,
|
||||
windowManagerPid: null,
|
||||
windowManagerGroupAfterCleanup: []
|
||||
}
|
||||
let ibusProcess
|
||||
let windowManagerProcess
|
||||
let testExitCode = 1
|
||||
|
||||
try {
|
||||
configureHangulEngine()
|
||||
windowManagerProcess = spawn('xfwm4', ['--compositor=off'], {
|
||||
detached: true,
|
||||
env: process.env,
|
||||
stdio: ['ignore', windowManagerLogFd, windowManagerLogFd]
|
||||
})
|
||||
if (!windowManagerProcess.pid) {
|
||||
throw new Error('xfwm4 did not return a PID')
|
||||
}
|
||||
evidence.windowManagerPid = windowManagerProcess.pid
|
||||
console.error(`[terminal-ime] started xfwm4 PID ${windowManagerProcess.pid}`)
|
||||
|
||||
ibusProcess = spawn(
|
||||
'ibus-daemon',
|
||||
['--xim', '--verbose', '--panel=disable', '--emoji-extension=disable'],
|
||||
{
|
||||
detached: true,
|
||||
env: process.env,
|
||||
stdio: ['ignore', ibusLogFd, ibusLogFd]
|
||||
}
|
||||
)
|
||||
if (!ibusProcess.pid) {
|
||||
throw new Error('ibus-daemon did not return a PID')
|
||||
}
|
||||
evidence.ibusDaemonPid = ibusProcess.pid
|
||||
console.error(`[terminal-ime] started ibus-daemon PID ${ibusProcess.pid}`)
|
||||
await waitForHangulEngine(ibusProcess)
|
||||
console.error(`[terminal-ime] IBus version: ${commandOutput('ibus', ['version'])}`)
|
||||
console.error(`[terminal-ime] IBus engine: ${commandOutput('ibus', ['engine'])}`)
|
||||
console.error(
|
||||
`[terminal-ime] Hangul initial mode: ${commandOutput('gsettings', [
|
||||
'get',
|
||||
'org.freedesktop.ibus.engine.hangul',
|
||||
'initial-input-mode'
|
||||
])}`
|
||||
)
|
||||
console.error(
|
||||
`[terminal-ime] Hangul keyboard: ${commandOutput('gsettings', [
|
||||
'get',
|
||||
'org.freedesktop.ibus.engine.hangul',
|
||||
'hangul-keyboard'
|
||||
])}`
|
||||
)
|
||||
evidence.ibusGroupBeforeCleanup = processGroupMembers(ibusProcess.pid)
|
||||
console.error(`[terminal-ime] owned IBus group: ${evidence.ibusGroupBeforeCleanup.join('; ')}`)
|
||||
|
||||
const testProcess = spawn(
|
||||
process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm',
|
||||
[
|
||||
'run',
|
||||
'test:e2e:headful',
|
||||
'--workers=1',
|
||||
'--',
|
||||
'tests/e2e/terminal-ibus-hangul-native.spec.ts'
|
||||
],
|
||||
{
|
||||
cwd: projectDir,
|
||||
env: {
|
||||
...process.env,
|
||||
ORCA_E2E_FORWARD_APP_LOGS: '1',
|
||||
ORCA_E2E_NATIVE_IBUS_HANGUL: '1'
|
||||
},
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
if (!testProcess.pid) {
|
||||
throw new Error('Playwright did not return a PID')
|
||||
}
|
||||
evidence.playwrightPid = testProcess.pid
|
||||
console.error(`[terminal-ime] started Playwright PID ${testProcess.pid}`)
|
||||
testExitCode = await waitForExit(testProcess)
|
||||
} finally {
|
||||
if (ibusProcess?.pid) {
|
||||
evidence.ibusGroupBeforeCleanup = processGroupMembers(ibusProcess.pid)
|
||||
evidence.ibusGroupAfterCleanup = await stopOwnedProcessGroup(ibusProcess.pid)
|
||||
}
|
||||
if (windowManagerProcess?.pid) {
|
||||
evidence.windowManagerGroupAfterCleanup = await stopOwnedProcessGroup(
|
||||
windowManagerProcess.pid
|
||||
)
|
||||
}
|
||||
closeSync(ibusLogFd)
|
||||
closeSync(windowManagerLogFd)
|
||||
mkdirSync(path.join(projectDir, 'test-results'), { recursive: true })
|
||||
copyFileSync(
|
||||
ibusLogPath,
|
||||
path.join(projectDir, 'test-results', 'terminal-ibus-hangul-native-ibus.log')
|
||||
)
|
||||
copyFileSync(
|
||||
windowManagerLogPath,
|
||||
path.join(projectDir, 'test-results', 'terminal-ibus-hangul-native-xfwm4.log')
|
||||
)
|
||||
writeFileSync(
|
||||
path.join(projectDir, 'test-results', 'terminal-ibus-hangul-native-processes.json'),
|
||||
`${JSON.stringify(evidence, null, 2)}\n`
|
||||
)
|
||||
}
|
||||
|
||||
if (evidence.ibusGroupAfterCleanup.length > 0) {
|
||||
throw new Error(
|
||||
`Owned IBus processes survived cleanup: ${evidence.ibusGroupAfterCleanup.join('; ')}`
|
||||
)
|
||||
}
|
||||
if (evidence.windowManagerGroupAfterCleanup.length > 0) {
|
||||
throw new Error(
|
||||
`Owned window-manager processes survived cleanup: ${evidence.windowManagerGroupAfterCleanup.join('; ')}`
|
||||
)
|
||||
}
|
||||
return testExitCode
|
||||
}
|
||||
|
||||
async function runOuter() {
|
||||
if (process.platform !== 'linux') {
|
||||
throw new Error('The native IBus Hangul E2E runner requires Linux/X11')
|
||||
}
|
||||
|
||||
const evidenceDir = mkdtempSync(path.join(os.tmpdir(), 'orca-terminal-ime-e2e-'))
|
||||
const runtimeDir = path.join(evidenceDir, 'runtime')
|
||||
mkdirSync(runtimeDir, { mode: 0o700 })
|
||||
mkdirSync(path.join(evidenceDir, 'config'))
|
||||
mkdirSync(path.join(evidenceDir, 'cache'))
|
||||
console.error(`[terminal-ime] evidence directory: ${evidenceDir}`)
|
||||
|
||||
const sessionProcess = spawn(
|
||||
'xvfb-run',
|
||||
[
|
||||
'--auto-servernum',
|
||||
'dbus-run-session',
|
||||
'--',
|
||||
process.execPath,
|
||||
scriptPath,
|
||||
insideSessionFlag,
|
||||
evidenceDir
|
||||
],
|
||||
{
|
||||
cwd: projectDir,
|
||||
detached: true,
|
||||
env: {
|
||||
...process.env,
|
||||
GTK_IM_MODULE: 'ibus',
|
||||
IBUS_ENABLE_SYNC_MODE: '1',
|
||||
LANG: process.env.LANG || 'C.UTF-8',
|
||||
QT_IM_MODULE: 'ibus',
|
||||
XDG_CACHE_HOME: path.join(evidenceDir, 'cache'),
|
||||
XDG_CONFIG_HOME: path.join(evidenceDir, 'config'),
|
||||
XDG_RUNTIME_DIR: runtimeDir,
|
||||
XMODIFIERS: '@im=ibus'
|
||||
},
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
if (!sessionProcess.pid) {
|
||||
throw new Error('xvfb-run did not return a PID')
|
||||
}
|
||||
console.error(`[terminal-ime] started isolated X11 session PID ${sessionProcess.pid}`)
|
||||
const exitCode = await waitForExit(sessionProcess)
|
||||
const remaining = await stopOwnedProcessGroup(sessionProcess.pid)
|
||||
if (remaining.length > 0) {
|
||||
throw new Error(`Owned X11 session processes survived cleanup: ${remaining.join('; ')}`)
|
||||
}
|
||||
return exitCode
|
||||
}
|
||||
|
||||
const insideSession = process.argv[2] === insideSessionFlag
|
||||
try {
|
||||
if (insideSession && !process.argv[3]) {
|
||||
throw new Error(`${insideSessionFlag} requires an evidence directory argument`)
|
||||
}
|
||||
process.exitCode = insideSession ? await runInsideSession(process.argv[3]) : await runOuter()
|
||||
} catch (error) {
|
||||
console.error(`[terminal-ime] ${error instanceof Error ? error.message : String(error)}`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { parse } from 'yaml'
|
||||
|
||||
const projectDir = resolve(import.meta.dirname, '../..')
|
||||
|
||||
describe('terminal IME e2e workflow', () => {
|
||||
const workflow = parse(
|
||||
readFileSync(join(projectDir, '.github/workflows/terminal-ime-e2e.yml'), 'utf8')
|
||||
)
|
||||
|
||||
it('runs for xterm patch and terminal IME regression changes', () => {
|
||||
expect(workflow.on.pull_request.paths).toEqual(
|
||||
expect.arrayContaining([
|
||||
'config/patches/@xterm__xterm@6.1.0-beta.287.patch',
|
||||
'config/scripts/run-terminal-ibus-hangul-e2e.mjs',
|
||||
'src/renderer/src/components/terminal-pane/keyboard-handlers.ts',
|
||||
'src/renderer/src/components/terminal-pane/keyboard-handlers-ime.test.tsx',
|
||||
'src/renderer/src/components/terminal-pane/pty-connection.ts',
|
||||
'src/renderer/src/components/terminal-pane/pty-connection.test.ts',
|
||||
'src/renderer/src/components/terminal-pane/terminal-ime-*',
|
||||
'src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts',
|
||||
'src/renderer/src/components/terminal-pane/xterm-bypass-policy.ts',
|
||||
'src/renderer/src/components/terminal-pane/xterm-bypass-policy.test.ts',
|
||||
'tests/e2e/korean-ime-terminal-shift-enter-commit.spec.ts',
|
||||
'tests/e2e/terminal-ibus-hangul-native.spec.ts',
|
||||
'tests/e2e/terminal-ime-*.ts'
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('installs native IBus Hangul and X11 input tools', () => {
|
||||
const runs = workflow.jobs['linux-x11'].steps
|
||||
.map((step) => step.run)
|
||||
.filter((run) => typeof run === 'string')
|
||||
const installRun = runs.find((run) => run.includes('apt-get install'))
|
||||
|
||||
expect(installRun).toBeDefined()
|
||||
expect(installRun).toContain('ibus-hangul')
|
||||
expect(installRun).toContain('xdotool')
|
||||
expect(installRun).toContain('xfwm4')
|
||||
expect(installRun).toContain('xvfb')
|
||||
expect(installRun).toContain('dbus-x11')
|
||||
expect(installRun).toContain('dconf-gsettings-backend')
|
||||
expect(installRun).toContain('libglib2.0-bin')
|
||||
})
|
||||
|
||||
it('runs deterministic boundaries before the real IBus suite', () => {
|
||||
const runs = workflow.jobs['linux-x11'].steps
|
||||
.map((step) => step.run)
|
||||
.filter((run) => typeof run === 'string')
|
||||
const deterministicIndex = runs.findIndex((run) =>
|
||||
run.includes('terminal-ime-exact-byte.spec.ts')
|
||||
)
|
||||
const nativeIndex = runs.findIndex((run) => run.includes('test:e2e:terminal-ime-native'))
|
||||
|
||||
expect(deterministicIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(nativeIndex).toBeGreaterThan(deterministicIndex)
|
||||
})
|
||||
|
||||
it('keeps IBus lifecycle scoped to owned processes', () => {
|
||||
const runner = readFileSync(
|
||||
join(projectDir, 'config/scripts/run-terminal-ibus-hangul-e2e.mjs'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(runner).toContain(
|
||||
"['--xim', '--verbose', '--panel=disable', '--emoji-extension=disable']"
|
||||
)
|
||||
expect(runner).toContain("spawn('xfwm4', ['--compositor=off']")
|
||||
expect(runner).toContain("['initial-input-mode', 'hangul']")
|
||||
expect(runner).toContain("['hangul-keyboard', '2']")
|
||||
expect(runner).toContain("process.kill(-processGroupId, 'SIGTERM')")
|
||||
expect(runner).toContain("process.kill(-processGroupId, 'SIGKILL')")
|
||||
expect(runner).toContain('const killDeadline = Date.now() + processKillTimeoutMs')
|
||||
expect(runner).toMatch(
|
||||
/'test:e2e:headful',\s*'--workers=1',\s*'--',\s*'tests\/e2e\/terminal-ibus-hangul-native\.spec\.ts'/
|
||||
)
|
||||
expect(runner).not.toContain("'--replace'")
|
||||
expect(runner).not.toContain('killall')
|
||||
expect(runner).not.toContain('pkill')
|
||||
})
|
||||
|
||||
it('bounds blocking native input commands', () => {
|
||||
const nativeSpec = readFileSync(
|
||||
join(projectDir, 'tests/e2e/terminal-ibus-hangul-native.spec.ts'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(nativeSpec.match(/timeout: NATIVE_COMMAND_TIMEOUT_MS/g)).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
@@ -15,6 +15,8 @@ export default defineConfig({
|
||||
},
|
||||
test: {
|
||||
environment: 'node',
|
||||
// Why: Node 26's undefined Web Storage globals prevent Vitest from installing happy-dom's.
|
||||
execArgv: ['--no-experimental-webstorage'],
|
||||
include: [
|
||||
'src/**/*.test.ts',
|
||||
'src/**/*.test.tsx',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -417,28 +417,10 @@ status. The command name comes only from the validated inherited
|
||||
than rendered. The launcher propagates status 75. No long wait starts on this
|
||||
path.
|
||||
|
||||
The candidate installer owns the backward edge: after the desktop has shut
|
||||
down, it may terminate only remaining processes whose resolved executable path
|
||||
is the exact installed Orca executable before replacement. This releases an
|
||||
already-blocked A CLI that cannot know the new marker. Its accepted question
|
||||
remains durable; retrying the exact old ask under B reconstructs the same
|
||||
pending question and prints the resume command. If launcher restart
|
||||
supervision can be proved, it may transparently reconnect instead, but
|
||||
seamless install cannot depend on that optimization.
|
||||
|
||||
After terminating the exact installed child, the installer waits boundedly for
|
||||
its exact descendant native launcher to exit and for the installed executable
|
||||
lock to clear. Replacement fails explicitly if either release cannot be
|
||||
proved; it never proceeds from a broad process-name match or an assumed delay.
|
||||
|
||||
A non-skipped packaged Windows A-to-B test uses the real launcher and updater,
|
||||
cuts a blocked ask, verifies post-commit output flush and status 75 propagation
|
||||
on native and packaged WSL marker-aware paths, including an executable
|
||||
`orca-ide` WSL resume command. It separately proves the candidate installer
|
||||
terminates an exact-path marker-unaware A child and observes the exact launcher
|
||||
exit plus lock release. The test fails if either release is unknown. It then
|
||||
installs B, resumes the same question, and records one question/reply. This
|
||||
fallback is product behavior, not test-only guidance.
|
||||
This compatibility path makes the accepted question recoverable without
|
||||
claiming Windows updater or uninstaller process ownership. Generic executable
|
||||
drain, replacement, and uninstall behavior remains separate updater
|
||||
reliability work and requires its own Windows artifact proof.
|
||||
|
||||
## Rendering and authoritative guidance
|
||||
|
||||
@@ -510,7 +492,7 @@ Task, and Dispatch; emits no provider-resume spawn or interruption; preserves
|
||||
coordinator focus; and remains inert after switching away and back. This
|
||||
candidate/candidate fixture covers the renderer regression deterministically
|
||||
but does not replace the distinct installed-A/candidate-B packaged cutover
|
||||
journey below.
|
||||
journey below. It does not claim packaged Windows updater behavior.
|
||||
|
||||
Use distinct installed A and B artifacts plus an external append-only fake
|
||||
agent spawn ledger:
|
||||
@@ -558,11 +540,9 @@ Run separate remote journeys:
|
||||
- a git-independent folder workspace parameter through the same adoption
|
||||
journey.
|
||||
|
||||
The Windows test is a non-skipped PR/release gate. It installs distinct A and
|
||||
candidate-B packages, drives the real packaged updater while a packaged
|
||||
`orca.exe` ask is blocked, proves the launcher and child release, verifies A
|
||||
and B hashes/versions differ, resumes the same question on B, and records
|
||||
exactly one question and reply.
|
||||
Packaged Windows updater continuity is outside this migration gate. A separate
|
||||
updater change must prove candidate-built A/B install, executable release,
|
||||
resume, and normal uninstall behavior on Windows before making that guarantee.
|
||||
|
||||
## Implementation slices
|
||||
|
||||
|
||||
@@ -208,9 +208,8 @@ export const electronViteConfig: UserConfig = {
|
||||
// Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults
|
||||
// can't take down the main process (issue #7547).
|
||||
'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'),
|
||||
// Why: forked with ELECTRON_RUN_AS_NODE so it survives a deadlocked
|
||||
// main thread (macOS 26 AppKit scene-update deadlock) and can record
|
||||
// the stall for the next launch to report.
|
||||
// Why: a worker thread survives the macOS 26 AppKit main-thread deadlock
|
||||
// without paying for another Electron process.
|
||||
'main-thread-hang-watchdog-entry': resolve(
|
||||
'src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts'
|
||||
),
|
||||
|
||||
@@ -2530,9 +2530,12 @@ private enum KeyMap {
|
||||
}
|
||||
|
||||
private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
private static let unclaimedSessionDeadline: TimeInterval = 30
|
||||
|
||||
private let socketPath: String
|
||||
private let token: String?
|
||||
private var listener: SocketListener?
|
||||
private var unclaimedSessionTimeout: DispatchWorkItem?
|
||||
|
||||
init(socketPath: String, token: String?) {
|
||||
self.socketPath = socketPath
|
||||
@@ -2541,9 +2544,31 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
do {
|
||||
let listener = try SocketListener(socketPath: socketPath, token: token)
|
||||
let timeout = DispatchWorkItem {
|
||||
fputs("computer-use agent received no authenticated session before its deadline\n", stderr)
|
||||
NSApp.terminate(nil)
|
||||
}
|
||||
unclaimedSessionTimeout = timeout
|
||||
let listener = try SocketListener(
|
||||
socketPath: socketPath,
|
||||
token: token,
|
||||
onSessionClaimed: {
|
||||
DispatchQueue.main.async {
|
||||
timeout.cancel()
|
||||
}
|
||||
},
|
||||
onSessionClosed: {
|
||||
DispatchQueue.main.async {
|
||||
NSApp.terminate(nil)
|
||||
}
|
||||
}
|
||||
)
|
||||
self.listener = listener
|
||||
listener.start()
|
||||
DispatchQueue.main.asyncAfter(
|
||||
deadline: .now() + Self.unclaimedSessionDeadline,
|
||||
execute: timeout
|
||||
)
|
||||
} catch {
|
||||
fputs("failed to start computer-use socket: \(error)\n", stderr)
|
||||
NSApp.terminate(nil)
|
||||
@@ -2551,6 +2576,8 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
}
|
||||
|
||||
func applicationWillTerminate(_ notification: Notification) {
|
||||
unclaimedSessionTimeout?.cancel()
|
||||
unclaimedSessionTimeout = nil
|
||||
listener?.stop()
|
||||
}
|
||||
}
|
||||
@@ -3456,14 +3483,25 @@ private final class ButtonTarget: NSObject {
|
||||
private final class SocketListener: @unchecked Sendable {
|
||||
private let socketPath: String
|
||||
private let token: String?
|
||||
private let onSessionClaimed: () -> Void
|
||||
private let onSessionClosed: () -> Void
|
||||
private let provider = Provider()
|
||||
private let providerLock = NSLock()
|
||||
private let sessionLock = NSLock()
|
||||
private var sessionOwnership = AgentSessionOwnership()
|
||||
private var socketFd: Int32 = -1
|
||||
private var isStopped = false
|
||||
|
||||
init(socketPath: String, token: String?) throws {
|
||||
init(
|
||||
socketPath: String,
|
||||
token: String?,
|
||||
onSessionClaimed: @escaping () -> Void,
|
||||
onSessionClosed: @escaping () -> Void
|
||||
) throws {
|
||||
self.socketPath = socketPath
|
||||
self.token = token
|
||||
self.onSessionClaimed = onSessionClaimed
|
||||
self.onSessionClosed = onSessionClosed
|
||||
try bindSocket()
|
||||
}
|
||||
|
||||
@@ -3545,15 +3583,39 @@ private final class SocketListener: @unchecked Sendable {
|
||||
}
|
||||
|
||||
private func handleConnection(_ fd: Int32) {
|
||||
defer { close(fd) }
|
||||
defer {
|
||||
sessionLock.lock()
|
||||
let shouldTerminate = sessionOwnership.disconnect(fd)
|
||||
sessionLock.unlock()
|
||||
close(fd)
|
||||
if shouldTerminate {
|
||||
onSessionClosed()
|
||||
}
|
||||
}
|
||||
let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true
|
||||
let decoder = JSONDecoder()
|
||||
var registrationComplete = false
|
||||
while let line = readLine(from: fd) {
|
||||
guard let data = line.data(using: .utf8),
|
||||
let request = try? decoder.decode(Request.self, from: data)
|
||||
else {
|
||||
continue
|
||||
}
|
||||
if !registrationComplete {
|
||||
sessionLock.lock()
|
||||
let registration = sessionOwnership.registerConnection(
|
||||
fd,
|
||||
authenticated: isAuthenticated(
|
||||
request: request,
|
||||
authorizedPeer: authorizedPeer
|
||||
)
|
||||
)
|
||||
sessionLock.unlock()
|
||||
registrationComplete = registration != .rejected
|
||||
if registration == .claimed {
|
||||
onSessionClaimed()
|
||||
}
|
||||
}
|
||||
let response = handleRequest(
|
||||
provider: provider,
|
||||
lock: providerLock,
|
||||
@@ -3564,6 +3626,12 @@ private final class SocketListener: @unchecked Sendable {
|
||||
writeJSON(response, to: fd)
|
||||
}
|
||||
}
|
||||
|
||||
private func isAuthenticated(request: Request, authorizedPeer: Bool) -> Bool {
|
||||
guard authorizedPeer else { return false }
|
||||
guard let token else { return true }
|
||||
return request.token == token
|
||||
}
|
||||
}
|
||||
|
||||
private func existingPathMode(_ path: String) -> mode_t? {
|
||||
@@ -3696,7 +3764,7 @@ private func handleRequest(
|
||||
if let expectedToken, request.token != expectedToken {
|
||||
return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "invalid computer-use agent token"]]
|
||||
}
|
||||
if expectedToken != nil && !authorizedPeer {
|
||||
if !authorizedPeer {
|
||||
return ["id": request.id, "ok": false, "error": ["code": "permission_denied", "message": "computer-use agent peer is not authorized"]]
|
||||
}
|
||||
if request.method == "terminate" {
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
public enum AgentSessionRegistration: Sendable, Equatable {
|
||||
case rejected
|
||||
case sessionReleased
|
||||
case joined
|
||||
case claimed
|
||||
}
|
||||
|
||||
public struct AgentSessionOwnership: Sendable {
|
||||
private var authenticatedConnections: Set<Int32> = []
|
||||
private var wasClaimed = false
|
||||
private var wasReleased = false
|
||||
|
||||
public init() {}
|
||||
|
||||
public mutating func registerConnection(
|
||||
_ connection: Int32,
|
||||
authenticated: Bool
|
||||
) -> AgentSessionRegistration {
|
||||
guard authenticated else { return .rejected }
|
||||
guard !wasReleased else { return .sessionReleased }
|
||||
let inserted = authenticatedConnections.insert(connection).inserted
|
||||
guard inserted, !wasClaimed else { return .joined }
|
||||
wasClaimed = true
|
||||
return .claimed
|
||||
}
|
||||
|
||||
public mutating func disconnect(_ connection: Int32) -> Bool {
|
||||
guard authenticatedConnections.remove(connection) != nil else { return false }
|
||||
let released = wasClaimed && authenticatedConnections.isEmpty
|
||||
if released {
|
||||
wasReleased = true
|
||||
}
|
||||
return released
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class AgentSessionOwnershipTests: XCTestCase {
|
||||
func testUnclaimedDisconnectDoesNotTerminateAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertFalse(ownership.disconnect(12))
|
||||
}
|
||||
|
||||
func testUnauthenticatedConnectionCannotClaimOrRetainAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: false), .rejected)
|
||||
XCTAssertFalse(ownership.disconnect(12))
|
||||
}
|
||||
|
||||
func testLastAuthenticatedDisconnectTerminatesAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: true), .claimed)
|
||||
XCTAssertTrue(ownership.disconnect(12))
|
||||
}
|
||||
|
||||
func testAgentWaitsForEveryAuthenticatedConnectionToClose() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: true), .claimed)
|
||||
XCTAssertEqual(ownership.registerConnection(13, authenticated: true), .joined)
|
||||
XCTAssertFalse(ownership.disconnect(12))
|
||||
XCTAssertTrue(ownership.disconnect(13))
|
||||
}
|
||||
|
||||
func testDuplicateRegistrationDoesNotRetainAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: true), .claimed)
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: true), .joined)
|
||||
XCTAssertTrue(ownership.disconnect(12))
|
||||
}
|
||||
|
||||
func testReleasedSessionCannotBeReclaimed() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(12, authenticated: true), .claimed)
|
||||
XCTAssertTrue(ownership.disconnect(12))
|
||||
XCTAssertEqual(ownership.registerConnection(13, authenticated: true), .sessionReleased)
|
||||
XCTAssertFalse(ownership.disconnect(13))
|
||||
}
|
||||
}
|
||||
@@ -100,6 +100,7 @@
|
||||
"win-crash-survival-e2e": "node tools/win-crash-survival-e2e/run.mjs",
|
||||
"test:e2e:ssh-codex-artifacts-repro": "node config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs",
|
||||
"test:e2e:headful": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headful",
|
||||
"test:e2e:terminal-ime-native": "node config/scripts/run-terminal-ibus-hangul-e2e.mjs",
|
||||
"test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts",
|
||||
"bench:idle-cpu": "pnpm run ensure:electron-runtime && node config/scripts/run-idle-cpu-benchmark.mjs",
|
||||
"bench:startup": "pnpm run ensure:electron-runtime && node tools/benchmarks/startup-time-bench.mjs",
|
||||
|
||||
Generated
+24
-24
@@ -19,7 +19,7 @@ patchedDependencies:
|
||||
hash: 6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258
|
||||
path: config/patches/@xterm__addon-webgl@0.20.0-beta.286.patch
|
||||
'@xterm/xterm@6.1.0-beta.287':
|
||||
hash: 8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3
|
||||
hash: 082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6
|
||||
path: config/patches/@xterm__xterm@6.1.0-beta.287.patch
|
||||
node-pty@1.1.0:
|
||||
hash: 8fc49f17011b6611a5b8c00e83a6f12e14e75aada2b0ef26dc5393f8376d20e8
|
||||
@@ -46,7 +46,7 @@ importers:
|
||||
version: 2.5.6
|
||||
'@xterm/addon-serialize':
|
||||
specifier: 0.15.0-beta.287
|
||||
version: 0.15.0-beta.287(patch_hash=81575700d58b62f9262d302aa4ae43b445a6273d579cd75dd6729c8883011ab9)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.15.0-beta.287(patch_hash=81575700d58b62f9262d302aa4ae43b445a6273d579cd75dd6729c8883011ab9)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/headless':
|
||||
specifier: 6.1.0-beta.287
|
||||
version: 6.1.0-beta.287
|
||||
@@ -206,25 +206,25 @@ importers:
|
||||
version: 5.2.0(rolldown-vite@7.3.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@25.9.5)(jiti@2.7.0)(yaml@2.8.4))
|
||||
'@xterm/addon-fit':
|
||||
specifier: 0.12.0-beta.287
|
||||
version: 0.12.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.12.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/addon-ligatures':
|
||||
specifier: 0.11.0-beta.287
|
||||
version: 0.11.0-beta.287(patch_hash=47405b9994b5acf1b4e90b49250358c1ca03649854d59560e7732b72fe336920)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.11.0-beta.287(patch_hash=47405b9994b5acf1b4e90b49250358c1ca03649854d59560e7732b72fe336920)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/addon-search':
|
||||
specifier: 0.17.0-beta.287
|
||||
version: 0.17.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.17.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/addon-unicode11':
|
||||
specifier: 0.10.0-beta.287
|
||||
version: 0.10.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.10.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/addon-web-links':
|
||||
specifier: 0.13.0-beta.287
|
||||
version: 0.13.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.13.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/addon-webgl':
|
||||
specifier: 0.20.0-beta.286
|
||||
version: 0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))
|
||||
version: 0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))
|
||||
'@xterm/xterm':
|
||||
specifier: 6.1.0-beta.287
|
||||
version: 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
version: 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
class-variance-authority:
|
||||
specifier: ^0.7.1
|
||||
version: 0.7.1
|
||||
@@ -9579,39 +9579,39 @@ snapshots:
|
||||
|
||||
'@xmldom/xmldom@0.8.13': {}
|
||||
|
||||
'@xterm/addon-fit@0.12.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-fit@0.12.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/addon-ligatures@0.11.0-beta.287(patch_hash=47405b9994b5acf1b4e90b49250358c1ca03649854d59560e7732b72fe336920)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-ligatures@0.11.0-beta.287(patch_hash=47405b9994b5acf1b4e90b49250358c1ca03649854d59560e7732b72fe336920)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
lru-cache: 11.5.1
|
||||
opentype.js: 2.0.0
|
||||
|
||||
'@xterm/addon-search@0.17.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-search@0.17.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/addon-serialize@0.15.0-beta.287(patch_hash=81575700d58b62f9262d302aa4ae43b445a6273d579cd75dd6729c8883011ab9)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-serialize@0.15.0-beta.287(patch_hash=81575700d58b62f9262d302aa4ae43b445a6273d579cd75dd6729c8883011ab9)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/addon-unicode11@0.10.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-unicode11@0.10.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/addon-web-links@0.13.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-web-links@0.13.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/addon-webgl@0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3))':
|
||||
'@xterm/addon-webgl@0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6))':
|
||||
dependencies:
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)
|
||||
'@xterm/xterm': 6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)
|
||||
|
||||
'@xterm/headless@6.1.0-beta.287': {}
|
||||
|
||||
'@xterm/xterm@6.1.0-beta.287(patch_hash=8a337bdef40a57723e23e548f6100feb49755075f58702a8ae4544ad3ae2b9d3)': {}
|
||||
'@xterm/xterm@6.1.0-beta.287(patch_hash=082a4e714662e81b21d386f2c1de4c9dfd13e195c2497e9162b7e87f86b6a9b6)': {}
|
||||
|
||||
abbrev@4.0.0: {}
|
||||
|
||||
|
||||
@@ -0,0 +1,559 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { AgentSubagentSnapshot } from '../../shared/agent-status-types'
|
||||
import { makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { toAppSshPtyId } from '../../shared/ssh-pty-id'
|
||||
import { AgentHookServer } from './server'
|
||||
import {
|
||||
indexPersistedPaneKeyPtyIds,
|
||||
isLocalExecutionHost,
|
||||
resolveAgentWorkspaceExecutionHostId,
|
||||
sweepRestoredSubagentsWithoutLiveAgent
|
||||
} from './restored-subagent-liveness-sweep'
|
||||
|
||||
const LEAF = '11111111-1111-4111-8111-111111111111'
|
||||
const PANE = makePaneKey('tab-1', LEAF)
|
||||
const PTY = 'wt-1__pty-1'
|
||||
const WORKING_CHILD: AgentSubagentSnapshot = {
|
||||
id: 'areview-loop-c237a4c577493352',
|
||||
state: 'working',
|
||||
startedAt: 1_000
|
||||
}
|
||||
|
||||
let dir: string
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'orca-restored-subagent-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
/** Persist a pane whose lead has finished but whose roster still holds a working
|
||||
* child, then restart into a fresh server — the shape a machine sleep leaves
|
||||
* behind when the child's SubagentStop is lost while Orca is down. */
|
||||
async function restartWithInFlightSubagent(options?: {
|
||||
connectionId?: string
|
||||
state?: 'working' | 'waiting'
|
||||
subagents?: AgentSubagentSnapshot[]
|
||||
additionalPaneKey?: string
|
||||
}): Promise<AgentHookServer> {
|
||||
const first = new AgentHookServer()
|
||||
await first.start({ env: 'production', userDataPath: dir })
|
||||
first.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
connectionId: options?.connectionId ?? null,
|
||||
payload: {
|
||||
state: options?.state ?? 'working',
|
||||
prompt: 'review the PR',
|
||||
agentType: 'claude',
|
||||
subagents: options?.subagents ?? [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
if (options?.additionalPaneKey) {
|
||||
first.ingestTerminalStatus({
|
||||
paneKey: options.additionalPaneKey,
|
||||
tabId: 'tab-2',
|
||||
worktreeId: 'wt-1',
|
||||
connectionId: options.connectionId ?? null,
|
||||
payload: {
|
||||
state: options.state ?? 'working',
|
||||
prompt: 'review the PR',
|
||||
agentType: 'claude',
|
||||
subagents: options.subagents ?? [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
}
|
||||
first.flushStatusPersistSync()
|
||||
first.stop()
|
||||
|
||||
const restarted = new AgentHookServer()
|
||||
await restarted.start({ env: 'production', userDataPath: dir })
|
||||
return restarted
|
||||
}
|
||||
|
||||
function sweepWith(
|
||||
server: AgentHookServer,
|
||||
overrides: {
|
||||
probeLiveLocalPty?: (ptyId: string) => boolean | null | Promise<boolean | null>
|
||||
executionHostId?: string | null
|
||||
boundPtyIdByPaneKey?: Record<string, string>
|
||||
persistedPtyIdByPaneKey?: Record<string, string>
|
||||
} = {}
|
||||
): Promise<number> {
|
||||
return sweepRestoredSubagentsWithoutLiveAgent({
|
||||
probeLiveLocalPty: async (ptyId) =>
|
||||
overrides.probeLiveLocalPty ? await overrides.probeLiveLocalPty(ptyId) : false,
|
||||
isLocalExecutionHost: () =>
|
||||
isLocalExecutionHost(
|
||||
overrides.executionHostId === undefined ? 'local' : overrides.executionHostId
|
||||
),
|
||||
getBoundPtyIdForPaneKey: (paneKey) => overrides.boundPtyIdByPaneKey?.[paneKey],
|
||||
getPersistedPtyIdForPaneKey: (paneKey) => overrides.persistedPtyIdByPaneKey?.[paneKey],
|
||||
reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) =>
|
||||
server.reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalHost,
|
||||
isLocalPaneAgentLive,
|
||||
isLocalPaneLivenessEvidenceCurrent
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function paneStatus(
|
||||
server: AgentHookServer,
|
||||
paneKey = PANE
|
||||
): { state: string; subagents?: AgentSubagentSnapshot[] } {
|
||||
const entry = server.getStatusSnapshotForPane(paneKey)[0]
|
||||
return { state: entry?.state ?? 'missing', subagents: entry?.subagents }
|
||||
}
|
||||
|
||||
describe('restored subagent liveness sweep', () => {
|
||||
it('reaps the phantom seed so a slept-through pane reaches done', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
const previous = server.getStatusSnapshotForPane(PANE)[0]
|
||||
const previousReceivedAt = previous?.receivedAt ?? 0
|
||||
const reconciledAt = previousReceivedAt + 1
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(previousReceivedAt)
|
||||
|
||||
expect(await sweepWith(server, { persistedPtyIdByPaneKey: { [PANE]: PTY } })).toBe(1)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'done', subagents: undefined })
|
||||
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
|
||||
receivedAt: reconciledAt,
|
||||
stateStartedAt: reconciledAt
|
||||
})
|
||||
now.mockRestore()
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a seed whose pane still has a live local PTY', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty: (ptyId) => ptyId === PTY,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves timing when reaping rows does not change the lead state', async () => {
|
||||
const server = await restartWithInFlightSubagent({ state: 'waiting' })
|
||||
try {
|
||||
const previous = server.getStatusSnapshotForPane(PANE)[0]
|
||||
|
||||
expect(await sweepWith(server, { persistedPtyIdByPaneKey: { [PANE]: PTY } })).toBe(1)
|
||||
|
||||
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
|
||||
state: 'waiting',
|
||||
subagents: undefined,
|
||||
receivedAt: previous?.receivedAt,
|
||||
stateStartedAt: previous?.stateStartedAt
|
||||
})
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('uses targeted liveness for a PTY bound in this runtime', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
const probeLiveLocalPty = vi.fn((ptyId: string) => ptyId === PTY)
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('probes only restored rosters and deduplicates a shared exact PTY', async () => {
|
||||
const otherPane = makePaneKey('tab-2', LEAF)
|
||||
const server = await restartWithInFlightSubagent({ additionalPaneKey: otherPane })
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY, [otherPane]: PTY }
|
||||
})
|
||||
).toBe(2)
|
||||
|
||||
expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not probe a hydrated Claude pane without restored rows', async () => {
|
||||
const server = await restartWithInFlightSubagent({ subagents: [] })
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps an SSH-launched pane, whose agent cannot appear in a local scan', async () => {
|
||||
const sshPtyId = toAppSshPtyId('conn-1', PTY)
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
executionHostId: 'ssh:conn-1',
|
||||
persistedPtyIdByPaneKey: { [PANE]: sshPtyId }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps a relay-owned pane even with no local PTY at all', async () => {
|
||||
const server = await restartWithInFlightSubagent({ connectionId: 'conn-1' })
|
||||
try {
|
||||
expect(await sweepWith(server)).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps a runtime-hosted pane from the desktop local provider', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
executionHostId: 'runtime:ephemeral-vm-1',
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: 'remote:ephemeral-vm-1@@pty-1' }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does nothing when targeted PTY liveness is unknown', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty: () => null,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does nothing without an exact pane PTY binding', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(await sweepWith(server, { probeLiveLocalPty })).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('skips panes that have reported to this runtime', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
payload: {
|
||||
state: 'working',
|
||||
prompt: 'now reviewing round two',
|
||||
agentType: 'claude',
|
||||
subagents: [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
|
||||
expect(await sweepWith(server)).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that reports while its liveness probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledOnce())
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
payload: {
|
||||
state: 'working',
|
||||
prompt: 'confirmed during probe',
|
||||
agentType: 'claude',
|
||||
subagents: [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that binds a newer PTY while the old probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const boundPtyIdByPaneKey = { [PANE]: PTY }
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, { probeLiveLocalPty, boundPtyIdByPaneKey })
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY))
|
||||
boundPtyIdByPaneKey[PANE] = 'wt-1__pty-new'
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that rebinds its persisted PTY while the probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const boundPtyIdByPaneKey: Record<string, string> = {}
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY))
|
||||
boundPtyIdByPaneKey[PANE] = PTY
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a same-id rebind that lands after its probe but before the batch settles', async () => {
|
||||
const otherPane = makePaneKey('tab-2', LEAF)
|
||||
const otherPty = 'wt-1__pty-2'
|
||||
const server = await restartWithInFlightSubagent({ additionalPaneKey: otherPane })
|
||||
const boundPtyIdByPaneKey: Record<string, string> = {}
|
||||
let resolveOtherProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn((ptyId: string) =>
|
||||
ptyId === PTY
|
||||
? Promise.resolve(false)
|
||||
: new Promise<boolean | null>((resolve) => {
|
||||
resolveOtherProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY, [otherPane]: otherPty }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledTimes(2))
|
||||
boundPtyIdByPaneKey[PANE] = PTY
|
||||
resolveOtherProbe(false)
|
||||
|
||||
expect(await sweep).toBe(1)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
expect(paneStatus(server, otherPane).state).toBe('done')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves a pane whose lead is genuinely mid-turn at working', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
// Why: the lead's own tool event proves the process is alive; only the
|
||||
// child-gated 'working' may be re-derived, so the state must survive.
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: makePaneKey('tab-2', LEAF),
|
||||
tabId: 'tab-2',
|
||||
worktreeId: 'wt-1',
|
||||
payload: { state: 'working', prompt: 'still going', agentType: 'claude' }
|
||||
})
|
||||
|
||||
await sweepWith(server)
|
||||
|
||||
expect(paneStatus(server, makePaneKey('tab-2', LEAF)).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('reports zero and leaves state alone when targeted liveness throws', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => {
|
||||
throw new Error('daemon unreachable')
|
||||
})
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('indexPersistedPaneKeyPtyIds', () => {
|
||||
it('maps layout leaves to pane keys and ignores empty bindings', () => {
|
||||
expect(
|
||||
indexPersistedPaneKeyPtyIds({
|
||||
'tab-1': { ptyIdsByLeafId: { [LEAF]: PTY, 'leaf-empty': '' } },
|
||||
'tab-2': undefined,
|
||||
'tab-3': {}
|
||||
})
|
||||
).toEqual(new Map([[PANE, PTY]]))
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveAgentWorkspaceExecutionHostId', () => {
|
||||
const localRepo = {
|
||||
id: 'local-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'local' as const
|
||||
}
|
||||
const runtimeRepo = {
|
||||
id: 'runtime-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'runtime:ephemeral-vm-1' as const
|
||||
}
|
||||
const futureHostRepo = {
|
||||
id: 'future-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'container:future-host'
|
||||
}
|
||||
const deps = {
|
||||
getRepo: (repoId: string) =>
|
||||
[localRepo, runtimeRepo, futureHostRepo].find((candidate) => candidate.id === repoId),
|
||||
getWorktreeMeta: (worktreeId: string) => {
|
||||
if (worktreeId === 'local-repo::/runtime-worktree') {
|
||||
return { hostId: 'runtime:worktree-owner' }
|
||||
}
|
||||
if (worktreeId === 'runtime-repo::/local-worktree') {
|
||||
return { hostId: 'local' }
|
||||
}
|
||||
if (worktreeId === 'local-repo::/future-worktree') {
|
||||
return { hostId: 'container:future-host' }
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
getFolderWorkspace: (id: string) =>
|
||||
id === 'folder-runtime' ? { projectGroupId: 'group-runtime', connectionId: null } : undefined,
|
||||
getProjectGroups: () => [
|
||||
{
|
||||
id: 'group-runtime',
|
||||
connectionId: null,
|
||||
executionHostId: 'runtime:ephemeral-vm-1'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
it('positively identifies local ownership and rejects runtime hosts', () => {
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/repo', deps)).toBe('local')
|
||||
expect(resolveAgentWorkspaceExecutionHostId('runtime-repo::/repo', deps)).toBe(
|
||||
'runtime:ephemeral-vm-1'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('folder:folder-runtime', deps)).toBe(
|
||||
'runtime:ephemeral-vm-1'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/runtime-worktree', deps)).toBe(
|
||||
'runtime:worktree-owner'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('runtime-repo::/local-worktree', deps)).toBe(
|
||||
'local'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('future-repo::/repo', deps)).toBeNull()
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/future-worktree', deps)).toBeNull()
|
||||
expect(isLocalExecutionHost('local')).toBe(true)
|
||||
expect(isLocalExecutionHost('runtime:ephemeral-vm-1')).toBe(false)
|
||||
})
|
||||
|
||||
it('treats missing workspace provenance as unknown', () => {
|
||||
expect(resolveAgentWorkspaceExecutionHostId('missing::/repo', deps)).toBeNull()
|
||||
expect(resolveAgentWorkspaceExecutionHostId(undefined, deps)).toBeNull()
|
||||
expect(isLocalExecutionHost(null)).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
import {
|
||||
LOCAL_EXECUTION_HOST_ID,
|
||||
parseExecutionHostId,
|
||||
toSshExecutionHostId,
|
||||
type ExecutionHostId
|
||||
} from '../../shared/execution-host'
|
||||
import type { FolderWorkspace, ProjectGroup } from '../../shared/types'
|
||||
import { getRepoIdFromWorktreeId } from '../../shared/worktree-id'
|
||||
import { parseWorkspaceKey } from '../../shared/workspace-scope'
|
||||
|
||||
export type RestoredSubagentLivenessSweepDeps = {
|
||||
/** Targeted provider liveness, or null when the provider cannot prove either state. */
|
||||
probeLiveLocalPty: (ptyId: string) => Promise<boolean | null>
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean
|
||||
/** PTY bound to this pane in the current session, if it has one. */
|
||||
getBoundPtyIdForPaneKey: (paneKey: string) => string | undefined
|
||||
/** PTY this pane was bound to when the session was last persisted; covers panes
|
||||
* whose surviving daemon session has not been reattached yet. */
|
||||
getPersistedPtyIdForPaneKey: (paneKey: string) => string | undefined
|
||||
reap: (
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean,
|
||||
isLocalPaneAgentLive: (paneKey: string) => Promise<boolean>,
|
||||
isLocalPaneLivenessEvidenceCurrent: (paneKey: string) => boolean
|
||||
) => Promise<number>
|
||||
}
|
||||
|
||||
/** Drop restored rows only when the owning host is local and its provider proves
|
||||
* the exact PTY absent. */
|
||||
export async function sweepRestoredSubagentsWithoutLiveAgent(
|
||||
deps: RestoredSubagentLivenessSweepDeps
|
||||
): Promise<number> {
|
||||
const probesByPtyId = new Map<string, Promise<boolean | null>>()
|
||||
const boundPtyIdAtProbeByPaneKey = new Map<string, string | undefined>()
|
||||
return await deps.reap(
|
||||
(worktreeId) => deps.isLocalExecutionHost(worktreeId),
|
||||
async (paneKey) => {
|
||||
const boundPtyId = deps.getBoundPtyIdForPaneKey(paneKey)
|
||||
boundPtyIdAtProbeByPaneKey.set(paneKey, boundPtyId)
|
||||
const ptyId = boundPtyId ?? deps.getPersistedPtyIdForPaneKey(paneKey)
|
||||
if (!ptyId) {
|
||||
return true
|
||||
}
|
||||
try {
|
||||
let probe = probesByPtyId.get(ptyId)
|
||||
if (!probe) {
|
||||
probe = deps.probeLiveLocalPty(ptyId)
|
||||
probesByPtyId.set(ptyId, probe)
|
||||
}
|
||||
const live = await probe
|
||||
const currentBoundPtyId = deps.getBoundPtyIdForPaneKey(paneKey)
|
||||
// Why: cold restore can rebind the persisted id while its absence probe is in flight.
|
||||
return currentBoundPtyId !== boundPtyId || live !== false
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
},
|
||||
(paneKey) =>
|
||||
!boundPtyIdAtProbeByPaneKey.has(paneKey) ||
|
||||
deps.getBoundPtyIdForPaneKey(paneKey) === boundPtyIdAtProbeByPaneKey.get(paneKey)
|
||||
)
|
||||
}
|
||||
|
||||
/** Index the persisted terminal layouts as `paneKey -> ptyId`. Layout leaves are
|
||||
* the only persisted binding that carries a stable pane key, so tab-level PTY ids
|
||||
* (legacy numeric panes) are deliberately skipped. */
|
||||
export function indexPersistedPaneKeyPtyIds(
|
||||
layoutsByTabId: Record<string, { ptyIdsByLeafId?: Record<string, string> } | undefined>
|
||||
): Map<string, string> {
|
||||
const byPaneKey = new Map<string, string>()
|
||||
for (const [tabId, layout] of Object.entries(layoutsByTabId)) {
|
||||
for (const [leafId, ptyId] of Object.entries(layout?.ptyIdsByLeafId ?? {})) {
|
||||
if (ptyId) {
|
||||
byPaneKey.set(`${tabId}:${leafId}`, ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
return byPaneKey
|
||||
}
|
||||
|
||||
type AgentWorkspaceExecutionHostDeps = {
|
||||
getRepo: (repoId: string) => ExecutionHostOwner | null | undefined
|
||||
getWorktreeMeta: (worktreeId: string) => { hostId?: string | null } | null | undefined
|
||||
getFolderWorkspace: (
|
||||
folderWorkspaceId: string
|
||||
) => Pick<FolderWorkspace, 'projectGroupId' | 'connectionId'> | null | undefined
|
||||
getProjectGroups: () => readonly Pick<ProjectGroup, 'id' | 'connectionId' | 'executionHostId'>[]
|
||||
}
|
||||
|
||||
type ExecutionHostOwner = {
|
||||
connectionId?: string | null
|
||||
executionHostId?: string | null
|
||||
}
|
||||
|
||||
function resolveDeclaredExecutionHost(owner: ExecutionHostOwner): ExecutionHostId | null {
|
||||
if (owner.executionHostId?.trim()) {
|
||||
return parseExecutionHostId(owner.executionHostId)?.id ?? null
|
||||
}
|
||||
const connectionId = owner.connectionId?.trim()
|
||||
return connectionId ? toSshExecutionHostId(connectionId) : LOCAL_EXECUTION_HOST_ID
|
||||
}
|
||||
|
||||
/** Resolve persisted workspace ownership; unknown provenance is not local authority. */
|
||||
export function resolveAgentWorkspaceExecutionHostId(
|
||||
workspaceId: string | undefined,
|
||||
deps: AgentWorkspaceExecutionHostDeps
|
||||
): ExecutionHostId | null {
|
||||
if (!workspaceId) {
|
||||
return null
|
||||
}
|
||||
const scope = parseWorkspaceKey(workspaceId)
|
||||
if (scope?.type === 'folder') {
|
||||
const workspace = deps.getFolderWorkspace(scope.folderWorkspaceId)
|
||||
const group = workspace
|
||||
? deps.getProjectGroups().find((candidate) => candidate.id === workspace.projectGroupId)
|
||||
: undefined
|
||||
if (!workspace || !group) {
|
||||
return null
|
||||
}
|
||||
return resolveDeclaredExecutionHost({
|
||||
connectionId: workspace.connectionId ?? group.connectionId,
|
||||
executionHostId: group.executionHostId
|
||||
})
|
||||
}
|
||||
const worktreeId = scope?.type === 'worktree' ? scope.worktreeId : workspaceId
|
||||
const declaredWorktreeHost = deps.getWorktreeMeta(worktreeId)?.hostId?.trim()
|
||||
if (declaredWorktreeHost) {
|
||||
return parseExecutionHostId(declaredWorktreeHost)?.id ?? null
|
||||
}
|
||||
const repo = deps.getRepo(getRepoIdFromWorktreeId(worktreeId))
|
||||
return repo ? resolveDeclaredExecutionHost(repo) : null
|
||||
}
|
||||
|
||||
export function isLocalExecutionHost(hostId: string | null | undefined): boolean {
|
||||
return parseExecutionHostId(hostId)?.kind === 'local'
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
normalizeHookPayload,
|
||||
parseFormEncodedBody,
|
||||
readRequestBody,
|
||||
reapRestoredClaudeSubagentsForDeadPane,
|
||||
reconcileRemoteCodexState,
|
||||
resolveHookSource,
|
||||
preparePendingGrokResultDiscovery,
|
||||
@@ -35,6 +36,11 @@ import {
|
||||
type AgentHookEventPayload,
|
||||
type HookListenerState
|
||||
} from '../../shared/agent-hook-listener'
|
||||
import {
|
||||
claudeRosterHasRestoredSnapshotSubagent,
|
||||
claudeRosterHasWorkingSubagent,
|
||||
claudeRosterToSnapshots
|
||||
} from '../../shared/claude-subagent-roster'
|
||||
import type { AgentHookSource } from '../../shared/agent-hook-relay'
|
||||
import {
|
||||
CLAUDE_STATUSLINE_PATHNAME,
|
||||
@@ -2171,6 +2177,91 @@ export class AgentHookServer {
|
||||
}
|
||||
}
|
||||
|
||||
/** Second reap path for restored Claude subagent rows: drop the ones whose pane
|
||||
* has no live local agent process behind it any more. A PTY that dies while Orca
|
||||
* is down never runs the teardown that clears pane state, so hydrate rebuilds a
|
||||
* roster nothing can ever retire — the inventory reap needs the parent to emit a
|
||||
* complete `background_tasks` list and an idle parent never does. The row then
|
||||
* gates the pane 'working' for the rest of its life and hibernation, which
|
||||
* requires 'done', can never reclaim the agent's heap.
|
||||
*
|
||||
* Both the execution host and relay binding must prove local ownership before
|
||||
* targeted PTY liveness is consulted. Panes that reported in this runtime are
|
||||
* also skipped. Returns the number of panes changed. */
|
||||
async reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean,
|
||||
isLocalPaneAgentLive: (paneKey: string) => Promise<boolean>,
|
||||
isLocalPaneLivenessEvidenceCurrent: (paneKey: string) => boolean
|
||||
): Promise<number> {
|
||||
const candidates: { paneKey: string; entry: EnrichedAgentHookEventPayload }[] = []
|
||||
for (const [paneKey, entry] of this.state.lastStatusByPaneKey) {
|
||||
const enriched = entry as EnrichedAgentHookEventPayload
|
||||
if (
|
||||
enriched.payload.agentType === 'claude' &&
|
||||
enriched.connectionId === null &&
|
||||
isLocalExecutionHost(enriched.worktreeId) &&
|
||||
claudeRosterHasRestoredSnapshotSubagent(
|
||||
this.state.claudeSubagentRosterByPaneKey.get(paneKey)
|
||||
) &&
|
||||
!this.runtimeObservedStatusPaneKeys.has(paneKey)
|
||||
) {
|
||||
candidates.push({ paneKey, entry: enriched })
|
||||
}
|
||||
}
|
||||
const liveness = await Promise.all(
|
||||
candidates.map(async (candidate) => {
|
||||
try {
|
||||
return await isLocalPaneAgentLive(candidate.paneKey)
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
})
|
||||
)
|
||||
let changedPanes = 0
|
||||
for (const [index, candidate] of candidates.entries()) {
|
||||
const { paneKey, entry: enriched } = candidate
|
||||
if (
|
||||
liveness[index] ||
|
||||
!isLocalPaneLivenessEvidenceCurrent(paneKey) ||
|
||||
this.state.lastStatusByPaneKey.get(paneKey) !== enriched ||
|
||||
this.runtimeObservedStatusPaneKeys.has(paneKey) ||
|
||||
!isLocalExecutionHost(enriched.worktreeId)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
if (!reapRestoredClaudeSubagentsForDeadPane(this.state, paneKey)) {
|
||||
continue
|
||||
}
|
||||
changedPanes += 1
|
||||
const roster = this.state.claudeSubagentRosterByPaneKey.get(paneKey)
|
||||
const subagents = claudeRosterToSnapshots(roster)
|
||||
// Why: the pane's persisted 'working' was the child gate holding a finished
|
||||
// lead open (subagent events never set lead state). With the last working row
|
||||
// gone and no process left to report, 'done' is the only truthful state — and
|
||||
// the one hibernation needs once this pane's agent is restored.
|
||||
const state =
|
||||
enriched.payload.state === 'working' && !claudeRosterHasWorkingSubagent(roster)
|
||||
? 'done'
|
||||
: enriched.payload.state
|
||||
const stateChanged = state !== enriched.payload.state
|
||||
const reconciledAt = stateChanged
|
||||
? Math.max(Date.now(), enriched.receivedAt + 1)
|
||||
: enriched.receivedAt
|
||||
const reconciled: EnrichedAgentHookEventPayload = {
|
||||
...enriched,
|
||||
receivedAt: reconciledAt,
|
||||
stateStartedAt: stateChanged ? reconciledAt : enriched.stateStartedAt,
|
||||
payload: { ...enriched.payload, state, subagents }
|
||||
}
|
||||
this.state.lastStatusByPaneKey.set(paneKey, reconciled)
|
||||
}
|
||||
if (changedPanes > 0) {
|
||||
this.scheduleStatusPersist()
|
||||
this.notifyStatusChangeListeners()
|
||||
}
|
||||
return changedPanes
|
||||
}
|
||||
|
||||
buildPtyEnv(): Record<string, string> {
|
||||
if (this.port <= 0 || !this.token) {
|
||||
return {}
|
||||
|
||||
@@ -16,14 +16,20 @@ type FakeChild = ChildProcessWithoutNullStreams & { kill: ReturnType<typeof vi.f
|
||||
|
||||
function fakeChild(): FakeChild {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdout: EventEmitter
|
||||
stdout: EventEmitter & {
|
||||
pause: ReturnType<typeof vi.fn>
|
||||
resume: ReturnType<typeof vi.fn>
|
||||
}
|
||||
stderr: EventEmitter
|
||||
stdin: { write: ReturnType<typeof vi.fn> }
|
||||
stdin: EventEmitter & { write: ReturnType<typeof vi.fn> }
|
||||
kill: ReturnType<typeof vi.fn>
|
||||
}
|
||||
child.stdout = new EventEmitter()
|
||||
child.stdout = Object.assign(new EventEmitter(), {
|
||||
pause: vi.fn(),
|
||||
resume: vi.fn()
|
||||
})
|
||||
child.stderr = new EventEmitter()
|
||||
child.stdin = { write: vi.fn(() => true) }
|
||||
child.stdin = Object.assign(new EventEmitter(), { write: vi.fn(() => true) })
|
||||
child.kill = vi.fn()
|
||||
return child as unknown as FakeChild
|
||||
}
|
||||
@@ -62,6 +68,10 @@ describe('waitForWslRelaySentinel', () => {
|
||||
const transport = await promise
|
||||
expect(typeof transport.write).toBe('function')
|
||||
expect(typeof transport.onData).toBe('function')
|
||||
transport.pauseReads?.()
|
||||
transport.resumeReads?.()
|
||||
expect(child.stdout.pause).toHaveBeenCalledOnce()
|
||||
expect(child.stdout.resume).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('resolves past leading garbage and hands trailing bytes to onData', async () => {
|
||||
@@ -149,4 +159,25 @@ describe('waitForWslRelaySentinel', () => {
|
||||
expect(err.message).toContain('E_FAIL')
|
||||
expect(err.message).not.toContain(String.fromCharCode(0))
|
||||
})
|
||||
|
||||
it('forwards write(false), callback settlement, and drain from WSL stdin', async () => {
|
||||
const child = fakeChild()
|
||||
const callback = vi.fn()
|
||||
const drain = vi.fn()
|
||||
const writeMock = child.stdin.write as ReturnType<typeof vi.fn>
|
||||
writeMock.mockImplementation((_data, onWritten) => {
|
||||
onWritten(null)
|
||||
return false
|
||||
})
|
||||
const promise = waitForWslRelaySentinel(child)
|
||||
emitStdout(child, RELAY_SENTINEL)
|
||||
const transport = await promise
|
||||
|
||||
transport.onDrain?.(drain)
|
||||
expect(transport.write(Buffer.from('frame'), callback)).toBe(false)
|
||||
expect(callback).toHaveBeenCalledWith({ ok: true })
|
||||
child.stdin.emit('drain')
|
||||
expect(drain).toHaveBeenCalledOnce()
|
||||
expect(transport.supportsWriteSettlement).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -123,12 +123,15 @@ export function waitForWslRelaySentinel(
|
||||
pendingChunks.push(trailing)
|
||||
}
|
||||
const transport: MultiplexerTransport = {
|
||||
write: (data) => {
|
||||
try {
|
||||
child.stdin.write(data)
|
||||
} catch {
|
||||
// Channel already closing — mux close handling takes over.
|
||||
}
|
||||
write: (data, onSettled) => {
|
||||
return child.stdin.write(data, (error?: Error | null) => {
|
||||
onSettled?.(error ? { ok: false, error } : { ok: true })
|
||||
})
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onDrain: (cb) => {
|
||||
child.stdin.on('drain', cb)
|
||||
return () => child.stdin.off('drain', cb)
|
||||
},
|
||||
onData: (cb) => {
|
||||
dataCallbacks.push(cb)
|
||||
@@ -143,6 +146,8 @@ export function waitForWslRelaySentinel(
|
||||
}
|
||||
},
|
||||
onClose: (cb) => closeCallbacks.push(cb),
|
||||
pauseReads: () => child.stdout.pause(),
|
||||
resumeReads: () => child.stdout.resume(),
|
||||
close: () => child.kill()
|
||||
}
|
||||
resolve(transport)
|
||||
|
||||
@@ -25,27 +25,4 @@ describe('packaged Windows CLI launcher asset', () => {
|
||||
expect(source).toContain('child.WaitForExit();')
|
||||
expect(source).toContain('return child.ExitCode;')
|
||||
})
|
||||
|
||||
it('requires exact updater process and executable release proof', () => {
|
||||
const includePath = join(process.cwd(), 'config', 'nsis', 'daemon-host-uninstall.nsh')
|
||||
const source = readFileSync(includePath, 'utf8')
|
||||
|
||||
expect(source).toContain('!macro customCheckAppRunning')
|
||||
expect(source).toContain('!include "getProcessInfo.nsh"')
|
||||
expect(source).toContain('Var pid')
|
||||
expect(source).toContain('[String]::Equals')
|
||||
expect(source).toContain('"$INSTDIR\\Orca.exe" "$INSTDIR\\resources\\bin\\orca.exe"')
|
||||
expect(source).toContain('CreationDate=[string]$$parent.CreationDate')
|
||||
expect(source).toContain("'ProcessId=' + $$proof.ProcessId")
|
||||
expect(source).toContain('[IO.FileShare]::None')
|
||||
expect(source).toContain('$$released=$$false')
|
||||
expect(source).toContain('$$released=$$true')
|
||||
expect(source).not.toMatch(/(?<!\$)(?:\$\$)*\$(?:false|true)\b/)
|
||||
expect(source).toContain(
|
||||
' ${else}\n !insertmacro IS_POWERSHELL_AVAILABLE\n !insertmacro _CHECK_APP_RUNNING\n ${endIf}'
|
||||
)
|
||||
expect(source).toContain('SetErrorLevel 2')
|
||||
expect(source).toContain('Quit')
|
||||
expect(source).not.toContain('taskkill /IM Orca.exe')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1180,6 +1180,30 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('probePtyLiveness', () => {
|
||||
it('reads daemon truth before a fresh adapter has attached the session', async () => {
|
||||
const { id } = await adapter.spawn({ cols: 80, rows: 24 })
|
||||
const activeSessionIds = (adapter as unknown as { activeSessionIds: Set<string> })
|
||||
.activeSessionIds
|
||||
activeSessionIds.clear()
|
||||
|
||||
expect(adapter.hasPty(id)).toBe(false)
|
||||
await expect(adapter.probePtyLiveness(id)).resolves.toBe(true)
|
||||
await expect(adapter.probePtyLiveness('missing-session')).resolves.toBe(false)
|
||||
})
|
||||
|
||||
it('returns unknown when the daemon cannot answer', async () => {
|
||||
const client = (
|
||||
adapter as unknown as {
|
||||
client: { request: (type: string, payload?: unknown) => Promise<unknown> }
|
||||
}
|
||||
).client
|
||||
vi.spyOn(client, 'request').mockRejectedValueOnce(new Error('unavailable'))
|
||||
|
||||
await expect(adapter.probePtyLiveness('session')).resolves.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getBufferSnapshot', () => {
|
||||
it('returns the daemon model with its absolute stream sequence', async () => {
|
||||
const { id } = await adapter.spawn({ cols: 80, rows: 24 })
|
||||
|
||||
@@ -813,6 +813,18 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
||||
return this.activeSessionIds.has(id)
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
try {
|
||||
const result = await this.client.request<{ size: { cols: number; rows: number } | null }>(
|
||||
'getSize',
|
||||
{ sessionId: id }
|
||||
)
|
||||
return result.size !== null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
write(id: string, data: string): void {
|
||||
this.markSessionDirty(id)
|
||||
// Why recoverable and not just active: rejecting a write asks the pane to remount,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { IPtyProvider } from '../providers/types'
|
||||
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
|
||||
export async function probePtyOwners(
|
||||
id: string,
|
||||
routed: IPtyProvider | undefined,
|
||||
possibleOwners: readonly DaemonPtyAdapter[]
|
||||
): Promise<boolean | null> {
|
||||
if (routed) {
|
||||
return routed.probePtyLiveness
|
||||
? await routed.probePtyLiveness(id)
|
||||
: (routed.hasPty?.(id) ?? null)
|
||||
}
|
||||
const results = await Promise.all(possibleOwners.map((provider) => provider.probePtyLiveness(id)))
|
||||
return results.some((result) => result === true)
|
||||
? true
|
||||
: results.every((result) => result === false)
|
||||
? false
|
||||
: null
|
||||
}
|
||||
@@ -62,6 +62,7 @@ function createAdapter(
|
||||
}))
|
||||
),
|
||||
hasPty: vi.fn((id: string) => sessions.includes(id)),
|
||||
probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)),
|
||||
write: vi.fn((id: string, data: string) => {
|
||||
writes.push({ id, data })
|
||||
}),
|
||||
@@ -470,6 +471,25 @@ describe('DaemonPtyRouter', () => {
|
||||
expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session')
|
||||
})
|
||||
|
||||
it('probes every possible daemon owner for an unmapped session', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy', ['surviving-session'])
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
await expect(router.probePtyLiveness('surviving-session')).resolves.toBe(true)
|
||||
expect(current.probePtyLiveness).toHaveBeenCalledExactlyOnceWith('surviving-session')
|
||||
expect(legacy.probePtyLiveness).toHaveBeenCalledExactlyOnceWith('surviving-session')
|
||||
})
|
||||
|
||||
it('does not report absence while any possible daemon owner is unavailable', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy')
|
||||
vi.mocked(legacy.probePtyLiveness).mockResolvedValue(null)
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
await expect(router.probePtyLiveness('unknown-session')).resolves.toBeNull()
|
||||
})
|
||||
|
||||
it('fails listProcesses closed when any routed adapter cannot list sessions', async () => {
|
||||
const current = createAdapter('current', ['current-session'])
|
||||
const legacy = createAdapter('legacy', ['legacy-session'])
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
} from '../providers/types'
|
||||
import type { PtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtyProcessInspection } from '../providers/pty-process-inspection'
|
||||
import { probePtyOwners } from './daemon-pty-liveness-probe'
|
||||
|
||||
export class DaemonPtyRouter implements IPtyProvider {
|
||||
private current: DaemonPtyAdapter
|
||||
@@ -108,6 +109,10 @@ export class DaemonPtyRouter implements IPtyProvider {
|
||||
return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id))
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
return await probePtyOwners(id, this.sessionAdapters.get(id), this.allAdapters())
|
||||
}
|
||||
|
||||
write(id: string, data: string): void {
|
||||
this.adapterFor(id).write(id, data)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers
|
||||
import type { PtyProcessInspection } from '../providers/pty-process-inspection'
|
||||
|
||||
type ProviderMock = IPtyProvider & {
|
||||
probePtyLiveness: (id: string) => Promise<boolean | null>
|
||||
inspectProcess: (id: string) => Promise<PtyProcessInspection>
|
||||
emitData: (id: string, data: string, sequenceChars?: number) => void
|
||||
emitReplay: (id: string, data: string) => void
|
||||
@@ -31,6 +32,7 @@ function createProvider(
|
||||
}),
|
||||
attach: vi.fn(async () => {}),
|
||||
hasPty: vi.fn((id: string) => sessions.includes(id)),
|
||||
probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)),
|
||||
providesAgentSessionOwnerListings: vi.fn(() => authoritativeOwnerListings),
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
@@ -254,6 +256,20 @@ describe('DegradedDaemonPtyProvider', () => {
|
||||
expect(fallback.write).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('probes daemon owners without borrowing fallback liveness', async () => {
|
||||
const current = createDaemonAdapter('current')
|
||||
const legacy = createDaemonAdapter('legacy')
|
||||
const fallback = createProvider('fallback', ['unknown-session'])
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [legacy], fallback })
|
||||
vi.mocked(legacy.probePtyLiveness).mockResolvedValue(null)
|
||||
|
||||
await expect(provider.probePtyLiveness('unknown-session')).resolves.toBeNull()
|
||||
expect(fallback.probePtyLiveness).not.toHaveBeenCalled()
|
||||
|
||||
vi.mocked(current.probePtyLiveness).mockResolvedValue(true)
|
||||
await expect(provider.probePtyLiveness('unknown-session')).resolves.toBe(true)
|
||||
})
|
||||
|
||||
it('routes authoritative recovery snapshots to the owning daemon', async () => {
|
||||
const current = createDaemonAdapter('daemon', ['daemon-session'])
|
||||
const fallback = createProvider('fallback')
|
||||
|
||||
@@ -11,6 +11,8 @@ import type {
|
||||
PtySpawnOptions,
|
||||
PtySpawnResult
|
||||
} from '../providers/types'
|
||||
import { findDaemonAdapter, listProviderSessionIds } from './degraded-daemon-session-routing'
|
||||
import { probePtyOwners } from './daemon-pty-liveness-probe'
|
||||
|
||||
export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
readonly routesFreshSpawnsToLocalProvider = true
|
||||
@@ -81,6 +83,10 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
return mapped ? (mapped.hasPty?.(id) ?? true) : this.findProviderForExistingSession(id) !== null
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
return await probePtyOwners(id, this.sessionProviders.get(id), this.allDaemonAdapters())
|
||||
}
|
||||
|
||||
// Why: an unknown id cannot borrow listing authority from the fresh-spawn provider.
|
||||
providesAgentSessionOwnerListings = (ptyId: string): boolean =>
|
||||
(
|
||||
@@ -246,11 +252,15 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
}
|
||||
|
||||
ackColdRestore(sessionId: string): void {
|
||||
this.daemonAdapterFor(sessionId)?.ackColdRestore(sessionId)
|
||||
findDaemonAdapter(this.sessionProviders, this.allDaemonAdapters(), sessionId)?.ackColdRestore(
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
|
||||
clearTombstone(sessionId: string): void {
|
||||
this.daemonAdapterFor(sessionId)?.clearTombstone(sessionId)
|
||||
findDaemonAdapter(this.sessionProviders, this.allDaemonAdapters(), sessionId)?.clearTombstone(
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
|
||||
async reconcileOnStartup(validWorktreeIds: Set<string>): Promise<{
|
||||
@@ -289,7 +299,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
}
|
||||
|
||||
getCurrentDaemonSessionIds(): string[] {
|
||||
return this.sessionIdsForProvider(this.current)
|
||||
return listProviderSessionIds(this.sessionProviders, this.current)
|
||||
}
|
||||
|
||||
fanoutCurrentDaemonSyntheticExits(code: number): void {
|
||||
@@ -338,19 +348,6 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
return null
|
||||
}
|
||||
|
||||
private sessionIdsForProvider(provider: IPtyProvider): string[] {
|
||||
return [...this.sessionProviders]
|
||||
.filter(([, mappedProvider]) => mappedProvider === provider)
|
||||
.map(([id]) => id)
|
||||
}
|
||||
|
||||
private daemonAdapterFor(sessionId: string): DaemonPtyAdapter | null {
|
||||
const provider = this.sessionProviders.get(sessionId)
|
||||
return provider && this.allDaemonAdapters().includes(provider as DaemonPtyAdapter)
|
||||
? (provider as DaemonPtyAdapter)
|
||||
: null
|
||||
}
|
||||
|
||||
private allProviders(): IPtyProvider[] {
|
||||
return [this.fallback, ...this.allDaemonAdapters()]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { IPtyProvider } from '../providers/types'
|
||||
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
|
||||
export function listProviderSessionIds(
|
||||
sessionProviders: ReadonlyMap<string, IPtyProvider>,
|
||||
provider: IPtyProvider
|
||||
): string[] {
|
||||
return [...sessionProviders]
|
||||
.filter(([, mappedProvider]) => mappedProvider === provider)
|
||||
.map(([id]) => id)
|
||||
}
|
||||
|
||||
export function findDaemonAdapter(
|
||||
sessionProviders: ReadonlyMap<string, IPtyProvider>,
|
||||
daemonAdapters: readonly DaemonPtyAdapter[],
|
||||
sessionId: string
|
||||
): DaemonPtyAdapter | null {
|
||||
const provider = sessionProviders.get(sessionId)
|
||||
return provider && daemonAdapters.includes(provider as DaemonPtyAdapter)
|
||||
? (provider as DaemonPtyAdapter)
|
||||
: null
|
||||
}
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
|
||||
const TIMEOUT_MS = 45_000
|
||||
const CHECK_INTERVAL_MS = 5_000
|
||||
@@ -8,7 +8,7 @@ function loopWithClock(startAt = 0) {
|
||||
let now = startAt
|
||||
const onHangDetected = vi.fn()
|
||||
const onHangResolved = vi.fn()
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: TIMEOUT_MS,
|
||||
checkIntervalMs: CHECK_INTERVAL_MS,
|
||||
now: () => now,
|
||||
@@ -18,7 +18,7 @@ function loopWithClock(startAt = 0) {
|
||||
return { loop, onHangDetected, onHangResolved, advance: (ms: number) => (now += ms) }
|
||||
}
|
||||
|
||||
describe('createHangWatchdogChildLoop', () => {
|
||||
describe('createHangWatchdogDetectionLoop', () => {
|
||||
it('does not fire while heartbeats keep arriving', () => {
|
||||
const { loop, onHangDetected, advance } = loopWithClock()
|
||||
for (let i = 0; i < 100; i++) {
|
||||
+5
-5
@@ -1,4 +1,4 @@
|
||||
export type HangWatchdogChildLoopConfig = {
|
||||
export type HangWatchdogDetectionLoopConfig = {
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
now: () => number
|
||||
@@ -7,14 +7,14 @@ export type HangWatchdogChildLoopConfig = {
|
||||
onHangResolved: (unresponsiveMs: number) => void
|
||||
}
|
||||
|
||||
export type HangWatchdogChildLoop = {
|
||||
export type HangWatchdogDetectionLoop = {
|
||||
recordHeartbeat: () => void
|
||||
tick: () => void
|
||||
}
|
||||
|
||||
export function createHangWatchdogChildLoop(
|
||||
config: HangWatchdogChildLoopConfig
|
||||
): HangWatchdogChildLoop {
|
||||
export function createHangWatchdogDetectionLoop(
|
||||
config: HangWatchdogDetectionLoopConfig
|
||||
): HangWatchdogDetectionLoop {
|
||||
let lastHeartbeatAt = config.now()
|
||||
let lastTickAt = config.now()
|
||||
let detected = false
|
||||
@@ -1,17 +0,0 @@
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogEntryPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
// Why: ELECTRON_RUN_AS_NODE bypasses asar integration, so the packaged entry must be forked from app.asar.unpacked.
|
||||
const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath
|
||||
const adjacentBuildEntry = join(basePath, 'main-thread-hang-watchdog-entry.js')
|
||||
// Why: electron-vite's unpackaged appPath is already out/main; appending out/main again would silently disable the watchdog in dev and E2E builds.
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(basePath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
|
||||
describe('resolveHangWatchdogWorkerPath', () => {
|
||||
it('resolves packaged workers inside app.asar', () => {
|
||||
const appPath = join('/apps', 'orca', 'app.asar')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, true)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
|
||||
it('uses an adjacent entry when the dev app path is out/main', () => {
|
||||
const appPath = join('/repo', 'out', 'main')
|
||||
const adjacent = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
const pathExists = vi.fn((candidate: string) => candidate === adjacent)
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, pathExists)).toBe(adjacent)
|
||||
})
|
||||
|
||||
it('resolves through out/main from a dev project root', () => {
|
||||
const appPath = join('/repo', 'orca')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, () => false)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,14 @@
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogWorkerPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
const adjacentBuildEntry = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export const HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS = 2_000
|
||||
export const HANG_WATCHDOG_TIMEOUT_MS = 45_000
|
||||
export const HANG_WATCHDOG_CHECK_INTERVAL_MS = 5_000
|
||||
|
||||
export type HangWatchdogWorkerData = {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
|
||||
export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' }
|
||||
@@ -67,9 +67,6 @@ describe('recordHangObservation', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// Why: this is the safety contract of the whole PR. Observing a hang must never kill, relaunch,
|
||||
// or exit — a false positive would SIGKILL a live main thread mid-write. If a future change
|
||||
// reintroduces recovery, this test must fail loudly rather than ship silently.
|
||||
it('never spawns, signals, or exits', () => {
|
||||
recordHangObservation({
|
||||
parentPid: 4242,
|
||||
|
||||
@@ -1,20 +1,12 @@
|
||||
// Forked with ELECTRON_RUN_AS_NODE from the main process. Watches heartbeats from the main
|
||||
// thread; if they stop (e.g. the macOS 26 AppKit scene-update deadlock) it records a marker so
|
||||
// the next launch can report the stall, and rewrites it if the heartbeats come back.
|
||||
//
|
||||
// Why no kill: a deadlocked main thread has already lost whatever sat in the persistence debounce
|
||||
// window, so killing it recovers nothing the user could not recover by force-quitting. A false
|
||||
// positive, though, would SIGKILL a live main thread that was merely blocked — most plausibly on
|
||||
// I/O, i.e. exactly when writes are in flight. All of the downside sits in the misfire, so this
|
||||
// measures first: `selfRecovered` counts the stalls a killer would have gotten wrong.
|
||||
//
|
||||
// Must never import electron.
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { isMainThread, parentPort, workerData } from 'node:worker_threads'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
import { writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import type {
|
||||
HangWatchdogWorkerData,
|
||||
MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 45_000
|
||||
const DEFAULT_CHECK_INTERVAL_MS = 5_000
|
||||
|
||||
// Observation only: a false positive must never kill a live main thread mid-write.
|
||||
export function recordHangObservation(options: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
@@ -36,37 +28,63 @@ export function recordHangObservation(options: {
|
||||
}
|
||||
}
|
||||
|
||||
function runWatchdog(parentPid: number): void {
|
||||
const markerPath = process.env.ORCA_HANG_WATCHDOG_MARKER_PATH ?? ''
|
||||
const timeoutMs = Number(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS
|
||||
const checkIntervalMs =
|
||||
Number(process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS) || DEFAULT_CHECK_INTERVAL_MS
|
||||
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
timeoutMs,
|
||||
checkIntervalMs,
|
||||
function runWatchdog(config: HangWatchdogWorkerData): void {
|
||||
const port = parentPort
|
||||
if (!port) {
|
||||
return
|
||||
}
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: config.timeoutMs,
|
||||
checkIntervalMs: config.checkIntervalMs,
|
||||
now: () => Date.now(),
|
||||
onHangDetected: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: false }),
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: false
|
||||
}),
|
||||
// Why: rewriting the marker keeps one observation per stall rather than two rows to reconcile.
|
||||
onHangResolved: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: true })
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: true
|
||||
})
|
||||
})
|
||||
|
||||
process.on('message', (message) => {
|
||||
const type = (message as { type?: string } | null)?.type
|
||||
if (type === 'heartbeat') {
|
||||
let checkTimer: ReturnType<typeof setInterval> | null = setInterval(
|
||||
() => loop.tick(),
|
||||
config.checkIntervalMs
|
||||
)
|
||||
port.on('message', (message: MainToHangWatchdogWorkerMessage) => {
|
||||
if (message.type === 'heartbeat') {
|
||||
loop.recordHeartbeat()
|
||||
} else if (type === 'shutdown') {
|
||||
process.exit(0)
|
||||
} else if (message.type === 'shutdown') {
|
||||
if (checkTimer) {
|
||||
clearInterval(checkTimer)
|
||||
checkTimer = null
|
||||
}
|
||||
port.close()
|
||||
}
|
||||
})
|
||||
// Why: a normal parent exit closes the IPC channel; the watchdog must not outlive it and misfire.
|
||||
process.on('disconnect', () => process.exit(0))
|
||||
setInterval(() => loop.tick(), checkIntervalMs)
|
||||
}
|
||||
|
||||
const configuredParentPid = Number(process.env.ORCA_HANG_WATCHDOG_PARENT_PID)
|
||||
if (Number.isInteger(configuredParentPid) && configuredParentPid > 0) {
|
||||
runWatchdog(configuredParentPid)
|
||||
function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData {
|
||||
const data = value as Partial<HangWatchdogWorkerData> | null
|
||||
return (
|
||||
!!data &&
|
||||
Number.isInteger(data.parentPid) &&
|
||||
(data.parentPid ?? 0) > 0 &&
|
||||
typeof data.markerPath === 'string' &&
|
||||
Number.isFinite(data.timeoutMs) &&
|
||||
(data.timeoutMs ?? 0) > 0 &&
|
||||
Number.isFinite(data.checkIntervalMs) &&
|
||||
(data.checkIntervalMs ?? 0) > 0
|
||||
)
|
||||
}
|
||||
|
||||
if (!isMainThread && isHangWatchdogWorkerData(workerData)) {
|
||||
runWatchdog(workerData)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const { forkMock, appMock } = vi.hoisted(() => ({
|
||||
forkMock: vi.fn(),
|
||||
const { workerState, appMock } = vi.hoisted(() => ({
|
||||
workerState: {
|
||||
calls: [] as unknown[][],
|
||||
instance: null as object | null,
|
||||
error: null as Error | null
|
||||
},
|
||||
appMock: {
|
||||
isPackaged: true,
|
||||
getAppPath: vi.fn(() => '/apps/orca/app.asar'),
|
||||
@@ -9,8 +14,16 @@ const { forkMock, appMock } = vi.hoisted(() => ({
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('node:child_process', () => ({
|
||||
fork: forkMock
|
||||
vi.mock('node:worker_threads', () => ({
|
||||
Worker: class WorkerMock {
|
||||
constructor(...args: unknown[]) {
|
||||
workerState.calls.push(args)
|
||||
if (workerState.error) {
|
||||
throw workerState.error
|
||||
}
|
||||
return workerState.instance as WorkerMock
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
@@ -29,28 +42,33 @@ function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
|
||||
}
|
||||
}
|
||||
|
||||
function fakeChild() {
|
||||
function fakeWorker() {
|
||||
return {
|
||||
connected: true,
|
||||
stderr: { on: vi.fn() },
|
||||
postMessage: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
on: vi.fn(),
|
||||
send: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
kill: vi.fn()
|
||||
once: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
describe('installMainThreadHangWatchdog', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
forkMock.mockReset()
|
||||
workerState.calls = []
|
||||
workerState.instance = null
|
||||
workerState.error = null
|
||||
appMock.on.mockReset()
|
||||
appMock.isPackaged = true
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
it('is a no-op off macOS', () => {
|
||||
@@ -60,7 +78,7 @@ describe('installMainThreadHangWatchdog', () => {
|
||||
expect(
|
||||
withPlatform('linux', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
expect(workerState.calls).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('is a no-op in unpackaged builds unless forced', () => {
|
||||
@@ -69,62 +87,98 @@ describe('installMainThreadHangWatchdog', () => {
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
process.env.ORCA_HANG_WATCHDOG_FORCE = '1'
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).not.toBeNull()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
})
|
||||
|
||||
it('forks the watchdog as plain Node with pid, bundle, and marker config', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('starts a worker with pid, marker, and timing config', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
expect(handle).not.toBeNull()
|
||||
const [, , options] = forkMock.mock.calls[0]
|
||||
expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1')
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_PARENT_PID).toBe(String(process.pid))
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_MARKER_PATH).toContain('/ud')
|
||||
const [workerPath, rawOptions] = workerState.calls[0]
|
||||
const options = rawOptions as {
|
||||
name: string
|
||||
workerData: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
}
|
||||
expect(workerPath).toBe(
|
||||
join('/apps/orca/app.asar', 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
expect(options.name).toBe('orca-main-thread-hang-watchdog')
|
||||
expect(options.workerData).toMatchObject({
|
||||
parentPid: process.pid,
|
||||
markerPath: join('/ud', 'main-thread-hang.json'),
|
||||
timeoutMs: 45_000,
|
||||
checkIntervalMs: 5_000
|
||||
})
|
||||
expect(worker.unref).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('sends heartbeats on an interval and shutdown+disconnect on stop', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('sends heartbeats on an interval and shutdown on stop', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
vi.advanceTimersByTime(6_000)
|
||||
const heartbeats = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeats = worker.postMessage.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
expect(heartbeats.length).toBe(3)
|
||||
|
||||
handle?.stop()
|
||||
expect(child.send.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
expect(child.disconnect).toHaveBeenCalled()
|
||||
expect(worker.postMessage.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
|
||||
// Why: quit fires will-quit twice; a second stop must not resend or throw.
|
||||
handle?.stop()
|
||||
const shutdowns = child.send.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
const shutdowns = worker.postMessage.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
expect(shutdowns.length).toBe(1)
|
||||
|
||||
vi.advanceTimersByTime(10_000)
|
||||
const heartbeatsAfterStop = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeatsAfterStop = worker.postMessage.mock.calls.filter(
|
||||
([m]) => m.type === 'heartbeat'
|
||||
)
|
||||
expect(heartbeatsAfterStop.length).toBe(3)
|
||||
})
|
||||
|
||||
it('passes test timing overrides to the worker', () => {
|
||||
process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS = '900'
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS = '100'
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const options = workerState.calls[0][1] as {
|
||||
workerData: { timeoutMs: number; checkIntervalMs: number }
|
||||
}
|
||||
expect(options.workerData).toMatchObject({ timeoutMs: 900, checkIntervalMs: 100 })
|
||||
})
|
||||
|
||||
it('registers stop on will-quit', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
expect(appMock.on).toHaveBeenCalledWith('will-quit', expect.any(Function))
|
||||
})
|
||||
|
||||
it('returns null and stays inert when the fork itself fails', () => {
|
||||
forkMock.mockImplementation(() => {
|
||||
throw new Error('spawn failure')
|
||||
})
|
||||
it('stops heartbeat work when the worker exits', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const exitListener = worker.once.mock.calls.find(([event]) => event === 'exit')?.[1]
|
||||
expect(exitListener).toEqual(expect.any(Function))
|
||||
exitListener()
|
||||
vi.advanceTimersByTime(6_000)
|
||||
expect(worker.postMessage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns null and stays inert when worker construction fails', () => {
|
||||
workerState.error = new Error('worker failure')
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
import { fork, type ChildProcess } from 'node:child_process'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import { app } from 'electron'
|
||||
import { resolveHangWatchdogEntryPath } from './hang-watchdog-entry-path'
|
||||
import { hangDetectionMarkerPath } from './hang-detection-marker'
|
||||
|
||||
const HEARTBEAT_INTERVAL_MS = 2_000
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
import {
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS,
|
||||
HANG_WATCHDOG_TIMEOUT_MS,
|
||||
type HangWatchdogWorkerData,
|
||||
type MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
export type MainThreadHangWatchdogHandle = {
|
||||
stop: () => void
|
||||
child: ChildProcess
|
||||
worker: Worker
|
||||
}
|
||||
|
||||
function positiveTiming(value: string | undefined, fallback: number): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
||||
}
|
||||
|
||||
// Why: macOS 26 scene-backed AppKit windows can deadlock the main thread inside
|
||||
// FrontBoardServices with no crash and no event loop left to self-report. A plain-Node sibling
|
||||
// process watches heartbeats and records the stall so the next launch can report it. It observes
|
||||
// only — see main-thread-hang-watchdog-entry.ts for why it does not kill the parent.
|
||||
export function installMainThreadHangWatchdog(options: {
|
||||
userDataPath: string
|
||||
}): MainThreadHangWatchdogHandle | null {
|
||||
@@ -24,58 +30,57 @@ export function installMainThreadHangWatchdog(options: {
|
||||
if (!app.isPackaged && process.env.ORCA_HANG_WATCHDOG_FORCE !== '1') {
|
||||
return null
|
||||
}
|
||||
const entryPath = resolveHangWatchdogEntryPath(app.getAppPath(), app.isPackaged)
|
||||
let child: ChildProcess
|
||||
const workerPath = resolveHangWatchdogWorkerPath(app.getAppPath(), app.isPackaged)
|
||||
const workerData: HangWatchdogWorkerData = {
|
||||
parentPid: process.pid,
|
||||
markerPath: hangDetectionMarkerPath(options.userDataPath),
|
||||
timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS),
|
||||
checkIntervalMs: positiveTiming(
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
)
|
||||
}
|
||||
let worker: Worker
|
||||
try {
|
||||
child = fork(entryPath, [], {
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid),
|
||||
ORCA_HANG_WATCHDOG_MARKER_PATH: hangDetectionMarkerPath(options.userDataPath)
|
||||
}
|
||||
// Why: the worker survives an AppKit main-thread deadlock without another Electron process.
|
||||
worker = new Worker(workerPath, {
|
||||
name: 'orca-main-thread-hang-watchdog',
|
||||
workerData
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[hang-watchdog] failed to fork watchdog process:', error)
|
||||
console.error('[hang-watchdog] failed to start watchdog worker:', error)
|
||||
return null
|
||||
}
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
console.error('[hang-watchdog]', String(chunk).trimEnd())
|
||||
worker.on('error', (error) => {
|
||||
console.error('[hang-watchdog] watchdog worker failed:', error)
|
||||
})
|
||||
// Why: the watchdog is a safety net — if it dies, run without it; a restart loop here must never affect the app.
|
||||
child.on('error', () => {})
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'heartbeat' }, () => {})
|
||||
} catch {
|
||||
// Channel raced closed between the check and the send.
|
||||
}
|
||||
}
|
||||
}, HEARTBEAT_INTERVAL_MS)
|
||||
let stopped = false
|
||||
const postMessage = (message: MainToHangWatchdogWorkerMessage): void => {
|
||||
if (stopped && message.type === 'heartbeat') {
|
||||
return
|
||||
}
|
||||
try {
|
||||
worker.postMessage(message)
|
||||
} catch {
|
||||
// The worker already exited.
|
||||
}
|
||||
}
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
postMessage({ type: 'heartbeat' })
|
||||
}, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS)
|
||||
const stop = (): void => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'shutdown' }, () => {})
|
||||
} catch {
|
||||
// Already disconnecting.
|
||||
}
|
||||
}
|
||||
// Why: disconnect guarantees the child observes parent shutdown and exits instead of misreading quit as a hang.
|
||||
try {
|
||||
child.disconnect()
|
||||
} catch {
|
||||
// Channel already closed.
|
||||
}
|
||||
postMessage({ type: 'shutdown' })
|
||||
}
|
||||
// Why: will-quit fires twice during quit; stop is idempotent.
|
||||
worker.once('exit', () => {
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
})
|
||||
worker.unref()
|
||||
app.on('will-quit', stop)
|
||||
return { stop, child }
|
||||
return { stop, worker }
|
||||
}
|
||||
|
||||
@@ -44,6 +44,12 @@ import { registerMobileHandlers } from './ipc/mobile'
|
||||
import { initTelemetry, shutdownTelemetry, trackAppOpenedOnce, track } from './telemetry/client'
|
||||
import { classifyError } from './telemetry/classify-error'
|
||||
import { runManagedHookInstallers } from './agent-hooks/install-telemetry'
|
||||
import {
|
||||
indexPersistedPaneKeyPtyIds,
|
||||
isLocalExecutionHost,
|
||||
resolveAgentWorkspaceExecutionHostId,
|
||||
sweepRestoredSubagentsWithoutLiveAgent
|
||||
} from './agent-hooks/restored-subagent-liveness-sweep'
|
||||
import {
|
||||
isAgentStatusHooksEnabled,
|
||||
MANAGED_AGENT_HOOK_INSTALLERS,
|
||||
@@ -763,6 +769,46 @@ ipcMain.handle(
|
||||
}
|
||||
)
|
||||
|
||||
/** A PTY that dies while Orca is down never runs the teardown that clears pane
|
||||
* state, so hydrate can rebuild a Claude subagent roster that no later hook can
|
||||
* retire — pinning the pane 'working' and locking its agent out of hibernation
|
||||
* for good. Once provider and hook hydration settle, targeted PTY liveness can
|
||||
* retire only rows whose local owner is proven gone. */
|
||||
async function reapRestoredSubagentsWithoutLiveAgent(): Promise<void> {
|
||||
const currentStore = store
|
||||
if (!currentStore) {
|
||||
return
|
||||
}
|
||||
const provider = getDaemonProvider()
|
||||
if (!provider) {
|
||||
return
|
||||
}
|
||||
const persistedPtyIdByPaneKey = indexPersistedPaneKeyPtyIds(
|
||||
currentStore.getWorkspaceSession().terminalLayoutsByTabId ?? {}
|
||||
)
|
||||
await sweepRestoredSubagentsWithoutLiveAgent({
|
||||
probeLiveLocalPty: (ptyId) => provider.probePtyLiveness(ptyId),
|
||||
isLocalExecutionHost: (worktreeId) =>
|
||||
isLocalExecutionHost(
|
||||
resolveAgentWorkspaceExecutionHostId(worktreeId, {
|
||||
getRepo: (repoId) => currentStore.getRepo(repoId),
|
||||
getWorktreeMeta: (resolvedWorktreeId) => currentStore.getWorktreeMeta(resolvedWorktreeId),
|
||||
getFolderWorkspace: (folderWorkspaceId) =>
|
||||
currentStore.getFolderWorkspace(folderWorkspaceId),
|
||||
getProjectGroups: () => currentStore.getProjectGroups()
|
||||
})
|
||||
),
|
||||
getBoundPtyIdForPaneKey: getPtyIdForPaneKey,
|
||||
getPersistedPtyIdForPaneKey: (paneKey) => persistedPtyIdByPaneKey.get(paneKey),
|
||||
reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) =>
|
||||
agentHookServer.reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalHost,
|
||||
isLocalPaneAgentLive,
|
||||
isLocalPaneLivenessEvidenceCurrent
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function startTerminalRuntimeStartupServices(): Promise<void> {
|
||||
logStartupMilestone('first-window-startup-services-start')
|
||||
const startupServices = startFirstWindowStartupServices({
|
||||
@@ -811,6 +857,9 @@ function startTerminalRuntimeStartupServices(): Promise<void> {
|
||||
})
|
||||
void localPtyStartupReady.then(() => {
|
||||
logStartupMilestone('local-pty-startup-ready')
|
||||
void reapRestoredSubagentsWithoutLiveAgent().catch((error) => {
|
||||
console.warn('[agent-hooks] restored-subagent liveness probe failed:', error)
|
||||
})
|
||||
})
|
||||
return firstWindowStartupServicesReady
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ export type PendingPtyData = {
|
||||
droppedOutput?: true
|
||||
droppedMode2031Data?: string
|
||||
droppedMode2031ScanState?: Mode2031ReplyScanState
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
projectionAdmissionsTransferred?: true
|
||||
}
|
||||
|
||||
export type PtyPendingDataDrainDisposition = 'active' | 'background' | 'blocked'
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
appendPendingProjectionAdmission,
|
||||
compactPendingProjectionAdmissions,
|
||||
PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS,
|
||||
propagatePendingProjectionRemainder
|
||||
} from './pty-pending-projection-admissions'
|
||||
|
||||
describe('pending PTY projection admissions', () => {
|
||||
it('retains the exact cap then transfers the whole ordered run on overflow', () => {
|
||||
const transfer = vi.fn()
|
||||
let state = compactPendingProjectionAdmissions()
|
||||
|
||||
for (let index = 0; index < PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS; index++) {
|
||||
state = appendPendingProjectionAdmission(state, `projection-${index}`, {
|
||||
isPending: () => true,
|
||||
transfer
|
||||
})
|
||||
}
|
||||
|
||||
expect(state.projectionAdmissionIds).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS)
|
||||
expect(transfer).not.toHaveBeenCalled()
|
||||
|
||||
state = appendPendingProjectionAdmission(state, 'projection-overflow', {
|
||||
isPending: () => true,
|
||||
transfer
|
||||
})
|
||||
|
||||
expect(state).toEqual({ projectionAdmissionsTransferred: true })
|
||||
expect(transfer).toHaveBeenCalledOnce()
|
||||
expect(transfer.mock.calls[0]?.[0]).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS + 1)
|
||||
})
|
||||
|
||||
it('compacts terminal prefixes and transfers later admissions until the remainder drains', () => {
|
||||
const transfer = vi.fn()
|
||||
const compacted = propagatePendingProjectionRemainder(
|
||||
{
|
||||
projectionAdmissionIds: ['projection-published', 'projection-partial', 'projection-tail']
|
||||
},
|
||||
{ sent: true, projectionsTransferred: false },
|
||||
{ isPending: (id) => id !== 'projection-published', transfer }
|
||||
)
|
||||
|
||||
expect(compacted).toEqual({
|
||||
projectionAdmissionIds: ['projection-partial', 'projection-tail']
|
||||
})
|
||||
|
||||
const transferred = appendPendingProjectionAdmission(
|
||||
{ projectionAdmissionsTransferred: true },
|
||||
'projection-after-transfer',
|
||||
{ isPending: () => true, transfer }
|
||||
)
|
||||
expect(transferred).toEqual({ projectionAdmissionsTransferred: true })
|
||||
expect(transfer).toHaveBeenCalledWith(['projection-after-transfer'], 'pending-projection-cap')
|
||||
|
||||
expect(
|
||||
propagatePendingProjectionRemainder(
|
||||
compacted,
|
||||
{ sent: true, projectionsTransferred: true },
|
||||
{ isPending: () => true, transfer }
|
||||
)
|
||||
).toEqual({ projectionAdmissionsTransferred: true })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,57 @@
|
||||
export const PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS = 1024
|
||||
|
||||
export type PendingProjectionAdmissions = Readonly<{
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
projectionAdmissionsTransferred?: true
|
||||
}>
|
||||
|
||||
type PendingProjectionAdmissionOptions = Readonly<{
|
||||
isPending: (id: string) => boolean
|
||||
transfer: (ids: readonly string[], reason: string) => void
|
||||
}>
|
||||
|
||||
const DEFAULT_OPTIONS: PendingProjectionAdmissionOptions = {
|
||||
isPending: () => true,
|
||||
transfer: () => {}
|
||||
}
|
||||
|
||||
export function compactPendingProjectionAdmissions(
|
||||
state: PendingProjectionAdmissions = {},
|
||||
options: PendingProjectionAdmissionOptions = DEFAULT_OPTIONS
|
||||
): PendingProjectionAdmissions {
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
const ids = Array.from(
|
||||
new Set((state.projectionAdmissionIds ?? []).filter((id) => options.isPending(id)))
|
||||
)
|
||||
return ids.length > 0 ? { projectionAdmissionIds: ids } : {}
|
||||
}
|
||||
|
||||
export function appendPendingProjectionAdmission(
|
||||
state: PendingProjectionAdmissions,
|
||||
id: string,
|
||||
options: PendingProjectionAdmissionOptions
|
||||
): PendingProjectionAdmissions {
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
options.transfer([id], 'pending-projection-cap')
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
const compacted = compactPendingProjectionAdmissions(state, options)
|
||||
const ids = [...(compacted.projectionAdmissionIds ?? []), id]
|
||||
if (ids.length <= PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS) {
|
||||
return { projectionAdmissionIds: ids }
|
||||
}
|
||||
options.transfer(ids, 'pending-projection-cap')
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
|
||||
export function propagatePendingProjectionRemainder(
|
||||
state: PendingProjectionAdmissions,
|
||||
delivery: Readonly<{ sent: boolean; projectionsTransferred: boolean }>,
|
||||
options: PendingProjectionAdmissionOptions
|
||||
): PendingProjectionAdmissions {
|
||||
return delivery.sent && !delivery.projectionsTransferred
|
||||
? compactPendingProjectionAdmissions(state, options)
|
||||
: { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
+198
-1
@@ -247,6 +247,11 @@ import { resolveWindowsShellLaunchArgs } from '../providers/windows-shell-args'
|
||||
import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl'
|
||||
import { wslHookRelayManager } from '../agent-hooks/wsl-hook-relay-manager'
|
||||
import { acquireWatcherRemovalGate } from './watcher-removal-gate'
|
||||
import {
|
||||
acceptSshPtyOutputData,
|
||||
acceptSshPtyOutputExit,
|
||||
closeSshPtyOutputGeneration
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
|
||||
// Why: Windows resolves a bare PowerShell name to an absolute exe before ConPTY, else CreateProcessW fails with error 5 (PR #6537 / #5161).
|
||||
const RESOLVED_WINDOWS_POWERSHELL = 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe'
|
||||
@@ -449,7 +454,8 @@ describe('registerPtyHandlers', () => {
|
||||
'ssh-reattach-1',
|
||||
'ssh-reattach-fail',
|
||||
'ssh-reattach-ok',
|
||||
'ssh-runtime-env'
|
||||
'ssh-runtime-env',
|
||||
'ssh-generation-replacement'
|
||||
]) {
|
||||
unregisterSshPtyProvider(leakedConnectionId)
|
||||
}
|
||||
@@ -11556,6 +11562,197 @@ describe('registerPtyHandlers', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps negotiated source-credit overflow off the legacy PTY-global pause path', async () => {
|
||||
vi.useFakeTimers()
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
try {
|
||||
const provider = installObservableDaemonTestProvider()
|
||||
let modelSequence = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => modelSequence),
|
||||
onPtyData: vi.fn(
|
||||
(_id: string, data: string, _at: number, rawLength = data.length) =>
|
||||
(modelSequence += rawLength)
|
||||
),
|
||||
acceptPtyDataBounded: vi.fn(
|
||||
(_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
modelSequence += rawLength
|
||||
return { sequence: modelSequence, completion: Promise.resolve() }
|
||||
}
|
||||
)
|
||||
}
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
mainWindow.webContents.send.mockClear()
|
||||
|
||||
const sourceChunk = 's'.repeat(128 * 1024)
|
||||
for (let index = 0; index < 17; index++) {
|
||||
const sourceStartSu = index * sourceChunk.length
|
||||
await acceptSshPtyOutputData({
|
||||
id: 'source-credit-pty',
|
||||
data: sourceChunk,
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'source-incarnation',
|
||||
rawLength: sourceChunk.length,
|
||||
transformed: false,
|
||||
source: {
|
||||
relayPtyId: 'relay-source-pty',
|
||||
spanId: `source-token:${sourceStartSu}:${sourceStartSu + sourceChunk.length}`,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'source-token',
|
||||
sourceStartSu,
|
||||
sourceEndSu: sourceStartSu + sourceChunk.length
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
expect(getPtyRendererDeliveryDebugSnapshot()).toMatchObject({
|
||||
pendingPtyCount: 1,
|
||||
pendingChars: 0
|
||||
})
|
||||
expect(provider.pauseProducer).not.toHaveBeenCalledWith('source-credit-pty')
|
||||
expect(provider.resumeProducer).not.toHaveBeenCalledWith('source-credit-pty')
|
||||
|
||||
provider.emitData('legacy-pty', 'l'.repeat(320 * 1024))
|
||||
expect(provider.pauseProducer).toHaveBeenCalledTimes(1)
|
||||
expect(provider.pauseProducer).toHaveBeenCalledWith('legacy-pty')
|
||||
expect(provider.pauseProducer).not.toHaveBeenCalledWith('unrelated-pty')
|
||||
|
||||
vi.runAllTimers()
|
||||
expect(provider.resumeProducer).toHaveBeenCalledTimes(1)
|
||||
expect(provider.resumeProducer).toHaveBeenCalledWith('legacy-pty')
|
||||
} finally {
|
||||
errorSpy.mockRestore()
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('pauses and resumes the exact SSH provider generation across reconnect replacement', async () => {
|
||||
vi.useFakeTimers()
|
||||
const completion = makeDeferred()
|
||||
let sequence = 0
|
||||
let captures = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => sequence),
|
||||
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
sequence += rawLength
|
||||
captures++
|
||||
return {
|
||||
sequence,
|
||||
completion: captures === 1 ? completion.promise : Promise.resolve()
|
||||
}
|
||||
})
|
||||
}
|
||||
const original = {
|
||||
providerGeneration: 41,
|
||||
hasPtyDeliveryPauseAdapter: () => true,
|
||||
pauseProducer: vi.fn(),
|
||||
resumeProducer: vi.fn()
|
||||
}
|
||||
const replacement = {
|
||||
providerGeneration: 42,
|
||||
hasPtyDeliveryPauseAdapter: () => true,
|
||||
pauseProducer: vi.fn(),
|
||||
resumeProducer: vi.fn()
|
||||
}
|
||||
const id = 'ssh:ssh-generation-replacement@@relay-pty'
|
||||
const receipts: Promise<unknown>[] = []
|
||||
|
||||
try {
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
registerSshPtyProvider('ssh-generation-replacement', original as never)
|
||||
const running = acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'a'.repeat(256 * 1024),
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'incarnation-41',
|
||||
rawLength: 256 * 1024,
|
||||
transformed: false
|
||||
})
|
||||
receipts.push(running)
|
||||
registerSshPtyProvider('ssh-generation-replacement', replacement as never)
|
||||
const pressured = acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'b',
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'incarnation-41',
|
||||
rawLength: 1,
|
||||
transformed: false
|
||||
})
|
||||
receipts.push(pressured)
|
||||
|
||||
expect(original.pauseProducer).toHaveBeenCalledWith(id)
|
||||
expect(replacement.pauseProducer).not.toHaveBeenCalled()
|
||||
|
||||
completion.resolve()
|
||||
await Promise.all([running, pressured])
|
||||
expect(original.resumeProducer).toHaveBeenCalledWith(id)
|
||||
expect(replacement.resumeProducer).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
completion.resolve()
|
||||
await Promise.allSettled(receipts)
|
||||
closeSshPtyOutputGeneration(41, 'test-cleanup')
|
||||
unregisterSshPtyProvider('ssh-generation-replacement')
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects local data while an SSH renderer exit waits for projection settlement', async () => {
|
||||
const provider = installObservableDaemonTestProvider()
|
||||
let sequence = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => sequence),
|
||||
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
sequence += rawLength
|
||||
return { sequence, completion: Promise.resolve() }
|
||||
}),
|
||||
onPtyData: vi.fn(),
|
||||
onPtyExit: vi.fn()
|
||||
}
|
||||
const id = 'ssh:exit-data-race@@relay-pty'
|
||||
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
mainWindow.webContents.send.mockClear()
|
||||
await acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'before-exit',
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51',
|
||||
rawLength: 'before-exit'.length,
|
||||
transformed: false
|
||||
})
|
||||
const exit = acceptSshPtyOutputExit({
|
||||
id,
|
||||
code: 0,
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51'
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
provider.emitData(id, 'must-not-follow-exit')
|
||||
expect(mainWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', {
|
||||
id,
|
||||
data: 'must-not-follow-exit'
|
||||
})
|
||||
|
||||
getPtyAckDataListener()(null, { id, processedChars: 'before-exit'.length })
|
||||
await exit
|
||||
expect(mainWindow.webContents.send.mock.calls.at(-1)).toEqual([
|
||||
'pty:exit',
|
||||
{
|
||||
id,
|
||||
code: 0,
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51'
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a paused producer when the PTY exits before draining', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
|
||||
+471
-168
@@ -158,6 +158,19 @@ import {
|
||||
unmarkHiddenRendererPty
|
||||
} from './pty-hidden-delivery-gate'
|
||||
import { PtyPendingDataDrainQueue, type PendingPtyData } from './pty-pending-data-drain-queue'
|
||||
import {
|
||||
appendPendingProjectionAdmission,
|
||||
compactPendingProjectionAdmissions,
|
||||
propagatePendingProjectionRemainder,
|
||||
type PendingProjectionAdmissions
|
||||
} from './pty-pending-projection-admissions'
|
||||
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import {
|
||||
cancelSshPtySourceDelivery,
|
||||
installSshPtyOutputIntake,
|
||||
publishSshPtySourceAck
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import {
|
||||
clearNativeWindowsConptyPty,
|
||||
isNativeWindowsLocalPtySpawn,
|
||||
@@ -202,6 +215,7 @@ type FreshLocalFallbackProvider = IPtyProvider & {
|
||||
routesFreshSpawnsToLocalProvider?: true
|
||||
}
|
||||
const sshProviders = new Map<string, IPtyProvider>()
|
||||
const sshProvidersByGeneration = new Map<number, IPtyProvider>()
|
||||
|
||||
type RegisteredPtyProvider = {
|
||||
provider: IPtyProvider
|
||||
@@ -1296,10 +1310,19 @@ function beginPtySpawnForWorktree(
|
||||
/** Register an SSH PTY provider for a connection. */
|
||||
export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void {
|
||||
sshProviders.set(connectionId, provider)
|
||||
const generation = (provider as { providerGeneration?: number }).providerGeneration
|
||||
if (Number.isSafeInteger(generation) && generation! > 0) {
|
||||
sshProvidersByGeneration.set(generation!, provider)
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove an SSH PTY provider when a connection is closed. */
|
||||
export function unregisterSshPtyProvider(connectionId: string): void {
|
||||
const provider = sshProviders.get(connectionId)
|
||||
const generation = (provider as { providerGeneration?: number } | undefined)?.providerGeneration
|
||||
if (generation !== undefined && sshProvidersByGeneration.get(generation) === provider) {
|
||||
sshProvidersByGeneration.delete(generation)
|
||||
}
|
||||
sshProviders.delete(connectionId)
|
||||
}
|
||||
|
||||
@@ -1467,6 +1490,7 @@ let rendererDidStartLoadingHandler: (() => void) | null = null
|
||||
|
||||
// Why: Restart daemon must re-bind provider→renderer listeners after replaceDaemonProvider swaps localProvider, else subscribers stay bound to the disposed adapter and new PTY data silently drops.
|
||||
let rebindProviderListeners: (() => void) | null = null
|
||||
let sshOutputIntakeCleanup: (() => void) | null = null
|
||||
|
||||
export function rebindLocalProviderListeners(): void {
|
||||
rebindProviderListeners?.()
|
||||
@@ -1840,8 +1864,10 @@ export function registerPtyHandlers(
|
||||
},
|
||||
() => isHiddenPtyDeliveryGateEnabled(getSettings?.())
|
||||
)
|
||||
let sshOutputIntake: SshPtyOutputIntake | null = null
|
||||
// Why: resuming a paused producer during exit can synchronously emit; those bytes must not queue behind pty:exit.
|
||||
const rendererExitingPtyIds = new Set<string>()
|
||||
const rendererCreditBeforeExitByPty = new Map<string, boolean>()
|
||||
const rendererDeliveryRestoreNeededPtys = new Set<string>()
|
||||
|
||||
function transitionHiddenRendererPtyDeliveryState(id: string, hidden: boolean) {
|
||||
@@ -1927,11 +1953,18 @@ export function registerPtyHandlers(
|
||||
pauseProducer: (id) => tryGetProviderForPty(id)?.pauseProducer?.(id),
|
||||
resumeProducer: (id) => tryGetProviderForPty(id)?.resumeProducer?.(id)
|
||||
})
|
||||
const sourceCreditPendingPtys = new Set<string>()
|
||||
|
||||
function updateProducerFlowControl(id: string): void {
|
||||
if (!PRODUCER_FLOW_CONTROL_ENABLED) {
|
||||
return
|
||||
}
|
||||
if (sourceCreditPendingPtys.has(id)) {
|
||||
if (pendingData.get(id)) {
|
||||
return
|
||||
}
|
||||
sourceCreditPendingPtys.delete(id)
|
||||
}
|
||||
producerFlowControl.update(id, pendingData.get(id)?.data.length ?? 0)
|
||||
}
|
||||
|
||||
@@ -2000,7 +2033,16 @@ export function registerPtyHandlers(
|
||||
}
|
||||
|
||||
function clearPendingPtyData(): void {
|
||||
for (const pending of pendingData.values()) {
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(
|
||||
pending.projectionAdmissionIds,
|
||||
'renderer-lifecycle-reset'
|
||||
)
|
||||
}
|
||||
}
|
||||
pendingData.clear()
|
||||
sourceCreditPendingPtys.clear()
|
||||
}
|
||||
|
||||
function readCurrentPtyRendererDeliveryDebugSnapshot(): PtyRendererDeliveryDebugSnapshot {
|
||||
@@ -2169,6 +2211,9 @@ export function registerPtyHandlers(
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
deliveryResyncUnansweredWarnLogged = false
|
||||
for (const id of rendererDeliveryAccountingByPty.keys()) {
|
||||
sshOutputIntake?.transferPtyProjections(id, 'renderer-lifecycle-reset')
|
||||
}
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearPendingPtyData()
|
||||
@@ -2265,6 +2310,9 @@ export function registerPtyHandlers(
|
||||
accounting.lastAckAtMs = Date.now()
|
||||
}
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - acknowledged)
|
||||
if (acknowledged > 0) {
|
||||
sshOutputIntake?.settleProjectionPrefix(id, acknowledged)
|
||||
}
|
||||
return acknowledged
|
||||
}
|
||||
|
||||
@@ -2336,6 +2384,12 @@ export function registerPtyHandlers(
|
||||
// Why drop pending: everything at/before markerSeq comes from the snapshot, so flushing pre-marker bytes would double-paint the restore.
|
||||
const pending = pendingData.get(id)
|
||||
if (pending) {
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(
|
||||
pending.projectionAdmissionIds,
|
||||
'renderer-delivery-writeoff'
|
||||
)
|
||||
}
|
||||
pendingDroppedChars += pending.data.length
|
||||
deletePendingPtyData(id)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
@@ -2365,7 +2419,11 @@ export function registerPtyHandlers(
|
||||
return writtenOff
|
||||
}
|
||||
|
||||
function sendPtyDataToRenderer(id: string, payload: PtyDataPayload): boolean {
|
||||
function sendPtyDataToRenderer(
|
||||
id: string,
|
||||
payload: PtyDataPayload,
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
): { sent: boolean; projectionsTransferred: boolean } {
|
||||
const charCount = getPtyPayloadCharCount(payload)
|
||||
const accounting = rendererDeliveryAccountingByPty.get(id)
|
||||
const hadAccounting = accounting !== undefined
|
||||
@@ -2400,12 +2458,28 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
rendererDeliveryRestoreNeededPtys.add(id)
|
||||
if (projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'renderer-send-failed')
|
||||
}
|
||||
mainDeliveryBreadcrumbs.record('pty-data-send-failed', {
|
||||
id: redactPtyIdForDiagnostics(id),
|
||||
chars: charCount
|
||||
})
|
||||
console.error('[pty] renderer data send failed; payload will not be retried', error)
|
||||
return false
|
||||
return { sent: false, projectionsTransferred: projectionAdmissionIds !== undefined }
|
||||
}
|
||||
let projectionsTransferred = false
|
||||
if (projectionAdmissionIds) {
|
||||
try {
|
||||
sshOutputIntake?.publishProjectionPrefix(
|
||||
projectionAdmissionIds,
|
||||
payload.data.length,
|
||||
charCount
|
||||
)
|
||||
} catch {
|
||||
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'projection-publish-failed')
|
||||
projectionsTransferred = true
|
||||
}
|
||||
}
|
||||
if (rendererDeliveryRestoreNeededPtys.has(id)) {
|
||||
try {
|
||||
@@ -2418,7 +2492,7 @@ export function registerPtyHandlers(
|
||||
)
|
||||
}
|
||||
}
|
||||
return true
|
||||
return { sent: true, projectionsTransferred }
|
||||
}
|
||||
|
||||
function rendererPtyIsKnownHidden(id: string): boolean {
|
||||
@@ -2521,6 +2595,9 @@ export function registerPtyHandlers(
|
||||
pendingOverflowMarkedPtys.add(id)
|
||||
}
|
||||
pendingDroppedChars += pending.data.length
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'pending-cap')
|
||||
}
|
||||
const mode2031 = scanDroppedMode2031Data(pending.data, INITIAL_MODE_2031_REPLY_SCAN_STATE)
|
||||
// Why no trimmed content tail: a mid-stream gap would corrupt the pane; the droppedOutput sentinel repaints from the snapshot and realigns by sequence (only query bytes ride along).
|
||||
return {
|
||||
@@ -2531,6 +2608,39 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
|
||||
function updatePendingProjectionAdmissions(
|
||||
pending: PendingPtyData,
|
||||
state: PendingProjectionAdmissions
|
||||
): void {
|
||||
delete pending.projectionAdmissionIds
|
||||
delete pending.projectionAdmissionsTransferred
|
||||
if (state.projectionAdmissionIds) {
|
||||
pending.projectionAdmissionIds = state.projectionAdmissionIds
|
||||
}
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
pending.projectionAdmissionsTransferred = true
|
||||
}
|
||||
}
|
||||
|
||||
function compactPendingProjectionState(
|
||||
pending: PendingProjectionAdmissions,
|
||||
projectionSemanticsId?: string
|
||||
): PendingProjectionAdmissions {
|
||||
const options = pendingProjectionAdmissionOptions()
|
||||
const compacted = compactPendingProjectionAdmissions(pending, options)
|
||||
return projectionSemanticsId
|
||||
? appendPendingProjectionAdmission(compacted, projectionSemanticsId, options)
|
||||
: compacted
|
||||
}
|
||||
|
||||
function pendingProjectionAdmissionOptions() {
|
||||
return {
|
||||
isPending: (id: string) => sshOutputIntake?.hasUnpublishedProjection(id) ?? false,
|
||||
transfer: (ids: readonly string[], reason: string) =>
|
||||
sshOutputIntake?.transferProjections(ids, reason)
|
||||
}
|
||||
}
|
||||
|
||||
function appendPendingPtyData(
|
||||
id: string,
|
||||
existing: PendingPtyData | undefined,
|
||||
@@ -2539,10 +2649,14 @@ export function registerPtyHandlers(
|
||||
preservesSeq: boolean,
|
||||
containsBackgroundOutput: boolean,
|
||||
rawLength = data.length,
|
||||
transformed = false
|
||||
transformed = false,
|
||||
projectionSemanticsId?: string
|
||||
): PendingPtyData {
|
||||
// Why stay dropped at O(1): once over the cap the restore sentinel supersedes interim bytes; queries still get carved out (bounded) so replies survive the whole episode.
|
||||
if (existing?.droppedOutput === true) {
|
||||
if (projectionSemanticsId) {
|
||||
sshOutputIntake?.transferProjections([projectionSemanticsId], 'pending-cap')
|
||||
}
|
||||
const mode2031 = scanDroppedMode2031Data(
|
||||
data,
|
||||
existing.droppedMode2031ScanState ?? INITIAL_MODE_2031_REPLY_SCAN_STATE
|
||||
@@ -2559,16 +2673,19 @@ export function registerPtyHandlers(
|
||||
droppedMode2031ScanState: mode2031.state
|
||||
}
|
||||
}
|
||||
const projectionState = compactPendingProjectionState(existing ?? {}, projectionSemanticsId)
|
||||
const nextContainsBackgroundOutput =
|
||||
existing?.containsBackgroundOutput === true || containsBackgroundOutput
|
||||
if (!existing) {
|
||||
return dropOversizedPendingPtyData(id, {
|
||||
const pending: PendingPtyData = {
|
||||
data,
|
||||
...(typeof startSeq === 'number' ? { startSeq } : {}),
|
||||
...(rawLength !== data.length ? { rawLength } : {}),
|
||||
...(transformed ? { transformed: true } : {}),
|
||||
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
|
||||
})
|
||||
}
|
||||
updatePendingProjectionAdmissions(pending, projectionState)
|
||||
return dropOversizedPendingPtyData(id, pending)
|
||||
}
|
||||
const existingRawLength = existing.rawLength ?? existing.data.length
|
||||
const next: PendingPtyData = {
|
||||
@@ -2578,6 +2695,7 @@ export function registerPtyHandlers(
|
||||
: {}),
|
||||
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
|
||||
}
|
||||
updatePendingProjectionAdmissions(next, projectionState)
|
||||
if (typeof existing.startSeq === 'number') {
|
||||
next.startSeq = existing.startSeq
|
||||
}
|
||||
@@ -2661,6 +2779,9 @@ export function registerPtyHandlers(
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
updateProducerFlowControl(id)
|
||||
const drop = recordHiddenRendererPtyDataDrop(id, pending.data.length)
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
|
||||
}
|
||||
warnIfDroppingHiddenBytesForVisiblePty(id, pending.data.length)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(id, 'hidden-drop', runtime?.getPtyOutputSequence(id))
|
||||
@@ -2676,11 +2797,15 @@ export function registerPtyHandlers(
|
||||
updateProducerFlowControl(id)
|
||||
// Why droppedOutput sentinel: pending-cap drop means the pane must repaint from the snapshot, not continue a gapped stream (data = carved query bytes only).
|
||||
if (
|
||||
!sendPtyDataToRenderer(id, {
|
||||
!sendPtyDataToRenderer(
|
||||
id,
|
||||
data: pending.data + getDroppedMode2031RendererData(pending),
|
||||
droppedOutput: true
|
||||
})
|
||||
{
|
||||
id,
|
||||
data: pending.data + getDroppedMode2031RendererData(pending),
|
||||
droppedOutput: true
|
||||
},
|
||||
pending.projectionAdmissionIds
|
||||
).sent
|
||||
) {
|
||||
sendFailed = true
|
||||
break
|
||||
@@ -2692,33 +2817,50 @@ export function registerPtyHandlers(
|
||||
const indivisible = pending.transformed === true
|
||||
const chunk = indivisible ? data : data.slice(0, PTY_BATCH_FLUSH_CHUNK_CHARS)
|
||||
const remaining = indivisible ? '' : data.slice(PTY_BATCH_FLUSH_CHUNK_CHARS)
|
||||
let nextPending: PendingPtyData | undefined
|
||||
if (remaining) {
|
||||
const nextPending: PendingPtyData = { data: remaining }
|
||||
nextPending = { data: remaining }
|
||||
if (typeof pending.startSeq === 'number') {
|
||||
nextPending.startSeq = pending.startSeq + chunk.length
|
||||
}
|
||||
if (pending.containsBackgroundOutput === true) {
|
||||
nextPending.containsBackgroundOutput = true
|
||||
}
|
||||
if (pending.projectionAdmissionIds) {
|
||||
nextPending.projectionAdmissionIds = pending.projectionAdmissionIds
|
||||
}
|
||||
if (pending.projectionAdmissionsTransferred) {
|
||||
nextPending.projectionAdmissionsTransferred = true
|
||||
}
|
||||
pendingData.replaceWithRemainder(selection, nextPending)
|
||||
} else {
|
||||
pendingData.remove(selection)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
}
|
||||
updateProducerFlowControl(id)
|
||||
if (
|
||||
!sendPtyDataToRenderer(
|
||||
const delivery = sendPtyDataToRenderer(
|
||||
id,
|
||||
makePtyDataPayload(
|
||||
id,
|
||||
makePtyDataPayload(
|
||||
id,
|
||||
chunk,
|
||||
pending.startSeq,
|
||||
pending.containsBackgroundOutput,
|
||||
pending.rawLength,
|
||||
pending.transformed
|
||||
chunk,
|
||||
pending.startSeq,
|
||||
pending.containsBackgroundOutput,
|
||||
pending.rawLength,
|
||||
pending.transformed
|
||||
),
|
||||
pending.projectionAdmissionIds
|
||||
)
|
||||
if (nextPending) {
|
||||
updatePendingProjectionAdmissions(
|
||||
nextPending,
|
||||
propagatePendingProjectionRemainder(
|
||||
nextPending,
|
||||
delivery,
|
||||
pendingProjectionAdmissionOptions()
|
||||
)
|
||||
)
|
||||
) {
|
||||
}
|
||||
if (!delivery.sent) {
|
||||
sendFailed = true
|
||||
break
|
||||
}
|
||||
@@ -2781,27 +2923,45 @@ export function registerPtyHandlers(
|
||||
return true
|
||||
}
|
||||
|
||||
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
|
||||
function preparePtyExitForRenderer(payload: { id: string; code: number }): (() => void) | null {
|
||||
if (mainWindow.isDestroyed()) {
|
||||
return
|
||||
sshOutputIntake?.transferPtyProjections(payload.id, 'renderer-destroyed')
|
||||
return () => {}
|
||||
}
|
||||
if (rendererExitingPtyIds.has(payload.id)) {
|
||||
return
|
||||
return null
|
||||
}
|
||||
rendererExitingPtyIds.add(payload.id)
|
||||
let released = false
|
||||
const release = (): void => {
|
||||
if (released) {
|
||||
return
|
||||
}
|
||||
released = true
|
||||
rendererExitingPtyIds.delete(payload.id)
|
||||
}
|
||||
try {
|
||||
const hadReleasableRendererCredit = getRendererInFlightCharsForPty(payload.id) > 0
|
||||
if (!rendererCreditBeforeExitByPty.has(payload.id)) {
|
||||
rendererCreditBeforeExitByPty.set(
|
||||
payload.id,
|
||||
getRendererInFlightCharsForPty(payload.id) > 0
|
||||
)
|
||||
}
|
||||
// Why flush before exit: the renderer tears down the terminal on pty:exit, so any batched output not yet flushed would be silently lost.
|
||||
const remaining = pendingData.delete(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (remaining) {
|
||||
if (remaining.droppedOutput === true) {
|
||||
// Sentinel entry: only salvaged query bytes remain; keep the flag so the renderer knows the span was dropped.
|
||||
sendPtyDataToRenderer(payload.id, {
|
||||
id: payload.id,
|
||||
data: remaining.data,
|
||||
droppedOutput: true
|
||||
})
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
{
|
||||
id: payload.id,
|
||||
data: remaining.data,
|
||||
droppedOutput: true
|
||||
},
|
||||
remaining.projectionAdmissionIds
|
||||
)
|
||||
} else {
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
@@ -2812,35 +2972,63 @@ export function registerPtyHandlers(
|
||||
remaining.containsBackgroundOutput,
|
||||
remaining.rawLength,
|
||||
remaining.transformed
|
||||
)
|
||||
),
|
||||
remaining.projectionAdmissionIds
|
||||
)
|
||||
}
|
||||
}
|
||||
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
|
||||
producerFlowControl.release(payload.id)
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
rendererDeliveryRestoreNeededPtys.delete(payload.id)
|
||||
lastInputAtByPty.delete(payload.id)
|
||||
interactiveOutputCharsByPty.delete(payload.id)
|
||||
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
|
||||
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
|
||||
rendererDeliveryAccountingByPty.delete(payload.id)
|
||||
if (hadReleasableRendererCredit) {
|
||||
if (pendingDataFlushActive) {
|
||||
// Why: let the open round coalesce this wake into its one post-round continuation.
|
||||
const reactivatedBlocked = pendingData.reactivateBlocked()
|
||||
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
|
||||
} else {
|
||||
schedulePendingDataAfterCreditReport(true)
|
||||
}
|
||||
return release
|
||||
} catch (error) {
|
||||
release()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function finalizePtyExitForRenderer(payload: { id: string; code: number }): void {
|
||||
if (mainWindow.isDestroyed()) {
|
||||
rendererCreditBeforeExitByPty.delete(payload.id)
|
||||
return
|
||||
}
|
||||
const hadReleasableRendererCredit =
|
||||
rendererCreditBeforeExitByPty.get(payload.id) ??
|
||||
getRendererInFlightCharsForPty(payload.id) > 0
|
||||
rendererCreditBeforeExitByPty.delete(payload.id)
|
||||
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
|
||||
producerFlowControl.release(payload.id)
|
||||
sourceCreditPendingPtys.delete(payload.id)
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
rendererDeliveryRestoreNeededPtys.delete(payload.id)
|
||||
lastInputAtByPty.delete(payload.id)
|
||||
interactiveOutputCharsByPty.delete(payload.id)
|
||||
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
|
||||
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
|
||||
rendererDeliveryAccountingByPty.delete(payload.id)
|
||||
if (hadReleasableRendererCredit) {
|
||||
if (pendingDataFlushActive) {
|
||||
// Why: let the open round coalesce this wake into its one post-round continuation.
|
||||
const reactivatedBlocked = pendingData.reactivateBlocked()
|
||||
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
|
||||
} else {
|
||||
schedulePendingDataAfterCreditReport(true)
|
||||
}
|
||||
mainWindow.webContents.send('pty:exit', {
|
||||
...payload,
|
||||
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
|
||||
})
|
||||
}
|
||||
mainWindow.webContents.send('pty:exit', {
|
||||
...payload,
|
||||
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
|
||||
})
|
||||
}
|
||||
|
||||
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
|
||||
const release = preparePtyExitForRenderer(payload)
|
||||
if (!release) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
sshOutputIntake?.transferPtyProjections(payload.id, 'legacy-pty-exit')
|
||||
finalizePtyExitForRenderer(payload)
|
||||
} finally {
|
||||
rendererExitingPtyIds.delete(payload.id)
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2850,6 +3038,231 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
|
||||
function acceptPtyDataForRenderer(
|
||||
payload: {
|
||||
id: string
|
||||
data: string
|
||||
sequenceChars?: number
|
||||
transformed?: boolean
|
||||
},
|
||||
outputSeq: number | undefined,
|
||||
projection?: LegacySshProjectionSemantics
|
||||
): void {
|
||||
const rawLength = payload.sequenceChars ?? payload.data.length
|
||||
const preservesSeq = !payload.transformed && rawLength === payload.data.length
|
||||
const startSeq = typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
|
||||
const projectionId = projection?.identity.projectionSemanticsId
|
||||
if (mainWindow.isDestroyed()) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'renderer-destroyed')
|
||||
}
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer)
|
||||
flushTimer = null
|
||||
}
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
clearPendingPtyData()
|
||||
pendingOverflowMarkedPtys.clear()
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearDispatcherReadyWatchdog()
|
||||
return
|
||||
}
|
||||
if (rendererExitingPtyIds.has(payload.id)) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'pty-exiting')
|
||||
}
|
||||
return
|
||||
}
|
||||
if (shouldDropHiddenRendererPtyData(payload.id, getSettings?.())) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'hidden-drop')
|
||||
}
|
||||
const droppedChars = projection ? rawLength : payload.data.length
|
||||
const drop = recordHiddenRendererPtyDataDrop(payload.id, droppedChars)
|
||||
warnIfDroppingHiddenBytesForVisiblePty(payload.id, droppedChars)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (payload.data.length === 0 && !payload.transformed) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'empty-projection')
|
||||
}
|
||||
return
|
||||
}
|
||||
const containsBackgroundOutput =
|
||||
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
|
||||
if (containsBackgroundOutput) {
|
||||
markHiddenRendererResizeOutputDelivered(payload.id)
|
||||
}
|
||||
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
|
||||
if (projection?.desktopSpan) {
|
||||
sourceCreditPendingPtys.add(payload.id)
|
||||
}
|
||||
const pending = appendPendingPtyData(
|
||||
payload.id,
|
||||
pendingData.get(payload.id),
|
||||
payload.data,
|
||||
startSeq,
|
||||
preservesSeq,
|
||||
containsBackgroundOutput,
|
||||
rawLength,
|
||||
payload.transformed === true,
|
||||
projectionId
|
||||
)
|
||||
const shouldEmitPendingCapRestoreMarker =
|
||||
pending.droppedOutput === true &&
|
||||
!overflowMarkedBeforeAppend &&
|
||||
pendingOverflowMarkedPtys.has(payload.id)
|
||||
const nextData = pending.data + getDroppedMode2031RendererData(pending)
|
||||
const isInteractiveOutput = shouldSendInteractiveOutputNow(
|
||||
payload.id,
|
||||
nextData,
|
||||
performance.now()
|
||||
)
|
||||
if (isInteractiveOutput && rendererPtyDispatcherReady) {
|
||||
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
requestDeliveryResyncForGatedPty()
|
||||
return
|
||||
}
|
||||
deletePendingPtyData(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
try {
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
{
|
||||
id: payload.id,
|
||||
data: nextData,
|
||||
...(typeof pending.startSeq === 'number'
|
||||
? {
|
||||
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
|
||||
rawLength: pending.rawLength ?? nextData.length
|
||||
}
|
||||
: {}),
|
||||
...(pending.transformed ? { transformed: true } : {}),
|
||||
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
|
||||
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
|
||||
},
|
||||
pending.projectionAdmissionIds
|
||||
)
|
||||
} finally {
|
||||
updateProducerFlowControl(payload.id)
|
||||
}
|
||||
return
|
||||
}
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
if (
|
||||
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
|
||||
) {
|
||||
requestDeliveryResyncForGatedPty()
|
||||
}
|
||||
if (!flushTimer) {
|
||||
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
sshOutputIntakeCleanup?.()
|
||||
sshOutputIntake = new SshPtyOutputIntake({
|
||||
getModelSequence: (id) => runtime?.getPtyOutputSequence(id) ?? 0,
|
||||
acceptModel: (event, projection) => {
|
||||
if (!runtime) {
|
||||
throw new Error('SSH PTY output requires the main terminal model')
|
||||
}
|
||||
return runtime.acceptPtyDataBounded(
|
||||
event.id,
|
||||
event.data,
|
||||
Date.now(),
|
||||
event.rawLength,
|
||||
event.transformed,
|
||||
projection.desktopSpan ? [projection.desktopSpan] : undefined
|
||||
)
|
||||
},
|
||||
project: (event, projection) =>
|
||||
acceptPtyDataForRenderer(
|
||||
{
|
||||
id: event.id,
|
||||
data: event.data,
|
||||
sequenceChars: event.rawLength,
|
||||
transformed: event.transformed
|
||||
},
|
||||
projection.identity.sequenceEnd,
|
||||
projection
|
||||
),
|
||||
prepareExit: (event) => {
|
||||
const release = preparePtyExitForRenderer(event)
|
||||
if (!release) {
|
||||
throw new Error('pty_renderer_exit_in_progress')
|
||||
}
|
||||
return release
|
||||
},
|
||||
finalizeExit: (event) => {
|
||||
runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation)
|
||||
finalizePtyExitForRenderer(event)
|
||||
},
|
||||
pauseProvider: (generation, id) => {
|
||||
const provider = sshProvidersByGeneration.get(generation) as
|
||||
| (IPtyProvider & { hasPtyDeliveryPauseAdapter?: () => boolean })
|
||||
| undefined
|
||||
if (!provider?.hasPtyDeliveryPauseAdapter?.()) {
|
||||
return false
|
||||
}
|
||||
provider.pauseProducer?.(id)
|
||||
return true
|
||||
},
|
||||
resumeProvider: (generation, id) =>
|
||||
sshProvidersByGeneration.get(generation)?.resumeProducer?.(id),
|
||||
closeProvider: (generation, reason) => {
|
||||
const provider = sshProvidersByGeneration.get(generation)
|
||||
;(
|
||||
provider as (IPtyProvider & { closeOutputIntake?: (reason: string) => void }) | undefined
|
||||
)?.closeOutputIntake?.(reason)
|
||||
},
|
||||
resetModelForMigration: (_generation, id) => runtime?.resetPtyModelAfterMigrationFailure(id),
|
||||
onGenerationClosed: (providerGeneration) => {
|
||||
for (const id of pendingData.keys()) {
|
||||
const pending = pendingData.get(id)
|
||||
if (
|
||||
pending?.projectionAdmissionIds &&
|
||||
sshOutputIntake?.hasProjectionFromGeneration(
|
||||
pending.projectionAdmissionIds,
|
||||
providerGeneration
|
||||
)
|
||||
) {
|
||||
pendingData.delete(id)
|
||||
updateProducerFlowControl(id)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
}
|
||||
}
|
||||
sshProvidersByGeneration.delete(providerGeneration)
|
||||
},
|
||||
publishSourceAck: publishSshPtySourceAck,
|
||||
cancelSourceDelivery: cancelSshPtySourceDelivery
|
||||
})
|
||||
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(
|
||||
sshOutputIntake.getRemoteSourceRangeConsumerHooks()
|
||||
)
|
||||
const cleanupSshOutputIntakeRegistry = installSshPtyOutputIntake(sshOutputIntake)
|
||||
sshOutputIntakeCleanup = () => {
|
||||
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(null)
|
||||
cleanupSshOutputIntakeRegistry()
|
||||
}
|
||||
|
||||
async function shutdownProviderAndDetectExit(
|
||||
provider: IPtyProvider,
|
||||
id: string,
|
||||
@@ -2933,120 +3346,7 @@ export function registerPtyHandlers(
|
||||
const outputSeq = isLocalProvider
|
||||
? runtime?.getPtyOutputSequence(payload.id)
|
||||
: runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed)
|
||||
const rendererData = payload.data
|
||||
const preservesSeq = !payload.transformed && rawLength === payload.data.length
|
||||
const startSeq =
|
||||
typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
|
||||
if (mainWindow.isDestroyed()) {
|
||||
// Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration.
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer)
|
||||
flushTimer = null
|
||||
}
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
clearPendingPtyData()
|
||||
pendingOverflowMarkedPtys.clear()
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearDispatcherReadyWatchdog()
|
||||
return
|
||||
}
|
||||
if (rendererExitingPtyIds.size > 0 && rendererExitingPtyIds.has(payload.id)) {
|
||||
return
|
||||
}
|
||||
const settings = getSettings?.()
|
||||
// Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot.
|
||||
if (shouldDropHiddenRendererPtyData(payload.id, settings)) {
|
||||
const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length)
|
||||
warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (rendererData.length === 0 && !payload.transformed) {
|
||||
return
|
||||
}
|
||||
const containsBackgroundOutput =
|
||||
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
|
||||
if (containsBackgroundOutput) {
|
||||
markHiddenRendererResizeOutputDelivered(payload.id)
|
||||
}
|
||||
const existing = pendingData.get(payload.id)
|
||||
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
|
||||
const pending = appendPendingPtyData(
|
||||
payload.id,
|
||||
existing,
|
||||
rendererData,
|
||||
startSeq,
|
||||
preservesSeq,
|
||||
containsBackgroundOutput,
|
||||
rawLength,
|
||||
payload.transformed === true
|
||||
)
|
||||
const shouldEmitPendingCapRestoreMarker =
|
||||
pending.droppedOutput === true &&
|
||||
!overflowMarkedBeforeAppend &&
|
||||
pendingOverflowMarkedPtys.has(payload.id)
|
||||
const nextData = pending.data + getDroppedMode2031RendererData(pending)
|
||||
const isInteractiveOutput = shouldSendInteractiveOutputNow(
|
||||
payload.id,
|
||||
nextData,
|
||||
performance.now()
|
||||
)
|
||||
// Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate.
|
||||
if (isInteractiveOutput && rendererPtyDispatcherReady) {
|
||||
// Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway.
|
||||
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
requestDeliveryResyncForGatedPty()
|
||||
return
|
||||
}
|
||||
deletePendingPtyData(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
// Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency.
|
||||
try {
|
||||
sendPtyDataToRenderer(payload.id, {
|
||||
id: payload.id,
|
||||
data: nextData,
|
||||
...(typeof pending.startSeq === 'number'
|
||||
? {
|
||||
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
|
||||
rawLength: pending.rawLength ?? nextData.length
|
||||
}
|
||||
: {}),
|
||||
...(pending.transformed ? { transformed: true } : {}),
|
||||
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
|
||||
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
|
||||
})
|
||||
} finally {
|
||||
updateProducerFlowControl(payload.id)
|
||||
}
|
||||
return
|
||||
}
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
// Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable.
|
||||
if (
|
||||
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
|
||||
) {
|
||||
requestDeliveryResyncForGatedPty()
|
||||
}
|
||||
if (!flushTimer) {
|
||||
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
|
||||
}
|
||||
acceptPtyDataForRenderer(payload, outputSeq)
|
||||
})
|
||||
localExitUnsub = localProvider.onExit((payload) => {
|
||||
if (!isCurrentPtyExit(payload)) {
|
||||
@@ -5646,6 +5946,9 @@ export function registerPtyHandlers(
|
||||
const pending = pendingData.get(args.id)
|
||||
if (pending && transition.droppable) {
|
||||
pendingData.delete(args.id)
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
|
||||
}
|
||||
updateProducerFlowControl(args.id)
|
||||
pendingOverflowMarkedPtys.delete(args.id)
|
||||
const drop = recordHiddenRendererPtyDataDrop(args.id, pending.data.length)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
type ClosedGenerationRange = {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export class SshPtyClosedGenerationRanges {
|
||||
private readonly ranges: ClosedGenerationRange[] = []
|
||||
|
||||
add(generation: number): void {
|
||||
let index = 0
|
||||
while (index < this.ranges.length && this.ranges[index]!.end + 1 < generation) {
|
||||
index++
|
||||
}
|
||||
const current = this.ranges[index]
|
||||
if (!current || generation + 1 < current.start) {
|
||||
this.ranges.splice(index, 0, { start: generation, end: generation })
|
||||
return
|
||||
}
|
||||
current.start = Math.min(current.start, generation)
|
||||
current.end = Math.max(current.end, generation)
|
||||
const next = this.ranges[index + 1]
|
||||
if (next && current.end + 1 >= next.start) {
|
||||
current.end = Math.max(current.end, next.end)
|
||||
this.ranges.splice(index + 1, 1)
|
||||
}
|
||||
}
|
||||
|
||||
has(generation: number): boolean {
|
||||
for (const range of this.ranges) {
|
||||
if (generation < range.start) {
|
||||
return false
|
||||
}
|
||||
if (generation <= range.end) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.ranges.length
|
||||
}
|
||||
|
||||
get activeGaps(): number {
|
||||
const highWater = this.ranges.at(-1)?.end ?? 0
|
||||
let closedGenerations = 0
|
||||
for (const range of this.ranges) {
|
||||
closedGenerations += range.end - range.start + 1
|
||||
}
|
||||
// Why: provider generations allocate from 1, so unclosed IDs below high-water remain active.
|
||||
return highWater - closedGenerations
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import {
|
||||
projectionError,
|
||||
reclaimProjectionRecord,
|
||||
type DesktopProjectionSpan,
|
||||
type ProjectionRecord
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
|
||||
export function publishLegacyProjectionPrefix(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
ids: readonly string[],
|
||||
displayChars: number,
|
||||
accountingChars: number
|
||||
): void {
|
||||
let displayRemaining = Math.max(0, displayChars)
|
||||
let accountingRemaining = Math.max(0, accountingChars)
|
||||
for (const id of ids) {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
const displayLength =
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
const unpublishedDisplay = displayLength - record.publishedDisplay
|
||||
const unpublishedAccounting = record.semantics.identity.rawLength - record.publishedAccounting
|
||||
if (displayLength === 0 && unpublishedAccounting > 0) {
|
||||
const publishAccounting = Math.min(accountingRemaining, unpublishedAccounting)
|
||||
if (publishAccounting !== unpublishedAccounting) {
|
||||
throw projectionError('ssh_projection_indivisible_split')
|
||||
}
|
||||
record.publishedAccounting += publishAccounting
|
||||
record.state = 'published'
|
||||
accountingRemaining -= publishAccounting
|
||||
continue
|
||||
}
|
||||
if (unpublishedDisplay <= 0) {
|
||||
continue
|
||||
}
|
||||
const publishDisplay = Math.min(displayRemaining, unpublishedDisplay)
|
||||
if (publishDisplay <= 0) {
|
||||
break
|
||||
}
|
||||
if (record.semantics.identity.transformed && publishDisplay !== unpublishedDisplay) {
|
||||
throw projectionError('ssh_projection_indivisible_split')
|
||||
}
|
||||
const publishAccounting =
|
||||
publishDisplay === unpublishedDisplay
|
||||
? Math.min(accountingRemaining, unpublishedAccounting)
|
||||
: publishDisplay
|
||||
record.publishedDisplay += publishDisplay
|
||||
record.publishedAccounting += publishAccounting
|
||||
record.state = 'published'
|
||||
displayRemaining -= publishDisplay
|
||||
accountingRemaining -= publishAccounting
|
||||
}
|
||||
if (displayRemaining !== 0 || accountingRemaining !== 0) {
|
||||
throw projectionError('ssh_projection_publish_range_mismatch')
|
||||
}
|
||||
}
|
||||
|
||||
export function hasUnpublishedLegacyProjection(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
id: string
|
||||
): boolean {
|
||||
const record = records.get(id)
|
||||
if (!record || record.state === 'reserved') {
|
||||
return false
|
||||
}
|
||||
const displayLength =
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
return (
|
||||
record.publishedDisplay < displayLength ||
|
||||
record.publishedAccounting < record.semantics.identity.rawLength
|
||||
)
|
||||
}
|
||||
|
||||
export function settlePublishedLegacyProjectionPrefix(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
ptyId: string,
|
||||
accountingChars: number,
|
||||
onSettled: ((span: DesktopProjectionSpan, reason: string) => void) | undefined
|
||||
): { settled: number; completed: number } {
|
||||
let remaining = Math.max(0, accountingChars)
|
||||
let settled = 0
|
||||
let completed = 0
|
||||
for (const id of idsByPty.get(ptyId)?.slice() ?? []) {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
const available = record.publishedAccounting - record.settledAccounting
|
||||
if (available <= 0) {
|
||||
continue
|
||||
}
|
||||
const take = Math.min(remaining, available)
|
||||
const finishes =
|
||||
record.settledAccounting + take === record.semantics.identity.rawLength &&
|
||||
record.publishedDisplay ===
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
if (finishes && record.semantics.desktopSpan) {
|
||||
onSettled?.(record.semantics.desktopSpan, 'renderer-parse')
|
||||
}
|
||||
record.settledAccounting += take
|
||||
settled += take
|
||||
remaining -= take
|
||||
if (finishes) {
|
||||
completed++
|
||||
reclaimProjectionRecord(records, idsByPty, id, ptyId)
|
||||
}
|
||||
if (remaining === 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return { settled, completed }
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import type {
|
||||
Mode2031ReplyDecision,
|
||||
Mode2031ReplyScanState
|
||||
} from '../../shared/terminal-color-scheme-protocol'
|
||||
import { INITIAL_MODE_2031_REPLY_SCAN_STATE } from '../../shared/terminal-color-scheme-protocol'
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
|
||||
export type LegacySshProjectionIdentity = Readonly<{
|
||||
projectionSemanticsId: string
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
displayStart: number
|
||||
displayEnd: number
|
||||
sequenceEnd: number
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
}>
|
||||
|
||||
export type LegacySshProjectionSemantics = Readonly<{
|
||||
identity: LegacySshProjectionIdentity
|
||||
desktopSpan?: DesktopProjectionSpan
|
||||
beforeScanner: Readonly<Mode2031ReplyScanState>
|
||||
afterScanner: Readonly<Mode2031ReplyScanState>
|
||||
decision: Mode2031ReplyDecision
|
||||
}>
|
||||
|
||||
export type DesktopProjectionSpan = Readonly<
|
||||
TerminalOutputSourceRange & {
|
||||
projectionSemanticsId: string
|
||||
transform: Readonly<{
|
||||
transformed: boolean
|
||||
rawLengthSu: number
|
||||
scalarSafe: boolean
|
||||
}>
|
||||
}
|
||||
>
|
||||
|
||||
export type ProjectionState = 'reserved' | 'committed' | 'published'
|
||||
|
||||
export type ProjectionRecord = {
|
||||
semantics: LegacySshProjectionSemantics
|
||||
state: ProjectionState
|
||||
publishedDisplay: number
|
||||
publishedAccounting: number
|
||||
settledAccounting: number
|
||||
}
|
||||
|
||||
export type PtyProjectionCursor = {
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
displayEnd: number
|
||||
scanner: Mode2031ReplyScanState
|
||||
}
|
||||
|
||||
export type LegacySshProjectionReservation = Readonly<{
|
||||
semantics: LegacySshProjectionSemantics
|
||||
}>
|
||||
|
||||
export type LegacySshProjectionDebugSnapshot = {
|
||||
reserved: number
|
||||
committed: number
|
||||
published: number
|
||||
settled: number
|
||||
transferred: number
|
||||
rolledBack: number
|
||||
records: number
|
||||
cursors: number
|
||||
}
|
||||
|
||||
export function scannerSnapshot(state: Mode2031ReplyScanState): Readonly<Mode2031ReplyScanState> {
|
||||
return Object.freeze({ tail: state.tail, pendingSubscribe: state.pendingSubscribe })
|
||||
}
|
||||
|
||||
export function resetProjectionCursorForGap(
|
||||
cursors: ReadonlyMap<string, PtyProjectionCursor>,
|
||||
ptyId: string
|
||||
): void {
|
||||
const cursor = cursors.get(ptyId)
|
||||
if (cursor) {
|
||||
cursor.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
}
|
||||
|
||||
export function projectionError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function projectionDebugSnapshot(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
cursors: ReadonlyMap<string, PtyProjectionCursor>,
|
||||
terminalCounts: { settled: number; transferred: number; rolledBack: number }
|
||||
): LegacySshProjectionDebugSnapshot {
|
||||
const result = {
|
||||
reserved: 0,
|
||||
committed: 0,
|
||||
published: 0,
|
||||
...terminalCounts,
|
||||
records: records.size,
|
||||
cursors: cursors.size
|
||||
}
|
||||
for (const record of records.values()) {
|
||||
result[record.state]++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function reclaimProjectionRecord(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
id: string,
|
||||
ptyId: string
|
||||
): void {
|
||||
records.delete(id)
|
||||
const ids = idsByPty.get(ptyId)
|
||||
if (!ids) {
|
||||
return
|
||||
}
|
||||
const index = ids.indexOf(id)
|
||||
if (index >= 0) {
|
||||
ids.splice(index, 1)
|
||||
}
|
||||
if (ids.length === 0) {
|
||||
idsByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
|
||||
export function requireProjectionRecord(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
id: string
|
||||
): ProjectionRecord {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
throw projectionError('ssh_projection_reservation_missing')
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
export function rollbackCommittedProjectionRecord(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
reservation: LegacySshProjectionReservation
|
||||
): string | null {
|
||||
const id = reservation.semantics.identity.projectionSemanticsId
|
||||
const record = records.get(id)
|
||||
if (!record || record.state !== 'committed') {
|
||||
return null
|
||||
}
|
||||
const { identity, beforeScanner } = record.semantics
|
||||
const cursor = cursors.get(identity.ptyId)
|
||||
if (
|
||||
idsByPty.get(identity.ptyId)?.at(-1) !== id ||
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== identity.providerGeneration ||
|
||||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
|
||||
cursor.displayEnd !== identity.displayEnd
|
||||
) {
|
||||
return null
|
||||
}
|
||||
cursor.displayEnd = identity.displayStart
|
||||
cursor.scanner = { ...beforeScanner }
|
||||
reclaimProjectionRecord(records, idsByPty, id, identity.ptyId)
|
||||
return identity.ptyId
|
||||
}
|
||||
|
||||
export function closeProjectionPty(
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
beforeDelete: () => void
|
||||
): void {
|
||||
const cursor = cursors.get(ptyId)
|
||||
if (
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== providerGeneration ||
|
||||
cursor.ptyIncarnation !== ptyIncarnation
|
||||
) {
|
||||
return
|
||||
}
|
||||
beforeDelete()
|
||||
cursors.delete(ptyId)
|
||||
}
|
||||
|
||||
export function getOrCreateProjectionCursor(
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
args: { ptyId: string; providerGeneration: number; ptyIncarnation: string },
|
||||
replaceGeneration: (providerGeneration: number) => void
|
||||
): PtyProjectionCursor {
|
||||
const existing = cursors.get(args.ptyId)
|
||||
if (!existing) {
|
||||
const cursor = {
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayEnd: 0,
|
||||
scanner: { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
cursors.set(args.ptyId, cursor)
|
||||
return cursor
|
||||
}
|
||||
if (args.providerGeneration < existing.providerGeneration) {
|
||||
throw projectionError('ssh_projection_stale_generation')
|
||||
}
|
||||
if (
|
||||
args.providerGeneration === existing.providerGeneration &&
|
||||
args.ptyIncarnation !== existing.ptyIncarnation
|
||||
) {
|
||||
throw projectionError('ssh_projection_stale_incarnation')
|
||||
}
|
||||
if (args.providerGeneration > existing.providerGeneration) {
|
||||
replaceGeneration(existing.providerGeneration)
|
||||
existing.providerGeneration = args.providerGeneration
|
||||
existing.ptyIncarnation = args.ptyIncarnation
|
||||
existing.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
return existing
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import { SshPtyProjectionTerminality } from './ssh-pty-projection-terminality'
|
||||
|
||||
function reserve(
|
||||
ledger: SshPtyLegacyProjectionLedger,
|
||||
overrides: Partial<Parameters<SshPtyLegacyProjectionLedger['reserve']>[0]> = {}
|
||||
) {
|
||||
return ledger.reserve({
|
||||
ptyId: 'pty-1',
|
||||
providerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
data: 'abc',
|
||||
sequenceEnd: 3,
|
||||
rawLength: 3,
|
||||
transformed: false,
|
||||
...overrides
|
||||
})
|
||||
}
|
||||
|
||||
function source(sourceStartSu: number, sourceEndSu: number, deliveryToken = 'token-1') {
|
||||
return {
|
||||
spanId: `span-${sourceStartSu}`,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken,
|
||||
sourceStartSu,
|
||||
sourceEndSu
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshPtyLegacyProjectionLedger', () => {
|
||||
it('drains exact terminality waiters when their provider generation closes', async () => {
|
||||
const terminality = new SshPtyProjectionTerminality()
|
||||
const terminal = terminality.whenTerminal('pty-1', 3, 'incarnation-1', () => true)
|
||||
let nextResolved = false
|
||||
const next = terminality
|
||||
.whenTerminal('pty-1', 4, 'incarnation-2', () => true)
|
||||
.then(() => {
|
||||
nextResolved = true
|
||||
})
|
||||
|
||||
terminality.closeGeneration(3)
|
||||
|
||||
await expect(terminal).resolves.toBeUndefined()
|
||||
expect(nextResolved).toBe(false)
|
||||
terminality.closeGeneration(4)
|
||||
await next
|
||||
})
|
||||
|
||||
it('rolls back scanner and display reservations before commit', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const partial = reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 })
|
||||
expect(ledger.rollback(partial)).toBe(true)
|
||||
|
||||
const next = reserve(ledger, { data: '31h', sequenceEnd: 3 })
|
||||
expect(next.semantics.identity.displayStart).toBe(0)
|
||||
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
expect(next.semantics.decision).toBeNull()
|
||||
})
|
||||
|
||||
it('keeps immutable generation, incarnation, display, sequence, raw length, and scanner facts', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const first = reserve(ledger, {
|
||||
data: '\x1b[?2031h',
|
||||
rawLength: 11,
|
||||
sequenceEnd: 11
|
||||
})
|
||||
const semantics = ledger.commit(first)
|
||||
|
||||
expect(semantics.identity).toMatchObject({
|
||||
providerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
displayStart: 0,
|
||||
displayEnd: 8,
|
||||
sequenceEnd: 11,
|
||||
rawLength: 11
|
||||
})
|
||||
expect(semantics.decision).toBe('subscribed')
|
||||
expect(Object.isFrozen(semantics)).toBe(true)
|
||||
expect(Object.isFrozen(semantics.identity)).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects stale generations and resets cross-chunk scanner state on gaps', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
ledger.commit(reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 }))
|
||||
ledger.resetForGap('pty-1')
|
||||
const next = reserve(ledger, { data: '31h', sequenceEnd: 8 })
|
||||
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
|
||||
expect(() => reserve(ledger, { providerGeneration: 2 })).toThrow(
|
||||
'ssh_projection_stale_generation'
|
||||
)
|
||||
})
|
||||
|
||||
it('publishes, settles, and transfers explicit ranges', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const first = ledger.commit(reserve(ledger))
|
||||
ledger.publishPrefix([first.identity.projectionSemanticsId], 3, 3)
|
||||
expect(ledger.settlePublishedPrefix('pty-1', 2)).toBe(2)
|
||||
expect(ledger.transfer([first.identity.projectionSemanticsId], 'renderer-reload')).toBe(1)
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ transferred: 1, records: 0 })
|
||||
})
|
||||
|
||||
it('resolves exact PTY terminal waiters only after settlement or transfer', async () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const projection = ledger.commit(reserve(ledger))
|
||||
ledger.publishPrefix([projection.identity.projectionSemanticsId], 3, 3)
|
||||
|
||||
let settled = false
|
||||
const terminal = ledger.whenPtyTerminal('pty-1', 3, 'incarnation-1').then(() => {
|
||||
settled = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(settled).toBe(false)
|
||||
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
await Promise.resolve()
|
||||
expect(settled).toBe(false)
|
||||
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
|
||||
await terminal
|
||||
expect(settled).toBe(true)
|
||||
})
|
||||
|
||||
it('publishes and settles transformed source accounting with no display text', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const projection = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: '',
|
||||
sequenceEnd: 9,
|
||||
rawLength: 9,
|
||||
transformed: true
|
||||
})
|
||||
)
|
||||
|
||||
ledger.publishPrefix([projection.identity.projectionSemanticsId], 0, 9)
|
||||
expect(ledger.settlePublishedPrefix('pty-1', 9)).toBe(9)
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ settled: 1, records: 0 })
|
||||
})
|
||||
|
||||
it('reclaims a closed PTY cursor so its id can be reused by a new incarnation', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
ledger.commit(reserve(ledger))
|
||||
ledger.closePty('pty-1', 3, 'incarnation-1', 'pty-exit')
|
||||
|
||||
const next = reserve(ledger, {
|
||||
ptyIncarnation: 'incarnation-2',
|
||||
data: 'next',
|
||||
sequenceEnd: 4,
|
||||
rawLength: 4
|
||||
})
|
||||
expect(next.semantics.identity).toMatchObject({
|
||||
ptyIncarnation: 'incarnation-2',
|
||||
displayStart: 0
|
||||
})
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ records: 1, cursors: 1 })
|
||||
})
|
||||
|
||||
it('keeps split publication attached to one immutable desktop span', () => {
|
||||
const settled: unknown[] = []
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onSettled: (span) => settled.push(span)
|
||||
})
|
||||
const projection = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: 'abcd',
|
||||
rawLength: 4,
|
||||
sequenceEnd: 4,
|
||||
source: source(0, 4)
|
||||
})
|
||||
)
|
||||
const id = projection.identity.projectionSemanticsId
|
||||
expect(ledger.hasUnpublished(id)).toBe(true)
|
||||
|
||||
ledger.publishPrefix([id], 2, 2)
|
||||
expect(ledger.hasUnpublished(id)).toBe(true)
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
expect(settled).toEqual([])
|
||||
ledger.publishPrefix([id], 2, 2)
|
||||
expect(ledger.hasUnpublished(id)).toBe(false)
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
|
||||
expect(settled).toEqual([
|
||||
expect.objectContaining({
|
||||
spanId: 'span-0',
|
||||
projectionSemanticsId: id,
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves scanner and display facts after transfer and delivery-token replacement', () => {
|
||||
const transferred: unknown[] = []
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onTransferred: (span) => transferred.push(span)
|
||||
})
|
||||
const partial = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: '\x1b[?20',
|
||||
rawLength: 5,
|
||||
sequenceEnd: 5,
|
||||
source: source(0, 5)
|
||||
})
|
||||
)
|
||||
ledger.transfer([partial.identity.projectionSemanticsId], 'renderer-reload')
|
||||
const continuation = reserve(ledger, {
|
||||
data: '31h',
|
||||
rawLength: 3,
|
||||
sequenceEnd: 8,
|
||||
source: source(5, 8, 'token-2')
|
||||
})
|
||||
|
||||
expect(continuation.semantics.identity.displayStart).toBe(5)
|
||||
expect(continuation.semantics.beforeScanner.tail).toBe('\x1b[?20')
|
||||
expect(continuation.semantics.decision).toBe('subscribed')
|
||||
expect(transferred).toEqual([
|
||||
expect.objectContaining({
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('leaves a projection intact when its terminal transition cannot commit', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onTransferred: () => {
|
||||
throw new Error('replacement unavailable')
|
||||
}
|
||||
})
|
||||
const projection = ledger.commit(reserve(ledger, { source: source(0, 3) }))
|
||||
|
||||
expect(() =>
|
||||
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
|
||||
).toThrow('replacement unavailable')
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
transferred: 0,
|
||||
records: 1
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,318 @@
|
||||
import { scanMode2031ReplyDecision } from '../../shared/terminal-color-scheme-protocol'
|
||||
import {
|
||||
closeProjectionPty,
|
||||
getOrCreateProjectionCursor,
|
||||
projectionDebugSnapshot,
|
||||
projectionError,
|
||||
reclaimProjectionRecord,
|
||||
resetProjectionCursorForGap,
|
||||
requireProjectionRecord,
|
||||
rollbackCommittedProjectionRecord,
|
||||
scannerSnapshot,
|
||||
type DesktopProjectionSpan,
|
||||
type LegacySshProjectionDebugSnapshot,
|
||||
type LegacySshProjectionReservation,
|
||||
type LegacySshProjectionSemantics,
|
||||
type ProjectionRecord,
|
||||
type PtyProjectionCursor
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
import {
|
||||
projectionHasOpen,
|
||||
resolveProjectionTerminality,
|
||||
SshPtyProjectionTerminality,
|
||||
unpublishedProjectionIds
|
||||
} from './ssh-pty-projection-terminality'
|
||||
import {
|
||||
hasUnpublishedLegacyProjection,
|
||||
publishLegacyProjectionPrefix,
|
||||
settlePublishedLegacyProjectionPrefix
|
||||
} from './ssh-pty-legacy-projection-publication'
|
||||
|
||||
export type {
|
||||
DesktopProjectionSpan,
|
||||
LegacySshProjectionIdentity,
|
||||
LegacySshProjectionReservation,
|
||||
LegacySshProjectionSemantics
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
|
||||
export type SshPtyLegacyProjectionLedgerOptions = {
|
||||
onSettled?: (span: DesktopProjectionSpan, reason: string) => void
|
||||
onTransferred?: (span: DesktopProjectionSpan, reason: string) => void
|
||||
}
|
||||
|
||||
export class SshPtyLegacyProjectionLedger {
|
||||
private nextId = 1
|
||||
private readonly records = new Map<string, ProjectionRecord>()
|
||||
private readonly cursorByPty = new Map<string, PtyProjectionCursor>()
|
||||
private readonly idsByPty = new Map<string, string[]>()
|
||||
private readonly terminality = new SshPtyProjectionTerminality()
|
||||
private settledCount = 0
|
||||
private transferredCount = 0
|
||||
private rolledBackCount = 0
|
||||
|
||||
constructor(private readonly options: SshPtyLegacyProjectionLedgerOptions = {}) {}
|
||||
|
||||
reserve(args: {
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
data: string
|
||||
sequenceEnd: number
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
source?: Readonly<{
|
||||
relayPtyId?: string
|
||||
spanId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
sourceStartSu: number
|
||||
sourceEndSu: number
|
||||
}>
|
||||
}): LegacySshProjectionReservation {
|
||||
const cursor = getOrCreateProjectionCursor(this.cursorByPty, args, (generation) =>
|
||||
this.transferGeneration(generation, 'provider-generation-replaced')
|
||||
)
|
||||
if (
|
||||
cursor.providerGeneration !== args.providerGeneration ||
|
||||
cursor.ptyIncarnation !== args.ptyIncarnation
|
||||
) {
|
||||
throw projectionError('ssh_projection_stale_generation')
|
||||
}
|
||||
const scan = scanMode2031ReplyDecision(cursor.scanner, args.data)
|
||||
const projectionSemanticsId = `ssh-projection:${args.providerGeneration}:${this.nextId++}`
|
||||
const identity = Object.freeze({
|
||||
projectionSemanticsId,
|
||||
ptyId: args.ptyId,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayStart: cursor.displayEnd,
|
||||
displayEnd: cursor.displayEnd + args.data.length,
|
||||
sequenceEnd: args.sequenceEnd,
|
||||
rawLength: args.rawLength,
|
||||
transformed: args.transformed
|
||||
})
|
||||
const semantics = Object.freeze({
|
||||
identity,
|
||||
...(args.source
|
||||
? {
|
||||
desktopSpan: Object.freeze({
|
||||
...args.source,
|
||||
id: args.source.relayPtyId ?? args.ptyId,
|
||||
projectionSemanticsId,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayStart: identity.displayStart,
|
||||
displayEnd: identity.displayEnd,
|
||||
splittable: !args.transformed,
|
||||
transform: Object.freeze({
|
||||
transformed: args.transformed,
|
||||
rawLengthSu: args.rawLength,
|
||||
scalarSafe: !args.transformed
|
||||
})
|
||||
})
|
||||
}
|
||||
: {}),
|
||||
beforeScanner: scannerSnapshot(cursor.scanner),
|
||||
afterScanner: scannerSnapshot(scan.state),
|
||||
decision: scan.decision
|
||||
})
|
||||
this.records.set(projectionSemanticsId, {
|
||||
semantics,
|
||||
state: 'reserved',
|
||||
publishedDisplay: 0,
|
||||
publishedAccounting: 0,
|
||||
settledAccounting: 0
|
||||
})
|
||||
return Object.freeze({ semantics })
|
||||
}
|
||||
|
||||
commit(reservation: LegacySshProjectionReservation): LegacySshProjectionSemantics {
|
||||
const record = requireProjectionRecord(
|
||||
this.records,
|
||||
reservation.semantics.identity.projectionSemanticsId
|
||||
)
|
||||
if (record.state !== 'reserved') {
|
||||
throw projectionError('ssh_projection_commit_invalid')
|
||||
}
|
||||
const { identity, afterScanner } = record.semantics
|
||||
const cursor = this.cursorByPty.get(identity.ptyId)
|
||||
if (
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== identity.providerGeneration ||
|
||||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
|
||||
cursor.displayEnd !== identity.displayStart
|
||||
) {
|
||||
this.records.delete(identity.projectionSemanticsId)
|
||||
throw projectionError('ssh_projection_commit_stale')
|
||||
}
|
||||
cursor.displayEnd = identity.displayEnd
|
||||
cursor.scanner = { ...afterScanner }
|
||||
record.state = 'committed'
|
||||
const ids = this.idsByPty.get(identity.ptyId) ?? []
|
||||
ids.push(identity.projectionSemanticsId)
|
||||
this.idsByPty.set(identity.ptyId, ids)
|
||||
return record.semantics
|
||||
}
|
||||
|
||||
rollback(reservation: LegacySshProjectionReservation): boolean {
|
||||
const id = reservation.semantics.identity.projectionSemanticsId
|
||||
const record = this.records.get(id)
|
||||
if (!record || record.state !== 'reserved') {
|
||||
return false
|
||||
}
|
||||
this.records.delete(id)
|
||||
this.rolledBackCount++
|
||||
return true
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: LegacySshProjectionReservation): boolean {
|
||||
const ptyId = rollbackCommittedProjectionRecord(
|
||||
this.records,
|
||||
this.idsByPty,
|
||||
this.cursorByPty,
|
||||
reservation
|
||||
)
|
||||
if (!ptyId) {
|
||||
return false
|
||||
}
|
||||
this.rolledBackCount++
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
return true
|
||||
}
|
||||
|
||||
publishPrefix(ids: readonly string[], displayChars: number, accountingChars: number): void {
|
||||
publishLegacyProjectionPrefix(this.records, ids, displayChars, accountingChars)
|
||||
}
|
||||
|
||||
settlePublishedPrefix(ptyId: string, accountingChars: number): number {
|
||||
const result = settlePublishedLegacyProjectionPrefix(
|
||||
this.records,
|
||||
this.idsByPty,
|
||||
ptyId,
|
||||
accountingChars,
|
||||
this.options.onSettled
|
||||
)
|
||||
this.settledCount += result.completed
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
return result.settled
|
||||
}
|
||||
|
||||
transfer(ids: readonly string[], reason: string): number {
|
||||
let transferred = 0
|
||||
const touchedPtys = new Set<string>()
|
||||
for (const id of ids.slice()) {
|
||||
const record = this.records.get(id)
|
||||
if (!record || record.state === 'reserved') {
|
||||
continue
|
||||
}
|
||||
const ptyId = record.semantics.identity.ptyId
|
||||
if (record.semantics.desktopSpan) {
|
||||
this.options.onTransferred?.(record.semantics.desktopSpan, reason)
|
||||
}
|
||||
this.transferredCount++
|
||||
reclaimProjectionRecord(this.records, this.idsByPty, id, ptyId)
|
||||
touchedPtys.add(ptyId)
|
||||
transferred++
|
||||
}
|
||||
for (const ptyId of touchedPtys) {
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
}
|
||||
return transferred
|
||||
}
|
||||
|
||||
whenPtyTerminal(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): Promise<void> {
|
||||
return this.terminality.whenTerminal(
|
||||
ptyId,
|
||||
providerGeneration,
|
||||
ptyIncarnation,
|
||||
projectionHasOpen(this.records, this.idsByPty, ptyId)
|
||||
)
|
||||
}
|
||||
|
||||
transferGeneration(providerGeneration: number, reason: string): number {
|
||||
const ids: string[] = []
|
||||
for (const [id, record] of this.records) {
|
||||
if (record.semantics.identity.providerGeneration === providerGeneration) {
|
||||
ids.push(id)
|
||||
}
|
||||
}
|
||||
return this.transfer(ids, reason)
|
||||
}
|
||||
|
||||
transferPty(ptyId: string, reason: string): number {
|
||||
return this.transfer(this.idsByPty.get(ptyId) ?? [], reason)
|
||||
}
|
||||
|
||||
transferUnpublishedPty(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
reason: string
|
||||
): number {
|
||||
const ids = unpublishedProjectionIds(
|
||||
this.records,
|
||||
this.idsByPty.get(ptyId) ?? [],
|
||||
providerGeneration,
|
||||
ptyIncarnation
|
||||
)
|
||||
return this.transfer(ids, reason)
|
||||
}
|
||||
|
||||
closePty(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
reason: string
|
||||
): void {
|
||||
closeProjectionPty(this.cursorByPty, ptyId, providerGeneration, ptyIncarnation, () => {
|
||||
this.transferPty(ptyId, reason)
|
||||
this.idsByPty.delete(ptyId)
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
for (const [id, record] of this.records) {
|
||||
if (record.semantics.identity.providerGeneration !== providerGeneration) {
|
||||
continue
|
||||
}
|
||||
if (record.state === 'reserved') {
|
||||
this.records.delete(id)
|
||||
this.rolledBackCount++
|
||||
} else {
|
||||
this.transfer([id], reason)
|
||||
}
|
||||
}
|
||||
for (const [ptyId, cursor] of this.cursorByPty) {
|
||||
if (cursor.providerGeneration === providerGeneration) {
|
||||
this.cursorByPty.delete(ptyId)
|
||||
this.idsByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
this.terminality.closeGeneration(providerGeneration)
|
||||
}
|
||||
|
||||
resetForGap(ptyId: string): void {
|
||||
resetProjectionCursorForGap(this.cursorByPty, ptyId)
|
||||
}
|
||||
|
||||
get(id: string): LegacySshProjectionSemantics | undefined {
|
||||
return this.records.get(id)?.semantics
|
||||
}
|
||||
|
||||
hasUnpublished(id: string): boolean {
|
||||
return hasUnpublishedLegacyProjection(this.records, id)
|
||||
}
|
||||
|
||||
getDebugSnapshot(): LegacySshProjectionDebugSnapshot {
|
||||
return projectionDebugSnapshot(this.records, this.cursorByPty, {
|
||||
settled: this.settledCount,
|
||||
transferred: this.transferredCount,
|
||||
rolledBack: this.rolledBackCount
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
export type SshPtyModelAdmissionKey = Readonly<{
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionReceipt = Readonly<{
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
sequence: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionDebugSnapshot = Readonly<{
|
||||
sourceUnits: number
|
||||
bytes: number
|
||||
pressureFrames: number
|
||||
pressureBytes: number
|
||||
pausedPtys: number
|
||||
migratingPtys: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionOptions = {
|
||||
perPtyHighSourceUnits?: number
|
||||
perPtyHighBytes?: number
|
||||
perPtyLowSourceUnits?: number
|
||||
perPtyLowBytes?: number
|
||||
globalHighSourceUnits?: number
|
||||
globalHighBytes?: number
|
||||
globalLowSourceUnits?: number
|
||||
globalLowBytes?: number
|
||||
pressureMaxFrames?: number
|
||||
pressureMaxBytes?: number
|
||||
pauseProvider?: (key: SshPtyModelAdmissionKey) => boolean
|
||||
resumeProvider?: (key: SshPtyModelAdmissionKey) => void
|
||||
closeProvider?: (providerGeneration: number, reason: string) => void
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
SshPtyModelAdmissionKey,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
|
||||
export type AdmissionCharge = { sourceUnits: number; bytes: number }
|
||||
|
||||
export type AdmissionEntry = {
|
||||
key: SshPtyModelAdmissionKey
|
||||
charge: AdmissionCharge
|
||||
run: () => { sequence: number; completion: Promise<void> }
|
||||
resolve: (receipt: SshPtyModelAdmissionReceipt) => void
|
||||
reject: (error: Error) => void
|
||||
state: 'queued' | 'running' | 'pressure' | 'settled'
|
||||
}
|
||||
|
||||
export type PtyUsage = AdmissionCharge & {
|
||||
queued: AdmissionEntry[]
|
||||
running: AdmissionEntry | null
|
||||
}
|
||||
|
||||
export function admissionError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function retainedBytes(data: string): number {
|
||||
return Math.max(Buffer.byteLength(data, 'utf8'), 2 * data.length) + 128
|
||||
}
|
||||
|
||||
export function admissionKeyId(key: SshPtyModelAdmissionKey): string {
|
||||
return `${key.providerGeneration}\0${key.ptyId}`
|
||||
}
|
||||
|
||||
export function canReserveAdmission(args: {
|
||||
key: SshPtyModelAdmissionKey
|
||||
charge: AdmissionCharge
|
||||
limits: SshPtyModelAdmissionLimits
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>
|
||||
closingGenerations: ReadonlySet<number>
|
||||
globalSourceUnits: number
|
||||
globalBytes: number
|
||||
}): boolean {
|
||||
if (args.closingGenerations.has(args.key.providerGeneration)) {
|
||||
return false
|
||||
}
|
||||
const usage = args.usageByPty.get(admissionKeyId(args.key))
|
||||
return (
|
||||
(usage?.sourceUnits ?? 0) + args.charge.sourceUnits <= args.limits.perPtyHighSourceUnits &&
|
||||
(usage?.bytes ?? 0) + args.charge.bytes <= args.limits.perPtyHighBytes &&
|
||||
args.globalSourceUnits + args.charge.sourceUnits <= args.limits.globalHighSourceUnits &&
|
||||
args.globalBytes + args.charge.bytes <= args.limits.globalHighBytes
|
||||
)
|
||||
}
|
||||
|
||||
export function pressureHasAdmissionKey(
|
||||
entries: readonly AdmissionEntry[],
|
||||
key: SshPtyModelAdmissionKey
|
||||
): boolean {
|
||||
const id = admissionKeyId(key)
|
||||
return entries.some((entry) => admissionKeyId(entry.key) === id)
|
||||
}
|
||||
|
||||
export function takePressureEntriesForGeneration(
|
||||
entries: AdmissionEntry[],
|
||||
providerGeneration: number
|
||||
): AdmissionEntry[] {
|
||||
const removed: AdmissionEntry[] = []
|
||||
for (let index = entries.length - 1; index >= 0; index--) {
|
||||
if (entries[index]!.key.providerGeneration === providerGeneration) {
|
||||
removed.unshift(entries.splice(index, 1)[0]!)
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
export function cancelAdmissionGeneration(args: {
|
||||
pressure: AdmissionEntry[]
|
||||
usageByPty: Map<string, PtyUsage>
|
||||
idleWaiters: Map<string, Set<() => void>>
|
||||
providerGeneration: number
|
||||
error: Error
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
}): number {
|
||||
let pressureBytes = 0
|
||||
for (const entry of takePressureEntriesForGeneration(args.pressure, args.providerGeneration)) {
|
||||
pressureBytes += entry.charge.bytes
|
||||
entry.state = 'settled'
|
||||
entry.reject(args.error)
|
||||
}
|
||||
for (const [id, usage] of args.usageByPty) {
|
||||
const canceled = usage.queued.filter(
|
||||
(entry) => entry.key.providerGeneration === args.providerGeneration
|
||||
)
|
||||
usage.queued = usage.queued.filter(
|
||||
(entry) => entry.key.providerGeneration !== args.providerGeneration
|
||||
)
|
||||
if (usage.running?.key.providerGeneration === args.providerGeneration) {
|
||||
canceled.push(usage.running)
|
||||
usage.running = null
|
||||
}
|
||||
for (const entry of canceled) {
|
||||
if (entry.state === 'settled') {
|
||||
continue
|
||||
}
|
||||
entry.state = 'settled'
|
||||
args.release(entry.key, entry.charge)
|
||||
entry.reject(args.error)
|
||||
}
|
||||
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
|
||||
args.usageByPty.delete(id)
|
||||
}
|
||||
resolveAdmissionIdleWaiters(args.usageByPty, args.pressure, args.idleWaiters, id)
|
||||
}
|
||||
return pressureBytes
|
||||
}
|
||||
|
||||
export function takePausedGeneration(
|
||||
paused: Map<string, SshPtyModelAdmissionKey>,
|
||||
providerGeneration: number
|
||||
): SshPtyModelAdmissionKey[] {
|
||||
const removed: SshPtyModelAdmissionKey[] = []
|
||||
for (const [id, key] of paused) {
|
||||
if (key.providerGeneration === providerGeneration) {
|
||||
paused.delete(id)
|
||||
removed.push(key)
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
export function resolveAdmissionIdleWaiters(
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>,
|
||||
pressure: readonly AdmissionEntry[],
|
||||
waitersById: Map<string, Set<() => void>>,
|
||||
id: string
|
||||
): void {
|
||||
const usage = usageByPty.get(id)
|
||||
const stillPressured = pressure.some((entry) => admissionKeyId(entry.key) === id)
|
||||
if (usage?.running || (usage?.queued.length ?? 0) > 0 || stillPressured) {
|
||||
return
|
||||
}
|
||||
const waiters = waitersById.get(id)
|
||||
if (!waiters) {
|
||||
return
|
||||
}
|
||||
waitersById.delete(id)
|
||||
for (const resolve of waiters) {
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { SshPtyModelAdmissionOptions } from './ssh-pty-model-admission-contract'
|
||||
|
||||
export type SshPtyModelAdmissionLimits = Readonly<{
|
||||
perPtyHighSourceUnits: number
|
||||
perPtyHighBytes: number
|
||||
perPtyLowSourceUnits: number
|
||||
perPtyLowBytes: number
|
||||
globalHighSourceUnits: number
|
||||
globalHighBytes: number
|
||||
globalLowSourceUnits: number
|
||||
globalLowBytes: number
|
||||
pressureMaxFrames: number
|
||||
pressureMaxBytes: number
|
||||
}>
|
||||
|
||||
const DEFAULT_PER_PTY_HIGH_SOURCE_UNITS = 256 * 1024
|
||||
const DEFAULT_PER_PTY_HIGH_BYTES = 2 * 1024 * 1024
|
||||
const DEFAULT_GLOBAL_HIGH_SOURCE_UNITS = 50 * DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
|
||||
const DEFAULT_GLOBAL_HIGH_BYTES = 64 * 1024 * 1024
|
||||
|
||||
export function resolveSshPtyModelAdmissionLimits(
|
||||
options: SshPtyModelAdmissionOptions
|
||||
): SshPtyModelAdmissionLimits {
|
||||
const perPtyHighSourceUnits = options.perPtyHighSourceUnits ?? DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
|
||||
const perPtyHighBytes = options.perPtyHighBytes ?? DEFAULT_PER_PTY_HIGH_BYTES
|
||||
return {
|
||||
perPtyHighSourceUnits,
|
||||
perPtyHighBytes,
|
||||
perPtyLowSourceUnits: options.perPtyLowSourceUnits ?? Math.floor(perPtyHighSourceUnits / 2),
|
||||
perPtyLowBytes: options.perPtyLowBytes ?? Math.floor(perPtyHighBytes / 2),
|
||||
globalHighSourceUnits: options.globalHighSourceUnits ?? DEFAULT_GLOBAL_HIGH_SOURCE_UNITS,
|
||||
globalHighBytes: options.globalHighBytes ?? DEFAULT_GLOBAL_HIGH_BYTES,
|
||||
globalLowSourceUnits: options.globalLowSourceUnits ?? 8 * 1024 * 1024,
|
||||
globalLowBytes: options.globalLowBytes ?? 48 * 1024 * 1024,
|
||||
pressureMaxFrames: options.pressureMaxFrames ?? 64,
|
||||
pressureMaxBytes: options.pressureMaxBytes ?? 1024 * 1024
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import type { AdmissionCharge, AdmissionEntry, PtyUsage } from './ssh-pty-model-admission-entry'
|
||||
import { admissionError, admissionKeyId } from './ssh-pty-model-admission-entry'
|
||||
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
|
||||
export function beginSshPtyModelAdmissionMigration(args: {
|
||||
key: SshPtyModelAdmissionKey
|
||||
migratingPtys: Set<string>
|
||||
pressure: SshPtyModelAdmissionPressure
|
||||
usageByPty: Map<string, PtyUsage>
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
cleanup: (id: string, usage: PtyUsage) => void
|
||||
}): void {
|
||||
const id = admissionKeyId(args.key)
|
||||
if (args.migratingPtys.has(id)) {
|
||||
return
|
||||
}
|
||||
args.migratingPtys.add(id)
|
||||
const error = admissionError('ssh_model_migration_queued_canceled')
|
||||
args.pressure.cancelQueuedPty(args.key, error)
|
||||
const usage = args.usageByPty.get(id)
|
||||
if (!usage) {
|
||||
return
|
||||
}
|
||||
const queued = usage.queued
|
||||
usage.queued = []
|
||||
for (const entry of queued) {
|
||||
cancelQueuedEntry(entry, error, args.release)
|
||||
}
|
||||
args.cleanup(id, usage)
|
||||
}
|
||||
|
||||
export function closeSshPtyModelAdmissionMigrations(
|
||||
migratingPtys: Set<string>,
|
||||
providerGeneration: number
|
||||
): void {
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const id of migratingPtys) {
|
||||
if (id.startsWith(prefix)) {
|
||||
migratingPtys.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function settleSshPtyModelAdmissionFailure(args: {
|
||||
id: string
|
||||
usage: PtyUsage
|
||||
entry: AdmissionEntry
|
||||
error: Error
|
||||
migratingPtys: ReadonlySet<string>
|
||||
closingGenerations: Set<number>
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
closeGeneration: (providerGeneration: number) => void
|
||||
cleanup: (id: string, usage: PtyUsage) => void
|
||||
}): void {
|
||||
if (args.entry.state !== 'running' || args.usage.running !== args.entry) {
|
||||
return
|
||||
}
|
||||
const migrationOwnsFailure = args.migratingPtys.has(args.id)
|
||||
if (!migrationOwnsFailure) {
|
||||
args.closingGenerations.add(args.entry.key.providerGeneration)
|
||||
}
|
||||
args.usage.running = null
|
||||
args.entry.state = 'settled'
|
||||
args.release(args.entry.key, args.entry.charge)
|
||||
args.entry.reject(migrationOwnsFailure ? migrationCompletionError(args.error) : args.error)
|
||||
if (!migrationOwnsFailure) {
|
||||
args.closeGeneration(args.entry.key.providerGeneration)
|
||||
}
|
||||
args.cleanup(args.id, args.usage)
|
||||
}
|
||||
|
||||
function migrationCompletionError(cause: Error): Error {
|
||||
return Object.assign(new Error(cause.message), {
|
||||
code: 'ssh_model_migration_completion_failed',
|
||||
cause
|
||||
})
|
||||
}
|
||||
|
||||
function cancelQueuedEntry(
|
||||
entry: AdmissionEntry,
|
||||
error: Error,
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
): void {
|
||||
if (entry.state === 'settled') {
|
||||
return
|
||||
}
|
||||
entry.state = 'settled'
|
||||
release(entry.key, entry.charge)
|
||||
entry.reject(error)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import {
|
||||
admissionKeyId,
|
||||
pressureHasAdmissionKey,
|
||||
takePausedGeneration,
|
||||
type AdmissionEntry,
|
||||
type PtyUsage
|
||||
} from './ssh-pty-model-admission-entry'
|
||||
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
|
||||
type PressureOptions = {
|
||||
limits: SshPtyModelAdmissionLimits
|
||||
pauseProvider: (key: SshPtyModelAdmissionKey) => boolean
|
||||
resumeProvider: (key: SshPtyModelAdmissionKey) => void
|
||||
}
|
||||
|
||||
type PromotionOptions = {
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>
|
||||
disposed: boolean
|
||||
canReserve: (entry: AdmissionEntry) => boolean
|
||||
reserve: (entry: AdmissionEntry) => void
|
||||
isBelowGlobalLowWatermark: () => boolean
|
||||
}
|
||||
|
||||
export class SshPtyModelAdmissionPressure {
|
||||
private readonly entries: AdmissionEntry[] = []
|
||||
private readonly pausedKeys = new Map<string, SshPtyModelAdmissionKey>()
|
||||
private retainedBytes = 0
|
||||
|
||||
constructor(private readonly options: PressureOptions) {}
|
||||
|
||||
get values(): readonly AdmissionEntry[] {
|
||||
return this.entries
|
||||
}
|
||||
|
||||
get frameCount(): number {
|
||||
return this.entries.length
|
||||
}
|
||||
|
||||
get bytes(): number {
|
||||
return this.retainedBytes
|
||||
}
|
||||
|
||||
get pausedPtyCount(): number {
|
||||
return this.pausedKeys.size
|
||||
}
|
||||
|
||||
get pausedProviderGenerations(): ReadonlySet<number> {
|
||||
return new Set(Array.from(this.pausedKeys.values(), (key) => key.providerGeneration))
|
||||
}
|
||||
|
||||
has(key: SshPtyModelAdmissionKey): boolean {
|
||||
return pressureHasAdmissionKey(this.entries, key)
|
||||
}
|
||||
|
||||
admit(entry: AdmissionEntry): boolean {
|
||||
const id = admissionKeyId(entry.key)
|
||||
const paused = this.pausedKeys.has(id) || this.options.pauseProvider(entry.key)
|
||||
if (paused) {
|
||||
this.pausedKeys.set(id, entry.key)
|
||||
}
|
||||
if (
|
||||
!paused ||
|
||||
this.entries.length >= this.options.limits.pressureMaxFrames ||
|
||||
this.retainedBytes + entry.charge.bytes > this.options.limits.pressureMaxBytes
|
||||
) {
|
||||
return false
|
||||
}
|
||||
entry.state = 'pressure'
|
||||
this.entries.push(entry)
|
||||
this.retainedBytes += entry.charge.bytes
|
||||
return true
|
||||
}
|
||||
|
||||
cancelGeneration(
|
||||
providerGeneration: number,
|
||||
cancelPressureAndReserved: (entries: AdmissionEntry[]) => number
|
||||
): void {
|
||||
this.retainedBytes -= cancelPressureAndReserved(this.entries)
|
||||
for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) {
|
||||
this.resume(key)
|
||||
}
|
||||
}
|
||||
|
||||
cancelPty(key: SshPtyModelAdmissionKey, error: Error, cancelReserved: () => void): void {
|
||||
this.cancelQueuedPty(key, error)
|
||||
cancelReserved()
|
||||
}
|
||||
|
||||
cancelQueuedPty(key: SshPtyModelAdmissionKey, error: Error): void {
|
||||
const id = admissionKeyId(key)
|
||||
const canceled: AdmissionEntry[] = []
|
||||
for (let index = this.entries.length - 1; index >= 0; index--) {
|
||||
if (admissionKeyId(this.entries[index]!.key) === id) {
|
||||
canceled.push(this.entries.splice(index, 1)[0]!)
|
||||
}
|
||||
}
|
||||
this.rejectPressureEntries(canceled, error)
|
||||
const paused = this.pausedKeys.get(id)
|
||||
if (paused) {
|
||||
this.pausedKeys.delete(id)
|
||||
this.resume(paused)
|
||||
}
|
||||
}
|
||||
|
||||
promoteAndResume(options: PromotionOptions): void {
|
||||
if (options.disposed) {
|
||||
return
|
||||
}
|
||||
for (let index = 0; index < this.entries.length; ) {
|
||||
const entry = this.entries[index]!
|
||||
const hasEarlierEntryForPty = this.entries
|
||||
.slice(0, index)
|
||||
.some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key))
|
||||
if (hasEarlierEntryForPty || !options.canReserve(entry)) {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
this.entries.splice(index, 1)
|
||||
this.retainedBytes -= entry.charge.bytes
|
||||
options.reserve(entry)
|
||||
}
|
||||
if (!options.isBelowGlobalLowWatermark()) {
|
||||
return
|
||||
}
|
||||
for (const [id, key] of this.pausedKeys) {
|
||||
const usage = options.usageByPty.get(id)
|
||||
if (
|
||||
this.has(key) ||
|
||||
(usage?.sourceUnits ?? 0) > this.options.limits.perPtyLowSourceUnits ||
|
||||
(usage?.bytes ?? 0) > this.options.limits.perPtyLowBytes
|
||||
) {
|
||||
continue
|
||||
}
|
||||
this.pausedKeys.delete(id)
|
||||
this.resume(key)
|
||||
}
|
||||
}
|
||||
|
||||
private rejectPressureEntries(entries: readonly AdmissionEntry[], error: Error): void {
|
||||
for (const entry of entries) {
|
||||
this.retainedBytes -= entry.charge.bytes
|
||||
entry.state = 'settled'
|
||||
entry.reject(error)
|
||||
}
|
||||
}
|
||||
|
||||
private resume(key: SshPtyModelAdmissionKey): void {
|
||||
try {
|
||||
this.options.resumeProvider(key)
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { SshPtyModelAdmissionDebugSnapshot } from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
|
||||
export function sshPtyModelAdmissionSnapshot(
|
||||
sourceUnits: number,
|
||||
bytes: number,
|
||||
pressure: SshPtyModelAdmissionPressure,
|
||||
migratingPtys: ReadonlySet<string>
|
||||
): SshPtyModelAdmissionDebugSnapshot {
|
||||
return {
|
||||
sourceUnits,
|
||||
bytes,
|
||||
pressureFrames: pressure.frameCount,
|
||||
pressureBytes: pressure.bytes,
|
||||
pausedPtys: pressure.pausedPtyCount,
|
||||
migratingPtys: migratingPtys.size
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void
|
||||
let reject!: (error: Error) => void
|
||||
const promise = new Promise<void>((promiseResolve, promiseReject) => {
|
||||
resolve = promiseResolve
|
||||
reject = promiseReject
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
function accept(admission: SshPtyModelAdmission, completion: Promise<void>) {
|
||||
return admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion
|
||||
}))
|
||||
}
|
||||
|
||||
describe('SshPtyModelAdmission', () => {
|
||||
it('freezes migration while retaining the running raw completion', async () => {
|
||||
const runningCompletion = deferred()
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const running = accept(admission, runningCompletion.promise)
|
||||
const queued = accept(admission, Promise.resolve())
|
||||
|
||||
admission.beginMigration({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
await expect(queued).rejects.toThrow('ssh_model_migration_queued_canceled')
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 4 })
|
||||
await expect(
|
||||
admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'late', 4, () => ({
|
||||
sequence: 8,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
).rejects.toThrow('ssh_model_admission_migrating')
|
||||
|
||||
runningCompletion.resolve()
|
||||
await expect(running).resolves.toMatchObject({ sequence: 4 })
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('cancels a never-settling running entry when its generation closes', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const receipt = accept(admission, new Promise<void>(() => {}))
|
||||
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
admission.closeGeneration(7, 'provider-closed')
|
||||
|
||||
await expect(receipt).rejects.toThrow('provider-closed')
|
||||
await expect(idle).resolves.toBeUndefined()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('cancels a never-settling running entry on disposal', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const receipt = accept(admission, new Promise<void>(() => {}))
|
||||
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
admission.dispose()
|
||||
|
||||
await expect(receipt).rejects.toThrow('ssh_model_admission_disposed')
|
||||
await expect(idle).resolves.toBeUndefined()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('keeps non-migrating callback failure generation-fatal across sibling PTYs', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const failedCompletion = deferred()
|
||||
const siblingCompletion = deferred()
|
||||
const failed = accept(admission, failedCompletion.promise)
|
||||
const sibling = admission.accept({ ptyId: 'pty-2', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: siblingCompletion.promise
|
||||
}))
|
||||
|
||||
failedCompletion.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(failed).rejects.toThrow('emulator failed')
|
||||
await expect(sibling).rejects.toThrow('ssh_model_admission_completion_failed')
|
||||
await expect(
|
||||
admission.accept({ ptyId: 'pty-3', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
).rejects.toThrow('ssh_model_admission_generation_closed')
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
siblingCompletion.resolve()
|
||||
await Promise.resolve()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('resumes every paused provider generation exactly once on disposal', async () => {
|
||||
const resumeProvider = vi.fn()
|
||||
const admission = new SshPtyModelAdmission({
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
pressureMaxFrames: 1,
|
||||
pressureMaxBytes: 1024,
|
||||
pauseProvider: () => true,
|
||||
resumeProvider
|
||||
})
|
||||
const running = accept(admission, new Promise<void>(() => {}))
|
||||
const pressured = accept(admission, Promise.resolve())
|
||||
const rejected = admission.accept({ ptyId: 'pty-2', providerGeneration: 8 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
|
||||
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
|
||||
admission.dispose()
|
||||
admission.dispose()
|
||||
|
||||
await expect(running).rejects.toThrow('ssh_model_admission_disposed')
|
||||
await expect(pressured).rejects.toThrow('ssh_model_admission_disposed')
|
||||
expect(resumeProvider.mock.calls).toEqual([
|
||||
[{ ptyId: 'pty-1', providerGeneration: 7 }],
|
||||
[{ ptyId: 'pty-2', providerGeneration: 8 }]
|
||||
])
|
||||
})
|
||||
|
||||
it.each(['resolve', 'reject'] as const)(
|
||||
'ignores a late completion %s after cancellation',
|
||||
async (settle) => {
|
||||
const completion = deferred()
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const onResolve = vi.fn()
|
||||
const onReject = vi.fn()
|
||||
const receipt = accept(admission, completion.promise)
|
||||
const observed = receipt.then(onResolve, onReject)
|
||||
|
||||
admission.closeGeneration(7, 'provider-closed')
|
||||
await observed
|
||||
if (settle === 'resolve') {
|
||||
completion.resolve()
|
||||
} else {
|
||||
completion.reject(new Error('late emulator failure'))
|
||||
}
|
||||
await Promise.resolve()
|
||||
|
||||
expect(onResolve).not.toHaveBeenCalled()
|
||||
expect(onReject).toHaveBeenCalledTimes(1)
|
||||
expect(onReject.mock.calls[0]?.[0]).toMatchObject({ message: 'provider-closed' })
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,315 @@
|
||||
import type {
|
||||
SshPtyModelAdmissionKey,
|
||||
SshPtyModelAdmissionOptions,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import {
|
||||
admissionError,
|
||||
admissionKeyId,
|
||||
canReserveAdmission,
|
||||
cancelAdmissionGeneration,
|
||||
retainedBytes,
|
||||
resolveAdmissionIdleWaiters,
|
||||
type AdmissionCharge,
|
||||
type AdmissionEntry,
|
||||
type PtyUsage
|
||||
} from './ssh-pty-model-admission-entry'
|
||||
import { resolveSshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
import * as modelAdmissionMigration from './ssh-pty-model-admission-migration'
|
||||
import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
import { sshPtyModelAdmissionSnapshot } from './ssh-pty-model-admission-snapshot'
|
||||
|
||||
export type * from './ssh-pty-model-admission-contract'
|
||||
|
||||
export class SshPtyModelAdmission {
|
||||
private readonly limits: ReturnType<typeof resolveSshPtyModelAdmissionLimits>
|
||||
private readonly closeProvider: (providerGeneration: number, reason: string) => void
|
||||
private readonly usageByPty = new Map<string, PtyUsage>()
|
||||
private readonly pressure: SshPtyModelAdmissionPressure
|
||||
private readonly idleWaiters = new Map<string, Set<() => void>>()
|
||||
private readonly closingGenerations = new Set<number>()
|
||||
private readonly migratingPtys = new Set<string>()
|
||||
private globalSourceUnits = 0
|
||||
private globalBytes = 0
|
||||
private disposed = false
|
||||
|
||||
constructor(options: SshPtyModelAdmissionOptions = {}) {
|
||||
this.limits = resolveSshPtyModelAdmissionLimits(options)
|
||||
this.pressure = new SshPtyModelAdmissionPressure({
|
||||
limits: this.limits,
|
||||
pauseProvider: options.pauseProvider ?? (() => false),
|
||||
resumeProvider: options.resumeProvider ?? (() => {})
|
||||
})
|
||||
this.closeProvider = options.closeProvider ?? (() => {})
|
||||
}
|
||||
|
||||
accept(
|
||||
key: SshPtyModelAdmissionKey,
|
||||
data: string,
|
||||
sourceUnits: number,
|
||||
run: () => { sequence: number; completion: Promise<void> }
|
||||
): Promise<SshPtyModelAdmissionReceipt> {
|
||||
if (this.disposed) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_disposed'))
|
||||
}
|
||||
if (this.closingGenerations.has(key.providerGeneration)) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_generation_closed'))
|
||||
}
|
||||
if (this.migratingPtys.has(admissionKeyId(key))) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_migrating'))
|
||||
}
|
||||
const charge = { sourceUnits, bytes: retainedBytes(data) }
|
||||
return new Promise((resolve, reject) => {
|
||||
const entry: AdmissionEntry = {
|
||||
key: { ...key },
|
||||
charge,
|
||||
run,
|
||||
resolve,
|
||||
reject,
|
||||
state: 'queued'
|
||||
}
|
||||
if (this.canReserve(key, charge) && !this.pressure.has(key)) {
|
||||
this.reserveAndQueue(entry)
|
||||
return
|
||||
}
|
||||
if (!this.pressure.admit(entry)) {
|
||||
entry.reject(admissionError('ssh_model_admission_pressure_exhausted'))
|
||||
this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason = 'provider-generation-closed'): void {
|
||||
this.closingGenerations.add(providerGeneration)
|
||||
const error = admissionError(reason)
|
||||
this.pressure.cancelGeneration(providerGeneration, (pressure) =>
|
||||
cancelAdmissionGeneration({
|
||||
pressure,
|
||||
usageByPty: this.usageByPty,
|
||||
idleWaiters: this.idleWaiters,
|
||||
providerGeneration,
|
||||
error,
|
||||
release: (key, charge) => this.release(key, charge)
|
||||
})
|
||||
)
|
||||
modelAdmissionMigration.closeSshPtyModelAdmissionMigrations(
|
||||
this.migratingPtys,
|
||||
providerGeneration
|
||||
)
|
||||
const generationPrefix = `${providerGeneration}\0`
|
||||
for (const id of this.idleWaiters.keys()) {
|
||||
if (id.startsWith(generationPrefix)) {
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
beginMigration(key: SshPtyModelAdmissionKey): void {
|
||||
modelAdmissionMigration.beginSshPtyModelAdmissionMigration({
|
||||
key,
|
||||
migratingPtys: this.migratingPtys,
|
||||
pressure: this.pressure,
|
||||
usageByPty: this.usageByPty,
|
||||
release: (entryKey, charge) => this.release(entryKey, charge),
|
||||
cleanup: (id, usage) => this.cleanupUsage(id, usage)
|
||||
})
|
||||
const id = admissionKeyId(key)
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
cancelPty(key: SshPtyModelAdmissionKey, reason: string): void {
|
||||
const id = admissionKeyId(key)
|
||||
const error = admissionError(reason)
|
||||
this.pressure.cancelPty(key, error, () => {
|
||||
const usage = this.usageByPty.get(id)
|
||||
if (usage) {
|
||||
const canceled = [...usage.queued, ...(usage.running ? [usage.running] : [])]
|
||||
usage.queued = []
|
||||
usage.running = null
|
||||
for (const entry of canceled) {
|
||||
if (entry.state === 'settled') {
|
||||
continue
|
||||
}
|
||||
entry.state = 'settled'
|
||||
this.release(entry.key, entry.charge)
|
||||
entry.reject(error)
|
||||
}
|
||||
this.usageByPty.delete(id)
|
||||
}
|
||||
})
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
whenIdle(key: SshPtyModelAdmissionKey): Promise<void> {
|
||||
const id = admissionKeyId(key)
|
||||
const usage = this.usageByPty.get(id)
|
||||
const hasPressure = this.pressure.has(key)
|
||||
if ((!usage || (!usage.running && usage.queued.length === 0)) && !hasPressure) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const waiters = this.idleWaiters.get(id) ?? new Set<() => void>()
|
||||
waiters.add(resolve)
|
||||
this.idleWaiters.set(id, waiters)
|
||||
})
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
const generations = new Set<number>()
|
||||
for (const usage of this.usageByPty.values()) {
|
||||
if (usage.running) {
|
||||
generations.add(usage.running.key.providerGeneration)
|
||||
}
|
||||
for (const entry of usage.queued) {
|
||||
generations.add(entry.key.providerGeneration)
|
||||
}
|
||||
}
|
||||
for (const entry of this.pressure.values) {
|
||||
generations.add(entry.key.providerGeneration)
|
||||
}
|
||||
for (const generation of this.pressure.pausedProviderGenerations) {
|
||||
generations.add(generation)
|
||||
}
|
||||
for (const generation of generations) {
|
||||
this.closeGeneration(generation, 'ssh_model_admission_disposed')
|
||||
}
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return sshPtyModelAdmissionSnapshot(
|
||||
this.globalSourceUnits,
|
||||
this.globalBytes,
|
||||
this.pressure,
|
||||
this.migratingPtys
|
||||
)
|
||||
}
|
||||
|
||||
private canReserve(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): boolean {
|
||||
if (this.migratingPtys.has(admissionKeyId(key))) {
|
||||
return false
|
||||
}
|
||||
return canReserveAdmission({
|
||||
key,
|
||||
charge,
|
||||
limits: this.limits,
|
||||
usageByPty: this.usageByPty,
|
||||
closingGenerations: this.closingGenerations,
|
||||
globalSourceUnits: this.globalSourceUnits,
|
||||
globalBytes: this.globalBytes
|
||||
})
|
||||
}
|
||||
|
||||
private reserveAndQueue(entry: AdmissionEntry): void {
|
||||
const id = admissionKeyId(entry.key)
|
||||
let usage = this.usageByPty.get(id)
|
||||
if (!usage) {
|
||||
usage = { sourceUnits: 0, bytes: 0, queued: [], running: null }
|
||||
this.usageByPty.set(id, usage)
|
||||
}
|
||||
usage.sourceUnits += entry.charge.sourceUnits
|
||||
usage.bytes += entry.charge.bytes
|
||||
this.globalSourceUnits += entry.charge.sourceUnits
|
||||
this.globalBytes += entry.charge.bytes
|
||||
entry.state = 'queued'
|
||||
usage.queued.push(entry)
|
||||
this.startNext(id, usage)
|
||||
}
|
||||
|
||||
private startNext(id: string, usage: PtyUsage): void {
|
||||
if (usage.running) {
|
||||
return
|
||||
}
|
||||
const entry = usage.queued.shift()
|
||||
if (!entry) {
|
||||
this.promotePressureAndResumeProvider()
|
||||
return
|
||||
}
|
||||
usage.running = entry
|
||||
entry.state = 'running'
|
||||
let execution: { sequence: number; completion: Promise<void> }
|
||||
try {
|
||||
execution = entry.run()
|
||||
} catch (error) {
|
||||
if (this.finishEntry(id, usage, entry)) {
|
||||
entry.reject(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
return
|
||||
}
|
||||
void execution.completion.then(
|
||||
() => {
|
||||
if (!this.finishEntry(id, usage, entry)) {
|
||||
return
|
||||
}
|
||||
entry.resolve({
|
||||
ptyId: entry.key.ptyId,
|
||||
providerGeneration: entry.key.providerGeneration,
|
||||
sequence: execution.sequence
|
||||
})
|
||||
},
|
||||
(error) => {
|
||||
this.failEntry(id, usage, entry, error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void {
|
||||
modelAdmissionMigration.settleSshPtyModelAdmissionFailure({
|
||||
id,
|
||||
usage,
|
||||
entry,
|
||||
error,
|
||||
migratingPtys: this.migratingPtys,
|
||||
closingGenerations: this.closingGenerations,
|
||||
release: (key, charge) => this.release(key, charge),
|
||||
closeGeneration: (providerGeneration) =>
|
||||
this.closeGeneration(providerGeneration, 'ssh_model_admission_completion_failed'),
|
||||
cleanup: (entryId, entryUsage) => this.cleanupUsage(entryId, entryUsage)
|
||||
})
|
||||
}
|
||||
|
||||
private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean {
|
||||
if (entry.state !== 'running' || usage.running !== entry) {
|
||||
return false
|
||||
}
|
||||
usage.running = null
|
||||
entry.state = 'settled'
|
||||
this.release(entry.key, entry.charge)
|
||||
this.cleanupUsage(id, usage)
|
||||
return true
|
||||
}
|
||||
|
||||
private cleanupUsage(id: string, usage: PtyUsage): void {
|
||||
this.startNext(id, usage)
|
||||
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
|
||||
this.usageByPty.delete(id)
|
||||
}
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
private release(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): void {
|
||||
const usage = this.usageByPty.get(admissionKeyId(key))
|
||||
if (usage) {
|
||||
usage.sourceUnits = Math.max(0, usage.sourceUnits - charge.sourceUnits)
|
||||
usage.bytes = Math.max(0, usage.bytes - charge.bytes)
|
||||
}
|
||||
this.globalSourceUnits = Math.max(0, this.globalSourceUnits - charge.sourceUnits)
|
||||
this.globalBytes = Math.max(0, this.globalBytes - charge.bytes)
|
||||
this.promotePressureAndResumeProvider()
|
||||
}
|
||||
|
||||
private promotePressureAndResumeProvider(): void {
|
||||
this.pressure.promoteAndResume({
|
||||
usageByPty: this.usageByPty,
|
||||
disposed: this.disposed,
|
||||
canReserve: (entry) => this.canReserve(entry.key, entry.charge),
|
||||
reserve: (entry) => this.reserveAndQueue(entry),
|
||||
isBelowGlobalLowWatermark: () =>
|
||||
this.globalSourceUnits <= this.limits.globalLowSourceUnits &&
|
||||
this.globalBytes <= this.limits.globalLowBytes
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer'
|
||||
import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputIntakeDependencies,
|
||||
type SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake'
|
||||
import type { SshPtySourceCancellationProof } from './ssh-pty-output-intake-contract'
|
||||
|
||||
type ExitDeadlineHarness = Readonly<{
|
||||
intake: SshPtyOutputIntake
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
cancelSourceDelivery: ReturnType<typeof vi.fn>
|
||||
releaseExit: ReturnType<typeof vi.fn>
|
||||
exits: string[]
|
||||
}>
|
||||
|
||||
const stream: RemoteTerminalSourceRangeStreamIdentity = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
|
||||
function createHarness(
|
||||
cancelSourceDelivery: NonNullable<SshPtyOutputIntakeDependencies['cancelSourceDelivery']>
|
||||
): ExitDeadlineHarness {
|
||||
const exits: string[] = []
|
||||
const releaseExit = vi.fn()
|
||||
const cancellation = vi.fn(cancelSourceDelivery)
|
||||
const dependencies: SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: () => 0,
|
||||
acceptModel: (event) => ({ sequence: event.rawLength, completion: Promise.resolve() }),
|
||||
project: vi.fn(),
|
||||
prepareExit: vi.fn(() => releaseExit),
|
||||
finalizeExit: () => exits.push('exit'),
|
||||
pauseProvider: vi.fn(() => true),
|
||||
resumeProvider: vi.fn(),
|
||||
closeProvider: vi.fn(),
|
||||
cancelSourceDelivery: cancellation
|
||||
}
|
||||
return {
|
||||
intake: new SshPtyOutputIntake(dependencies, {
|
||||
exitBarrierMs: 10,
|
||||
exitCancellationProofMs: 100
|
||||
}),
|
||||
dependencies,
|
||||
cancelSourceDelivery: cancellation,
|
||||
releaseExit,
|
||||
exits
|
||||
}
|
||||
}
|
||||
|
||||
async function publishSource(harness: ExitDeadlineHarness): Promise<SshPtyOutputReceipt> {
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
expect(remote.attach(stream)).toBe(true)
|
||||
const receipt = await harness.intake.acceptData({
|
||||
id: 'pty-1',
|
||||
data: 'aaaa',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
harness.intake.publishProjectionPrefix([receipt.projection.identity.projectionSemanticsId], 4, 4)
|
||||
return receipt
|
||||
}
|
||||
|
||||
function acceptExit(harness: ExitDeadlineHarness) {
|
||||
return harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
}
|
||||
|
||||
describe('SshPtyOutputExitDeadline', () => {
|
||||
it('transfers published projections before cancellation proof reclaims their spans', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const harness = createHarness(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const receipt = await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
|
||||
expect(await exitResult).toEqual({ ok: true })
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.dependencies.prepareExit).toHaveBeenCalledOnce()
|
||||
expect(harness.releaseExit).toHaveBeenCalledOnce()
|
||||
expect(harness.exits).toEqual(['exit'])
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(harness.exits).toEqual(['exit'])
|
||||
expect(() =>
|
||||
harness.intake
|
||||
.getRemoteSourceRangeConsumerHooks()
|
||||
.settle(stream, [receipt.projection.desktopSpan!])
|
||||
).not.toThrow()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('generation close fences a pending cancellation proof from final exit', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let resolveCancellation!: (proof: SshPtySourceCancellationProof) => void
|
||||
const cancellation = new Promise<SshPtySourceCancellationProof>((resolve) => {
|
||||
resolveCancellation = resolve
|
||||
})
|
||||
const harness = createHarness(() => cancellation)
|
||||
await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
const closed = await exitResult
|
||||
expect(closed.ok).toBe(false)
|
||||
expect(closed.ok ? '' : closed.error.message).toContain('provider-closed')
|
||||
resolveCancellation({ sentEndSu: 4, creditedEndSu: 0 })
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(harness.exits).toEqual([])
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,239 @@
|
||||
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import type {
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation'
|
||||
import type {
|
||||
SshPtyOutputSourceObligations,
|
||||
SshPtySourceCancellationProofCommit
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
|
||||
type SshPtyOutputExitDeadlineDependencies = Readonly<{
|
||||
admission: SshPtyModelAdmission
|
||||
projections: SshPtyLegacyProjectionLedger
|
||||
sourceObligations: SshPtyOutputSourceObligations
|
||||
intake: SshPtyOutputIntakeDependencies
|
||||
barrierMs?: number
|
||||
cancellationProofMs?: number
|
||||
}>
|
||||
|
||||
export class SshPtyOutputExitDeadline {
|
||||
private readonly barriersByGeneration = new Map<number, Set<SshPtyExitBarrier>>()
|
||||
private readonly preparedExits = new Set<string>()
|
||||
private readonly preparedExitReleases = new Map<string, () => void>()
|
||||
private readonly barrierMs: number
|
||||
private readonly cancellationProofMs: number
|
||||
|
||||
constructor(private readonly dependencies: SshPtyOutputExitDeadlineDependencies) {
|
||||
this.barrierMs = dependencies.barrierMs ?? 30_000
|
||||
this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000
|
||||
}
|
||||
|
||||
wait(
|
||||
event: SshPtyOutputExitEvent,
|
||||
start: (validateNormalExit: () => void) => Promise<void>
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeoutStarted = false
|
||||
let settled = false
|
||||
const validateNormalExit = (): void => {
|
||||
if (timeoutStarted) {
|
||||
throw outputIntakeError('ssh_exit_delivery_canceled')
|
||||
}
|
||||
}
|
||||
const settle = (result: { ok: true } | { ok: false; error: Error }): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearTimeout(barrier.timer)
|
||||
this.releasePreparedExit(event)
|
||||
this.remove(event.providerGeneration, barrier)
|
||||
if (result.ok) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(result.error)
|
||||
}
|
||||
}
|
||||
const barrier: SshPtyExitBarrier = {
|
||||
timer: setTimeout(() => {
|
||||
timeoutStarted = true
|
||||
void this.cancelTimedOutExit(event, barrier).then(
|
||||
() => settle({ ok: true }),
|
||||
(error) => {
|
||||
this.dependencies.intake.closeProvider?.(
|
||||
event.providerGeneration,
|
||||
'ssh-exit-cancellation-proof-failed'
|
||||
)
|
||||
settle({
|
||||
ok: false,
|
||||
error: error instanceof Error ? error : outputIntakeError(String(error))
|
||||
})
|
||||
}
|
||||
)
|
||||
}, this.barrierMs),
|
||||
reject: (error) => settle({ ok: false, error })
|
||||
}
|
||||
barrier.timer.unref?.()
|
||||
let barriers = this.barriersByGeneration.get(event.providerGeneration)
|
||||
if (!barriers) {
|
||||
barriers = new Set()
|
||||
this.barriersByGeneration.set(event.providerGeneration, barriers)
|
||||
}
|
||||
barriers.add(barrier)
|
||||
const promise = start(validateNormalExit)
|
||||
void promise.then(
|
||||
() => {
|
||||
if (!timeoutStarted) {
|
||||
settle({ ok: true })
|
||||
}
|
||||
},
|
||||
(error) => {
|
||||
if (!timeoutStarted) {
|
||||
settle({ ok: false, error })
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, error: Error): void {
|
||||
const barriers = this.barriersByGeneration.get(providerGeneration)
|
||||
if (barriers) {
|
||||
for (const barrier of barriers) {
|
||||
clearTimeout(barrier.timer)
|
||||
barrier.reject(error)
|
||||
}
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.preparedExits) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.releasePreparedExitKey(key)
|
||||
this.preparedExits.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prepareExitOnce(event: SshPtyOutputExitEvent): void {
|
||||
const key = this.exitKey(event)
|
||||
if (this.preparedExits.has(key)) {
|
||||
return
|
||||
}
|
||||
const release = this.dependencies.intake.prepareExit(event)
|
||||
this.preparedExits.add(key)
|
||||
if (release) {
|
||||
this.preparedExitReleases.set(key, release)
|
||||
}
|
||||
}
|
||||
|
||||
get activeBarriers(): number {
|
||||
return Array.from(this.barriersByGeneration.values()).reduce(
|
||||
(total, barriers) => total + barriers.size,
|
||||
0
|
||||
)
|
||||
}
|
||||
|
||||
private async cancelTimedOutExit(
|
||||
event: SshPtyOutputExitEvent,
|
||||
barrier: SshPtyExitBarrier
|
||||
): Promise<void> {
|
||||
const cancel = this.dependencies.intake.cancelSourceDelivery
|
||||
if (!cancel) {
|
||||
throw outputIntakeError('ssh_source_cancellation_publisher_unavailable')
|
||||
}
|
||||
this.dependencies.admission.cancelPty(
|
||||
{ ptyId: event.id, providerGeneration: event.providerGeneration },
|
||||
'ssh_exit_delivery_canceled'
|
||||
)
|
||||
this.dependencies.sourceObligations.sealPty(event)
|
||||
const cancellation = this.dependencies.sourceObligations.requestPtyCancellationProof(
|
||||
event,
|
||||
(request) => cancel(event.providerGeneration, request)
|
||||
)
|
||||
let commit: SshPtySourceCancellationProofCommit | null
|
||||
try {
|
||||
commit = await this.withCancellationProofDeadline(cancellation)
|
||||
} catch (error) {
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
if (!commit) {
|
||||
throw outputIntakeError('ssh_source_cancellation_identity_unavailable')
|
||||
}
|
||||
this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled')
|
||||
this.dependencies.sourceObligations.commitPtyCancellationProof(commit)
|
||||
this.prepareExitOnce(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.intake.finalizeExit(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'ssh-exit-delivery-canceled'
|
||||
)
|
||||
}
|
||||
|
||||
private isActive(providerGeneration: number, barrier: SshPtyExitBarrier): boolean {
|
||||
return this.barriersByGeneration.get(providerGeneration)?.has(barrier) ?? false
|
||||
}
|
||||
|
||||
private exitKey(event: SshPtyOutputExitEvent): string {
|
||||
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
|
||||
}
|
||||
|
||||
private releasePreparedExit(event: SshPtyOutputExitEvent): void {
|
||||
const key = this.exitKey(event)
|
||||
this.preparedExits.delete(key)
|
||||
this.releasePreparedExitKey(key)
|
||||
}
|
||||
|
||||
private releasePreparedExitKey(key: string): void {
|
||||
const release = this.preparedExitReleases.get(key)
|
||||
if (!release) {
|
||||
return
|
||||
}
|
||||
this.preparedExitReleases.delete(key)
|
||||
release()
|
||||
}
|
||||
|
||||
private withCancellationProofDeadline<T>(promise: Promise<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(outputIntakeError('ssh_source_cancellation_proof_timeout')),
|
||||
this.cancellationProofMs
|
||||
)
|
||||
timer.unref?.()
|
||||
void promise.then(
|
||||
(value) => {
|
||||
clearTimeout(timer)
|
||||
resolve(value)
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
private remove(providerGeneration: number, barrier: SshPtyExitBarrier): void {
|
||||
const barriers = this.barriersByGeneration.get(providerGeneration)
|
||||
barriers?.delete(barrier)
|
||||
if (barriers?.size === 0) {
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import type {
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
|
||||
export async function settleSshPtyOutputExit(args: {
|
||||
event: SshPtyOutputExitEvent
|
||||
admission: SshPtyModelAdmission
|
||||
projections: SshPtyLegacyProjectionLedger
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
validateGeneration: () => void
|
||||
prepareExit?: () => void
|
||||
afterAdmissionIdle?: () => void
|
||||
waitForSourceTerminal?: () => Promise<void>
|
||||
beforeFinalize?: () => void
|
||||
}): Promise<void> {
|
||||
const {
|
||||
event,
|
||||
admission,
|
||||
projections,
|
||||
dependencies,
|
||||
validateGeneration,
|
||||
prepareExit,
|
||||
afterAdmissionIdle,
|
||||
waitForSourceTerminal,
|
||||
beforeFinalize
|
||||
} = args
|
||||
await admission.whenIdle({
|
||||
ptyId: event.id,
|
||||
providerGeneration: event.providerGeneration
|
||||
})
|
||||
validateGeneration()
|
||||
afterAdmissionIdle?.()
|
||||
try {
|
||||
if (prepareExit) {
|
||||
prepareExit()
|
||||
} else {
|
||||
dependencies.prepareExit(event)
|
||||
}
|
||||
} catch (error) {
|
||||
projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-finalize-failed'
|
||||
)
|
||||
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
|
||||
throw error
|
||||
}
|
||||
projections.transferUnpublishedPty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-unpublished'
|
||||
)
|
||||
await projections.whenPtyTerminal(event.id, event.providerGeneration, event.ptyIncarnation)
|
||||
await waitForSourceTerminal?.()
|
||||
validateGeneration()
|
||||
try {
|
||||
beforeFinalize?.()
|
||||
dependencies.finalizeExit(event)
|
||||
projections.closePty(event.id, event.providerGeneration, event.ptyIncarnation, 'pty-exit')
|
||||
} catch (error) {
|
||||
projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-finalize-failed'
|
||||
)
|
||||
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
|
||||
|
||||
const event = (providerGeneration: number) => ({
|
||||
id: 'pty-1',
|
||||
providerGeneration,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
describe('SshPtyOutputGenerationGuard', () => {
|
||||
it('compacts sequential closures without weakening stale rejection', () => {
|
||||
const guard = new SshPtyOutputGenerationGuard(() => false)
|
||||
for (let generation = 1; generation <= 2_048; generation++) {
|
||||
guard.closeGeneration(generation)
|
||||
}
|
||||
|
||||
expect(guard.getDebugSnapshot()).toEqual({
|
||||
closedRanges: 1,
|
||||
activeGaps: 0,
|
||||
activePtys: 0,
|
||||
sealedPtys: 0
|
||||
})
|
||||
expect(() => guard.validate(event(1_024))).toThrow('ssh_output_stale_generation')
|
||||
expect(() => guard.validate(event(2_049))).not.toThrow()
|
||||
})
|
||||
|
||||
it('counts and merges out-of-order live generation gaps exactly', () => {
|
||||
const guard = new SshPtyOutputGenerationGuard(() => false)
|
||||
guard.closeGeneration(4)
|
||||
guard.closeGeneration(2)
|
||||
|
||||
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 2, activeGaps: 2 })
|
||||
guard.closeGeneration(3)
|
||||
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 1, activeGaps: 1 })
|
||||
expect(() => guard.validate(event(1))).not.toThrow()
|
||||
expect(() => guard.validate(event(2))).toThrow('ssh_output_stale_generation')
|
||||
|
||||
guard.closeGeneration(1)
|
||||
expect(guard.getDebugSnapshot()).toEqual({
|
||||
closedRanges: 1,
|
||||
activeGaps: 0,
|
||||
activePtys: 0,
|
||||
sealedPtys: 0
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent } from './ssh-pty-output-intake-contract'
|
||||
import {
|
||||
outputIntakeError,
|
||||
sshPtyGenerationKey,
|
||||
validOutputLength
|
||||
} from './ssh-pty-output-intake-validation'
|
||||
import { SshPtyClosedGenerationRanges } from './ssh-pty-closed-generation-ranges'
|
||||
|
||||
export class SshPtyOutputGenerationGuard {
|
||||
private readonly latestGenerationByPty = new Map<string, number>()
|
||||
private readonly incarnationByPty = new Map<string, string>()
|
||||
private readonly sealedPtys = new Set<string>()
|
||||
private readonly closedGenerations = new SshPtyClosedGenerationRanges()
|
||||
|
||||
constructor(private readonly isDisposed: () => boolean) {}
|
||||
|
||||
validateData(event: SshPtyOutputDataEvent): void {
|
||||
if (
|
||||
!event.id ||
|
||||
!event.ptyIncarnation ||
|
||||
!Number.isSafeInteger(event.providerGeneration) ||
|
||||
event.providerGeneration <= 0 ||
|
||||
!validOutputLength(event.rawLength) ||
|
||||
(event.source !== undefined &&
|
||||
(!event.source.spanId ||
|
||||
!event.source.deliveryToken ||
|
||||
!Number.isSafeInteger(event.source.clientGeneration) ||
|
||||
event.source.clientGeneration <= 0 ||
|
||||
!Number.isSafeInteger(event.source.ownerGeneration) ||
|
||||
event.source.ownerGeneration <= 0 ||
|
||||
!Number.isSafeInteger(event.source.sourceStartSu) ||
|
||||
event.source.sourceStartSu < 0 ||
|
||||
!Number.isSafeInteger(event.source.sourceEndSu) ||
|
||||
event.source.sourceEndSu - event.source.sourceStartSu !== event.rawLength))
|
||||
) {
|
||||
throw outputIntakeError('ssh_output_invalid_event')
|
||||
}
|
||||
this.validate(event)
|
||||
if (this.sealedPtys.has(sshPtyGenerationKey(event.id, event.providerGeneration))) {
|
||||
throw outputIntakeError('ssh_output_after_exit')
|
||||
}
|
||||
}
|
||||
|
||||
sealExit(event: SshPtyOutputExitEvent): void {
|
||||
this.validate(event)
|
||||
const key = sshPtyGenerationKey(event.id, event.providerGeneration)
|
||||
if (this.sealedPtys.has(key)) {
|
||||
throw outputIntakeError('ssh_output_duplicate_exit')
|
||||
}
|
||||
this.sealedPtys.add(key)
|
||||
}
|
||||
|
||||
validate(event: { id: string; providerGeneration: number; ptyIncarnation: string }): void {
|
||||
if (this.isDisposed()) {
|
||||
throw outputIntakeError('ssh_output_intake_disposed')
|
||||
}
|
||||
if (this.closedGenerations.has(event.providerGeneration)) {
|
||||
throw outputIntakeError('ssh_output_stale_generation')
|
||||
}
|
||||
const generation = this.latestGenerationByPty.get(event.id)
|
||||
if (generation !== undefined && event.providerGeneration < generation) {
|
||||
throw outputIntakeError('ssh_output_stale_generation')
|
||||
}
|
||||
const incarnation = this.incarnationByPty.get(event.id)
|
||||
if (
|
||||
generation === event.providerGeneration &&
|
||||
incarnation !== undefined &&
|
||||
incarnation !== event.ptyIncarnation
|
||||
) {
|
||||
throw outputIntakeError('ssh_output_stale_incarnation')
|
||||
}
|
||||
if (generation === undefined || event.providerGeneration > generation) {
|
||||
this.latestGenerationByPty.set(event.id, event.providerGeneration)
|
||||
this.incarnationByPty.set(event.id, event.ptyIncarnation)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number): void {
|
||||
this.closedGenerations.add(providerGeneration)
|
||||
for (const [ptyId, generation] of this.latestGenerationByPty) {
|
||||
if (generation === providerGeneration) {
|
||||
this.latestGenerationByPty.delete(ptyId)
|
||||
this.incarnationByPty.delete(ptyId)
|
||||
this.sealedPtys.delete(sshPtyGenerationKey(ptyId, generation))
|
||||
}
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.sealedPtys) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.sealedPtys.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
activeGenerations(): ReadonlySet<number> {
|
||||
return new Set(this.latestGenerationByPty.values())
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return {
|
||||
closedRanges: this.closedGenerations.size,
|
||||
activeGaps: this.closedGenerations.activeGaps,
|
||||
activePtys: this.latestGenerationByPty.size,
|
||||
sealedPtys: this.sealedPtys.size
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import type {
|
||||
SshPtyModelAdmissionOptions,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
|
||||
export type SshPtySourceCancellationRequest = Readonly<{
|
||||
id: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
}>
|
||||
|
||||
export type SshPtySourceCancellationProof = Readonly<{
|
||||
sentEndSu: number
|
||||
creditedEndSu: number
|
||||
}>
|
||||
|
||||
export type SshPtyOutputDataEvent = Readonly<{
|
||||
id: string
|
||||
data: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
sequence?: number
|
||||
source?: Readonly<{
|
||||
relayPtyId?: string
|
||||
spanId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
sourceStartSu: number
|
||||
sourceEndSu: number
|
||||
}>
|
||||
}>
|
||||
|
||||
export type SshPtyOutputExitEvent = Readonly<{
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}>
|
||||
|
||||
export type SshPtyOutputReceipt = SshPtyModelAdmissionReceipt &
|
||||
Readonly<{ projection: LegacySshProjectionSemantics }>
|
||||
|
||||
export type SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: (id: string) => number
|
||||
acceptModel: (
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: LegacySshProjectionSemantics
|
||||
) => { sequence: number; completion: Promise<void> }
|
||||
project: (event: SshPtyOutputDataEvent, projection: LegacySshProjectionSemantics) => void
|
||||
prepareExit: (event: SshPtyOutputExitEvent) => void | (() => void)
|
||||
finalizeExit: (event: SshPtyOutputExitEvent) => void
|
||||
pauseProvider?: (providerGeneration: number, id: string) => boolean
|
||||
resumeProvider?: (providerGeneration: number, id: string) => void
|
||||
closeProvider?: (providerGeneration: number, reason: string) => void
|
||||
resetModelForMigration?: (providerGeneration: number, id: string) => void
|
||||
onGenerationClosed?: (providerGeneration: number, reason: string) => void
|
||||
publishSourceAck?: (
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
) => void
|
||||
cancelSourceDelivery?: (
|
||||
providerGeneration: number,
|
||||
request: SshPtySourceCancellationRequest
|
||||
) => Promise<SshPtySourceCancellationProof>
|
||||
}
|
||||
|
||||
export type SshPtyOutputIntakeOptions = SshPtyModelAdmissionOptions & {
|
||||
exitBarrierMs?: number
|
||||
exitCancellationProofMs?: number
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputReceipt,
|
||||
SshPtySourceCancellationProof,
|
||||
SshPtySourceCancellationRequest
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import type { SshPtyAcceptedSourceCheckpoint } from './ssh-pty-output-source-obligations'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtyOutputGenerationMigration } from './ssh-pty-output-model-migration'
|
||||
|
||||
let installedIntake: SshPtyOutputIntake | null = null
|
||||
let nextProviderGeneration = 1
|
||||
const sourceAckPublishers = new Map<number, SshPtySourceAckPublisher>()
|
||||
const sourceCancellationPublishers = new Map<number, SshPtySourceCancellationPublisher>()
|
||||
|
||||
type SshPtySourceAckPublisher = (
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
) => void
|
||||
|
||||
type SshPtySourceCancellationPublisher = (
|
||||
request: SshPtySourceCancellationRequest
|
||||
) => Promise<SshPtySourceCancellationProof>
|
||||
|
||||
export function allocateSshPtyProviderGeneration(): number {
|
||||
return nextProviderGeneration++
|
||||
}
|
||||
|
||||
export function installSshPtyOutputIntake(intake: SshPtyOutputIntake): () => void {
|
||||
const previous = installedIntake
|
||||
installedIntake = intake
|
||||
previous?.dispose()
|
||||
return () => {
|
||||
if (installedIntake === intake) {
|
||||
installedIntake = null
|
||||
intake.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function acceptSshPtyOutputData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
|
||||
return installedIntake
|
||||
? installedIntake.acceptData(event)
|
||||
: Promise.reject(outputIntakeUnavailableError())
|
||||
}
|
||||
|
||||
export function acceptSshPtyOutputExit(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
return installedIntake
|
||||
? installedIntake.acceptExit(event)
|
||||
: Promise.reject(outputIntakeUnavailableError())
|
||||
}
|
||||
|
||||
export function closeSshPtyOutputGeneration(providerGeneration: number, reason: string): void {
|
||||
installedIntake?.closeGeneration(providerGeneration, reason)
|
||||
}
|
||||
|
||||
export function getSshPtyAcceptedSourceCheckpoints(
|
||||
providerGeneration: number
|
||||
): readonly SshPtyAcceptedSourceCheckpoint[] {
|
||||
return installedIntake?.getAcceptedSourceCheckpoints(providerGeneration) ?? []
|
||||
}
|
||||
|
||||
export function beginSshPtyOutputGenerationMigration(
|
||||
providerGeneration: number
|
||||
): SshPtyOutputGenerationMigration {
|
||||
return (
|
||||
installedIntake?.beginGenerationMigration(providerGeneration) ?? {
|
||||
byPty: new Map(),
|
||||
completion: Promise.resolve()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export function installSshPtySourceAckPublisher(
|
||||
providerGeneration: number,
|
||||
publish: SshPtySourceAckPublisher
|
||||
): () => void {
|
||||
if (sourceAckPublishers.has(providerGeneration)) {
|
||||
throw new Error('ssh_source_ack_publisher_duplicate_generation')
|
||||
}
|
||||
sourceAckPublishers.set(providerGeneration, publish)
|
||||
return () => {
|
||||
if (sourceAckPublishers.get(providerGeneration) === publish) {
|
||||
sourceAckPublishers.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function publishSshPtySourceAck(
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
): void {
|
||||
const publisher = sourceAckPublishers.get(providerGeneration)
|
||||
if (!publisher) {
|
||||
onSettled({ ok: false, error: new Error('ssh_source_ack_publisher_unavailable') })
|
||||
return
|
||||
}
|
||||
publisher(batch, onSettled)
|
||||
}
|
||||
|
||||
export function installSshPtySourceCancellationPublisher(
|
||||
providerGeneration: number,
|
||||
cancel: SshPtySourceCancellationPublisher
|
||||
): () => void {
|
||||
if (sourceCancellationPublishers.has(providerGeneration)) {
|
||||
throw new Error('ssh_source_cancellation_publisher_duplicate_generation')
|
||||
}
|
||||
sourceCancellationPublishers.set(providerGeneration, cancel)
|
||||
return () => {
|
||||
if (sourceCancellationPublishers.get(providerGeneration) === cancel) {
|
||||
sourceCancellationPublishers.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function cancelSshPtySourceDelivery(
|
||||
providerGeneration: number,
|
||||
request: SshPtySourceCancellationRequest
|
||||
): Promise<SshPtySourceCancellationProof> {
|
||||
const publisher = sourceCancellationPublishers.get(providerGeneration)
|
||||
return publisher
|
||||
? publisher(request)
|
||||
: Promise.reject(new Error('ssh_source_cancellation_publisher_unavailable'))
|
||||
}
|
||||
|
||||
export function applySshPtySourceCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
return installedIntake?.applySourceCancellationProof(event, proof) ?? false
|
||||
}
|
||||
|
||||
export function applySshPtySourceRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
if (!installedIntake) {
|
||||
return false
|
||||
}
|
||||
installedIntake.applySourceRecoveryCancellationProof(event, proof)
|
||||
return true
|
||||
}
|
||||
|
||||
function outputIntakeUnavailableError(): Error {
|
||||
return Object.assign(new Error('ssh_output_intake_unavailable'), {
|
||||
code: 'ssh_output_intake_unavailable'
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { vi } from 'vitest'
|
||||
import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputDataEvent,
|
||||
type SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake'
|
||||
|
||||
export function sshPtyOutputEvent(
|
||||
overrides: Partial<SshPtyOutputDataEvent> = {}
|
||||
): SshPtyOutputDataEvent {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
data: 'aaaa',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
export function createSshPtyOutputIntakeHarness(
|
||||
overrides: Partial<SshPtyOutputIntakeDependencies> = {},
|
||||
options: ConstructorParameters<typeof SshPtyOutputIntake>[1] = {}
|
||||
) {
|
||||
let sequence = 0
|
||||
const completions: ReturnType<typeof deferred>[] = []
|
||||
const order: string[] = []
|
||||
const dependencies: SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: () => sequence,
|
||||
acceptModel: (input) => {
|
||||
order.push(`model:${input.data}`)
|
||||
sequence += input.rawLength
|
||||
const completion = deferred()
|
||||
completions.push(completion)
|
||||
return { sequence, completion: completion.promise }
|
||||
},
|
||||
project: (input) => order.push(`project:${input.data}`),
|
||||
prepareExit: vi.fn(),
|
||||
finalizeExit: () => order.push('exit'),
|
||||
pauseProvider: vi.fn(() => true),
|
||||
resumeProvider: vi.fn(),
|
||||
closeProvider: vi.fn(),
|
||||
...overrides
|
||||
}
|
||||
return {
|
||||
intake: new SshPtyOutputIntake(dependencies, options),
|
||||
dependencies,
|
||||
completions,
|
||||
order
|
||||
}
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void
|
||||
let reject!: (error: Error) => void
|
||||
const promise = new Promise<void>((promiseResolve, promiseReject) => {
|
||||
resolve = promiseResolve
|
||||
reject = promiseReject
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export type SshPtyExitBarrier = {
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
reject: (error: Error) => void
|
||||
}
|
||||
|
||||
export function outputIntakeError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function validOutputLength(value: number): boolean {
|
||||
return Number.isSafeInteger(value) && value >= 0
|
||||
}
|
||||
|
||||
export function sshPtyGenerationKey(ptyId: string, providerGeneration: number): string {
|
||||
return `${providerGeneration}\0${ptyId}`
|
||||
}
|
||||
@@ -0,0 +1,760 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
describe('SshPtyOutputIntake', () => {
|
||||
it('plateaus at the model and pressure budgets, then resumes below low water', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
perPtyLowSourceUnits: 1,
|
||||
perPtyLowBytes: 256,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
globalLowSourceUnits: 1,
|
||||
globalLowBytes: 256,
|
||||
pressureMaxFrames: 2,
|
||||
pressureMaxBytes: 1024
|
||||
}
|
||||
)
|
||||
const first = harness.intake.acceptData(event())
|
||||
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const third = harness.intake.acceptData(event({ data: 'cccc' }))
|
||||
const rejected = harness.intake.acceptData(event({ data: 'dddd' }))
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 4,
|
||||
pressureFrames: 2
|
||||
})
|
||||
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, expect.any(String))
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
harness.completions[1]!.resolve()
|
||||
await second
|
||||
harness.completions[2]!.resolve()
|
||||
await third
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
pressureFrames: 0
|
||||
})
|
||||
expect(harness.dependencies.resumeProvider).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves per-PTY FIFO while differently sized pressure entries wait', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 4096,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 4096,
|
||||
pressureMaxFrames: 4,
|
||||
pressureMaxBytes: 4096
|
||||
}
|
||||
)
|
||||
const receipts = [
|
||||
harness.intake.acceptData(event({ data: 'a', rawLength: 1 })),
|
||||
harness.intake.acceptData(event({ data: 'bbb', rawLength: 3 })),
|
||||
harness.intake.acceptData(event({ data: 'cc', rawLength: 2 })),
|
||||
harness.intake.acceptData(event({ data: 'd', rawLength: 1 }))
|
||||
]
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await receipts[0]
|
||||
harness.completions[1]!.resolve()
|
||||
await receipts[1]
|
||||
harness.completions[2]!.resolve()
|
||||
await receipts[2]
|
||||
harness.completions[3]!.resolve()
|
||||
await receipts[3]
|
||||
|
||||
expect(harness.order).toEqual([
|
||||
'model:a',
|
||||
'project:a',
|
||||
'model:bbb',
|
||||
'project:bbb',
|
||||
'model:cc',
|
||||
'project:cc',
|
||||
'model:d',
|
||||
'project:d'
|
||||
])
|
||||
})
|
||||
|
||||
it('assigns projection sequence ends when queued model capture begins', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 12,
|
||||
perPtyHighBytes: 4096,
|
||||
globalHighSourceUnits: 12,
|
||||
globalHighBytes: 4096
|
||||
}
|
||||
)
|
||||
const first = harness.intake.acceptData(event({ data: 'aaaa' }))
|
||||
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const third = harness.intake.acceptData(event({ data: 'cccc' }))
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
expect((await first).projection.identity.sequenceEnd).toBe(4)
|
||||
harness.completions[1]!.resolve()
|
||||
expect((await second).projection.identity.sequenceEnd).toBe(8)
|
||||
harness.completions[2]!.resolve()
|
||||
expect((await third).projection.identity.sequenceEnd).toBe(12)
|
||||
})
|
||||
|
||||
it('transfers projection state and closes the provider on model failure', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(receipt).rejects.toThrow('emulator failed')
|
||||
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
|
||||
})
|
||||
|
||||
it('transfers a committed projection when desktop admission throws', async () => {
|
||||
const projections: LegacySshProjectionSemantics[] = []
|
||||
const harness = createHarness({
|
||||
project: (_event, projection) => {
|
||||
projections.push(projection)
|
||||
if (projections.length === 1) {
|
||||
throw new Error('send failed')
|
||||
}
|
||||
}
|
||||
})
|
||||
const receipt = harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5 }))
|
||||
harness.completions[0]!.resolve()
|
||||
|
||||
await expect(receipt).resolves.toMatchObject({ sequence: 5 })
|
||||
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
|
||||
|
||||
const next = harness.intake.acceptData(event({ data: '31h', rawLength: 3 }))
|
||||
harness.completions[1]!.resolve()
|
||||
await next
|
||||
expect(projections[1]).toMatchObject({
|
||||
identity: { displayStart: 5 },
|
||||
beforeScanner: { tail: '\x1b[?20', pendingSubscribe: false },
|
||||
decision: 'subscribed'
|
||||
})
|
||||
})
|
||||
|
||||
it('commits an immutable desktop source identity through projection admission', async () => {
|
||||
const projections: LegacySshProjectionSemantics[] = []
|
||||
const harness = createHarness({
|
||||
project: (_event, projection) => projections.push(projection)
|
||||
})
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
|
||||
await receipt
|
||||
expect(projections[0]?.desktopSpan).toMatchObject({
|
||||
spanId: 'span-1',
|
||||
projectionSemanticsId: projections[0]?.identity.projectionSemanticsId,
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
|
||||
})
|
||||
expect(Object.isFrozen(projections[0]?.desktopSpan)).toBe(true)
|
||||
})
|
||||
|
||||
it('exports only the model-settled source boundary before generation close', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'recovery-span-1',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(10)
|
||||
harness.completions[0]!.resolve()
|
||||
await receipt
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual([
|
||||
{
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'delivery-token-1',
|
||||
acceptedSourceEndSu: 14
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('rolls back projection staging when model capture throws synchronously', async () => {
|
||||
const project = vi.fn()
|
||||
const harness = createHarness({
|
||||
acceptModel: () => {
|
||||
throw new Error('model reservation failed')
|
||||
},
|
||||
project
|
||||
})
|
||||
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('model reservation failed')
|
||||
expect(project).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
|
||||
rolledBack: 1,
|
||||
records: 0
|
||||
})
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
|
||||
})
|
||||
|
||||
it('rolls back projection staging when source reservation validation fails', async () => {
|
||||
const harness = createHarness()
|
||||
const source = {
|
||||
spanId: 'duplicate',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
const first = harness.intake.acceptData(event({ source }))
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
await expect(
|
||||
harness.intake.acceptData(
|
||||
event({
|
||||
source: { ...source, sourceStartSu: 4, sourceEndSu: 8 }
|
||||
})
|
||||
)
|
||||
).rejects.toThrow('duplicate')
|
||||
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
|
||||
rolledBack: 1,
|
||||
records: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls back committed source and scanner facts when model capture throws', async () => {
|
||||
let attempts = 0
|
||||
const harness = createHarness({
|
||||
acceptModel: (accepted) => {
|
||||
attempts++
|
||||
if (attempts === 1) {
|
||||
throw new Error('model reservation failed')
|
||||
}
|
||||
return {
|
||||
sequence: accepted.rawLength,
|
||||
completion: Promise.resolve()
|
||||
}
|
||||
}
|
||||
})
|
||||
const source = {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
}
|
||||
await expect(
|
||||
harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5, source }))
|
||||
).rejects.toThrow('model reservation failed')
|
||||
|
||||
const retry = await harness.intake.acceptData(
|
||||
event({ data: '\x1b[?20', rawLength: 5, source: { ...source, spanId: 'span-2' } })
|
||||
)
|
||||
expect(retry.projection.identity.displayStart).toBe(0)
|
||||
expect(retry.projection.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
})
|
||||
|
||||
it('rejects stale provider generations without model capture', async () => {
|
||||
const harness = createHarness()
|
||||
const current = harness.intake.acceptData(event({ providerGeneration: 2 }))
|
||||
harness.completions[0]!.resolve()
|
||||
await current
|
||||
|
||||
await expect(harness.intake.acceptData(event({ providerGeneration: 1 }))).rejects.toThrow(
|
||||
'ssh_output_stale_generation'
|
||||
)
|
||||
expect(harness.completions).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('keeps exit behind accepted model and projection work', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
const exitReceipt = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa'])
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.all([dataReceipt, exitReceipt])
|
||||
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa', 'exit'])
|
||||
})
|
||||
|
||||
it('admits queued pre-exit source spans before sealing the token', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const first = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const second = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'bbbb',
|
||||
source: {
|
||||
spanId: 'span-2',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
harness.completions[1]!.resolve()
|
||||
await Promise.all([second, exit])
|
||||
expect(harness.order).toEqual([
|
||||
'model:aaaa',
|
||||
'project:aaaa',
|
||||
'model:bbbb',
|
||||
'project:bbbb',
|
||||
'exit'
|
||||
])
|
||||
})
|
||||
|
||||
it('does not pump queued model work after a running completion fails', async () => {
|
||||
const harness = createHarness()
|
||||
const first = harness.intake.acceptData(event({ data: 'first' }))
|
||||
const queued = harness.intake.acceptData(event({ data: 'queued' }))
|
||||
|
||||
harness.completions[0]!.reject(new Error('emulator failed'))
|
||||
await expect(first).rejects.toThrow('emulator failed')
|
||||
await expect(queued).rejects.toThrow('ssh_model_admission_completion_failed')
|
||||
expect(harness.order).toEqual(['model:first', 'project:first'])
|
||||
expect(harness.completions).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('retains exited delivery until published renderer projections settle', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
|
||||
let exited = false
|
||||
const exitReceipt = harness.intake
|
||||
.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
.then(() => {
|
||||
exited = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
expect(exited).toBe(false)
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
|
||||
expect(harness.intake.settleProjectionPrefix('pty-1', 4)).toBe(4)
|
||||
await exitReceipt
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
|
||||
})
|
||||
|
||||
it('owns renderer exit preparation through finalization and duplicate rejection', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: vi.fn(() => releaseRendererExit)
|
||||
})
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
const exitEvent = {
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
}
|
||||
const exit = harness.intake.acceptExit(exitEvent)
|
||||
await Promise.resolve()
|
||||
|
||||
expect(releaseRendererExit).not.toHaveBeenCalled()
|
||||
await expect(harness.intake.acceptExit(exitEvent)).rejects.toThrow('ssh_output_duplicate_exit')
|
||||
expect(releaseRendererExit).not.toHaveBeenCalled()
|
||||
|
||||
harness.intake.settleProjectionPrefix('pty-1', 4)
|
||||
await exit
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('releases renderer exit preparation when generation close aborts finalization', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: vi.fn(() => releaseRendererExit)
|
||||
})
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-replaced')
|
||||
|
||||
await expect(exit).rejects.toThrow('provider-replaced')
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
expect(harness.order).not.toContain('exit')
|
||||
})
|
||||
|
||||
it('retains exit until a required remote source consumer settles', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
const stream = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
expect(remote.attach(stream)).toBe(true)
|
||||
const dataReceipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
|
||||
let exited = false
|
||||
const exitReceipt = harness.intake
|
||||
.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
.then(() => {
|
||||
exited = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(exited).toBe(false)
|
||||
|
||||
remote.settle(stream, [receipt.projection.desktopSpan!])
|
||||
await exitReceipt
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps timed-out exit projections until generation-close proof', async () => {
|
||||
const harness = createHarness(
|
||||
{
|
||||
cancelSourceDelivery: () => Promise.reject(new Error('cancel transport failed'))
|
||||
},
|
||||
{ exitBarrierMs: 1, exitCancellationProofMs: 10 }
|
||||
)
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('ssh_source_cancellation_identity_unavailable')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(
|
||||
1,
|
||||
'ssh-exit-cancellation-proof-failed'
|
||||
)
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
|
||||
})
|
||||
|
||||
it('cancels only the timed-out source delivery and keeps the provider usable', async () => {
|
||||
const cancelSourceDelivery = vi.fn(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const harness = createHarness(
|
||||
{ cancelSourceDelivery },
|
||||
{ exitBarrierMs: 1, exitCancellationProofMs: 100 }
|
||||
)
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
const stream = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
remote.attach(stream)
|
||||
const dataReceipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await dataReceipt
|
||||
|
||||
await harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
expect(cancelSourceDelivery).toHaveBeenCalledWith(1, {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
|
||||
const sibling = harness.intake.acceptData(
|
||||
event({ id: 'pty-2', ptyIncarnation: 'incarnation-2' })
|
||||
)
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' })
|
||||
})
|
||||
|
||||
it('accepts recovery cancellation proof before any replacement span is admitted', () => {
|
||||
const harness = createHarness()
|
||||
|
||||
expect(() =>
|
||||
harness.intake.applySourceRecoveryCancellationProof(
|
||||
{
|
||||
id: 'pty-1',
|
||||
code: -1,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
},
|
||||
{ sentEndSu: 8, creditedEndSu: 4 }
|
||||
)
|
||||
).not.toThrow()
|
||||
expect(harness.intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('reclaims a partially admitted recovery prefix from authoritative proof', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'recovery-span',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'recovery-token',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await receipt
|
||||
|
||||
harness.intake.applySourceRecoveryCancellationProof(
|
||||
{
|
||||
id: 'pty-1',
|
||||
code: -1,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
},
|
||||
{ sentEndSu: 12, creditedEndSu: 4 }
|
||||
)
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects late same-generation data after ordered exit cleanup', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const first = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
await harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_after_exit')
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('ssh_output_duplicate_exit')
|
||||
|
||||
const next = harness.intake.acceptData(
|
||||
event({ providerGeneration: 2, ptyIncarnation: 'incarnation-2' })
|
||||
)
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(next).resolves.toMatchObject({
|
||||
projection: { identity: { ptyIncarnation: 'incarnation-2', displayStart: 0 } }
|
||||
})
|
||||
})
|
||||
|
||||
it('closes the provider when exit finalization fails', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: () => releaseRendererExit,
|
||||
finalizeExit: () => {
|
||||
throw new Error('renderer exit send failed')
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('renderer exit send failed')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'pty-exit-finalize-failed')
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('cancels queued work and exit waiters on generation cleanup', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 8,
|
||||
perPtyHighBytes: 2048,
|
||||
globalHighSourceUnits: 8,
|
||||
globalHighBytes: 2048
|
||||
}
|
||||
)
|
||||
const running = harness.intake.acceptData(event())
|
||||
const queued = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
|
||||
await expect(queued).rejects.toThrow('provider-closed')
|
||||
await expect(exit).rejects.toThrow('provider-closed')
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_stale_generation')
|
||||
await expect(running).rejects.toThrow('provider-closed')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(harness.intake.getDebugSnapshot().exitBarriers).toBe(0)
|
||||
})
|
||||
|
||||
it('resumes a paused provider during generation cleanup', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
pressureMaxFrames: 2,
|
||||
pressureMaxBytes: 1024
|
||||
}
|
||||
)
|
||||
const running = harness.intake.acceptData(event())
|
||||
const pressured = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
|
||||
await expect(pressured).rejects.toThrow('provider-closed')
|
||||
expect(harness.dependencies.resumeProvider).toHaveBeenCalledWith(1, 'pty-1')
|
||||
await expect(running).rejects.toThrow('provider-closed')
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,296 @@
|
||||
import {
|
||||
SshPtyLegacyProjectionLedger,
|
||||
type LegacySshProjectionSemantics
|
||||
} from './ssh-pty-legacy-projection'
|
||||
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import { SshPtyOutputExitDeadline } from './ssh-pty-output-exit-deadline'
|
||||
import { settleSshPtyOutputExit } from './ssh-pty-output-exit'
|
||||
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
|
||||
import {
|
||||
SshPtyOutputModelMigration,
|
||||
type SshPtyOutputGenerationMigration,
|
||||
type SshPtyTrackedModelAdmission
|
||||
} from './ssh-pty-output-model-migration'
|
||||
import {
|
||||
SshPtyOutputSourceObligations,
|
||||
type SshPtyOutputSourceReservation
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtyOutputIntakeOptions,
|
||||
SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { outputIntakeError } from './ssh-pty-output-intake-validation'
|
||||
|
||||
export type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtyOutputIntakeOptions,
|
||||
SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
|
||||
export class SshPtyOutputIntake {
|
||||
private readonly projections: SshPtyLegacyProjectionLedger
|
||||
private readonly sourceObligations: SshPtyOutputSourceObligations
|
||||
private readonly generationGuard = new SshPtyOutputGenerationGuard(() => this.disposed)
|
||||
private readonly admission: SshPtyModelAdmission
|
||||
private readonly modelMigration: SshPtyOutputModelMigration
|
||||
private readonly exitDeadline: SshPtyOutputExitDeadline
|
||||
private disposed = false
|
||||
|
||||
constructor(
|
||||
private readonly dependencies: SshPtyOutputIntakeDependencies,
|
||||
options: SshPtyOutputIntakeOptions = {}
|
||||
) {
|
||||
this.sourceObligations = new SshPtyOutputSourceObligations(dependencies.publishSourceAck)
|
||||
this.projections = new SshPtyLegacyProjectionLedger({
|
||||
onSettled: (span) => this.sourceObligations.settleDesktop(span, 'renderer-parse'),
|
||||
onTransferred: (span, reason) => this.sourceObligations.transferDesktop(span, reason)
|
||||
})
|
||||
this.admission = new SshPtyModelAdmission({
|
||||
...options,
|
||||
pauseProvider: (key) =>
|
||||
this.dependencies.pauseProvider?.(key.providerGeneration, key.ptyId) ?? false,
|
||||
resumeProvider: (key) =>
|
||||
this.dependencies.resumeProvider?.(key.providerGeneration, key.ptyId),
|
||||
closeProvider: (providerGeneration, reason) =>
|
||||
this.dependencies.closeProvider?.(providerGeneration, reason)
|
||||
})
|
||||
this.modelMigration = new SshPtyOutputModelMigration(
|
||||
this.admission,
|
||||
this.sourceObligations,
|
||||
(providerGeneration, ptyId) =>
|
||||
this.dependencies.resetModelForMigration?.(providerGeneration, ptyId)
|
||||
)
|
||||
this.exitDeadline = new SshPtyOutputExitDeadline({
|
||||
admission: this.admission,
|
||||
projections: this.projections,
|
||||
sourceObligations: this.sourceObligations,
|
||||
intake: this.dependencies,
|
||||
barrierMs: options.exitBarrierMs,
|
||||
cancellationProofMs: options.exitCancellationProofMs
|
||||
})
|
||||
}
|
||||
|
||||
acceptData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
|
||||
try {
|
||||
this.generationGuard.validateData(event)
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
}
|
||||
let projection: LegacySshProjectionSemantics | undefined
|
||||
let sourceReservation: SshPtyOutputSourceReservation | undefined
|
||||
const key = { ptyId: event.id, providerGeneration: event.providerGeneration }
|
||||
const tracked: SshPtyTrackedModelAdmission = { key, started: false }
|
||||
const receipt = this.admission.accept(key, event.data, event.rawLength, () => {
|
||||
tracked.started = true
|
||||
const expectedSequence = this.dependencies.getModelSequence(event.id) + event.rawLength
|
||||
const reservation = this.projections.reserve({
|
||||
ptyId: event.id,
|
||||
providerGeneration: event.providerGeneration,
|
||||
ptyIncarnation: event.ptyIncarnation,
|
||||
data: event.data,
|
||||
sequenceEnd: expectedSequence,
|
||||
rawLength: event.rawLength,
|
||||
transformed: event.transformed,
|
||||
source: event.source
|
||||
})
|
||||
try {
|
||||
if (reservation.semantics.desktopSpan) {
|
||||
sourceReservation = this.sourceObligations.reserve(
|
||||
event,
|
||||
reservation.semantics.desktopSpan
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
this.projections.rollback(reservation)
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
projection = this.projections.commit(reservation)
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.commit(
|
||||
sourceReservation,
|
||||
event.id,
|
||||
projection.identity.sequenceEnd
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.rollback(sourceReservation)
|
||||
}
|
||||
if (!this.projections.rollbackCommitted(reservation)) {
|
||||
this.projections.rollback(reservation)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
let model: { sequence: number; completion: Promise<void> }
|
||||
try {
|
||||
model = this.dependencies.acceptModel(event, projection)
|
||||
} catch (error) {
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.rollback(sourceReservation)
|
||||
}
|
||||
this.projections.rollbackCommitted(reservation)
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
this.dependencies.project(event, projection)
|
||||
} catch {
|
||||
const id = projection.identity.projectionSemanticsId
|
||||
this.projections.transfer([id], 'projection-admission-failed')
|
||||
}
|
||||
return model
|
||||
})
|
||||
const completion = receipt.then(
|
||||
(modelReceipt) => {
|
||||
if (!projection) {
|
||||
throw outputIntakeError('ssh_projection_receipt_missing')
|
||||
}
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.settleModel(sourceReservation.span)
|
||||
}
|
||||
return Object.freeze({ ...modelReceipt, projection })
|
||||
},
|
||||
(error) => {
|
||||
if (projection) {
|
||||
this.projections.transfer(
|
||||
[projection.identity.projectionSemanticsId],
|
||||
'model-admission-failed'
|
||||
)
|
||||
}
|
||||
const code = (error as { code?: unknown }).code
|
||||
if (
|
||||
code !== 'ssh_exit_delivery_canceled' &&
|
||||
!(typeof code === 'string' && code.startsWith('ssh_model_migration_'))
|
||||
) {
|
||||
this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
)
|
||||
tracked.completion = completion
|
||||
this.modelMigration.track(tracked)
|
||||
return completion
|
||||
}
|
||||
|
||||
async acceptExit(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
this.generationGuard.sealExit(event)
|
||||
await this.exitDeadline.wait(event, (validateNormalExit) =>
|
||||
this.finishExit(event, validateNormalExit)
|
||||
)
|
||||
}
|
||||
|
||||
private async finishExit(
|
||||
event: SshPtyOutputExitEvent,
|
||||
validateNormalExit: () => void
|
||||
): Promise<void> {
|
||||
await settleSshPtyOutputExit({
|
||||
event,
|
||||
admission: this.admission,
|
||||
projections: this.projections,
|
||||
dependencies: this.dependencies,
|
||||
validateGeneration: () => {
|
||||
this.generationGuard.validate(event)
|
||||
validateNormalExit()
|
||||
},
|
||||
prepareExit: () => this.exitDeadline.prepareExitOnce(event),
|
||||
afterAdmissionIdle: () => this.sourceObligations.sealPty(event),
|
||||
waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event),
|
||||
beforeFinalize: () => this.sourceObligations.markExitPublished(event)
|
||||
})
|
||||
}
|
||||
|
||||
publishProjectionPrefix(
|
||||
ids: readonly string[],
|
||||
displayChars: number,
|
||||
accountingChars: number
|
||||
): void {
|
||||
this.projections.publishPrefix(ids, displayChars, accountingChars)
|
||||
}
|
||||
|
||||
settleProjectionPrefix(ptyId: string, accountingChars: number): number {
|
||||
return this.projections.settlePublishedPrefix(ptyId, accountingChars)
|
||||
}
|
||||
|
||||
transferProjections(ids: readonly string[], reason: string): number {
|
||||
return this.projections.transfer(ids, reason)
|
||||
}
|
||||
|
||||
transferPtyProjections(ptyId: string, reason: string): number {
|
||||
return this.projections.transferPty(ptyId, reason)
|
||||
}
|
||||
|
||||
hasProjectionFromGeneration(ids: readonly string[], providerGeneration: number): boolean {
|
||||
return ids.some(
|
||||
(id) => this.projections.get(id)?.identity.providerGeneration === providerGeneration
|
||||
)
|
||||
}
|
||||
|
||||
hasUnpublishedProjection(id: string): boolean {
|
||||
return this.projections.hasUnpublished(id)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.generationGuard.closeGeneration(providerGeneration)
|
||||
this.admission.closeGeneration(providerGeneration, reason)
|
||||
this.dependencies.onGenerationClosed?.(providerGeneration, reason)
|
||||
this.projections.closeGeneration(providerGeneration, reason)
|
||||
this.sourceObligations.closeGeneration(providerGeneration, reason)
|
||||
this.exitDeadline.closeGeneration(providerGeneration, outputIntakeError(reason))
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
for (const generation of this.generationGuard.activeGenerations()) {
|
||||
this.closeGeneration(generation, 'ssh_output_intake_disposed')
|
||||
}
|
||||
this.admission.dispose()
|
||||
this.sourceObligations.dispose()
|
||||
}
|
||||
|
||||
getRemoteSourceRangeConsumerHooks() {
|
||||
return this.sourceObligations.remoteHooks
|
||||
}
|
||||
|
||||
getAcceptedSourceCheckpoints(providerGeneration: number) {
|
||||
return this.sourceObligations.acceptedCheckpoints(providerGeneration)
|
||||
}
|
||||
|
||||
beginGenerationMigration(
|
||||
providerGeneration: number,
|
||||
timeoutMs?: number
|
||||
): SshPtyOutputGenerationMigration {
|
||||
return this.modelMigration.beginGeneration(providerGeneration, timeoutMs)
|
||||
}
|
||||
|
||||
applySourceCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): boolean {
|
||||
return this.sourceObligations.applyCancellationProof(event, proof)
|
||||
}
|
||||
|
||||
applySourceRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
this.sourceObligations.applyRecoveryCancellationProof(event, proof)
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return {
|
||||
model: this.admission.getDebugSnapshot(),
|
||||
projection: this.projections.getDebugSnapshot(),
|
||||
source: this.sourceObligations.getDebugSnapshot(),
|
||||
generation: this.generationGuard.getDebugSnapshot(),
|
||||
exitBarriers: this.exitDeadline.activeBarriers
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
describe('SshPtyOutputModelMigration', () => {
|
||||
it('fences a running source span before exporting its migration checkpoint', async () => {
|
||||
const harness = createHarness()
|
||||
const first = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'aaaa',
|
||||
source: {
|
||||
spanId: 'span-a',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
const second = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'bbbb',
|
||||
source: {
|
||||
spanId: 'span-b',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const migration = harness.intake.beginGenerationMigration(1)
|
||||
const result = migration.byPty.get('pty-1')
|
||||
expect(result).toBeDefined()
|
||||
await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toBe('pending')
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(4)
|
||||
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(second).resolves.toMatchObject({ sequence: 8 })
|
||||
await expect(result).resolves.toMatchObject({
|
||||
status: 'settled',
|
||||
checkpoint: { acceptedSourceEndSu: 8 }
|
||||
})
|
||||
expect(harness.order.filter((entry) => entry === 'project:bbbb')).toHaveLength(1)
|
||||
await migration.completion
|
||||
})
|
||||
|
||||
it('times out one migration, resets its model, and releases retained admission once', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resetModelForMigration = vi.fn()
|
||||
const harness = createHarness({ resetModelForMigration })
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-b',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const migration = harness.intake.beginGenerationMigration(1, 10_000)
|
||||
const result = migration.byPty.get('pty-1')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
|
||||
await expect(result).resolves.toMatchObject({ status: 'checkpoint-unavailable' })
|
||||
await expect(receipt).rejects.toThrow('ssh_model_migration_timeout')
|
||||
expect(resetModelForMigration).toHaveBeenCalledOnce()
|
||||
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-1')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
migratingPtys: 1
|
||||
})
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(resetModelForMigration).toHaveBeenCalledOnce()
|
||||
harness.intake.closeGeneration(1, 'connection_lost')
|
||||
expect(harness.intake.getDebugSnapshot().model.migratingPtys).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('contains a running callback failure to its migrating PTY', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resetModelForMigration = vi.fn()
|
||||
const harness = createHarness({ resetModelForMigration })
|
||||
const sibling = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-sibling',
|
||||
ptyIncarnation: 'incarnation-sibling',
|
||||
source: {
|
||||
spanId: 'span-sibling',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-sibling',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await sibling
|
||||
const failed = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-failed',
|
||||
ptyIncarnation: 'incarnation-failed',
|
||||
source: {
|
||||
spanId: 'span-failed',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-failed',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const migration = harness.intake.beginGenerationMigration(1)
|
||||
|
||||
harness.completions[1]!.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(failed).rejects.toMatchObject({
|
||||
message: 'emulator failed',
|
||||
code: 'ssh_model_migration_completion_failed'
|
||||
})
|
||||
await expect(migration.byPty.get('pty-failed')).resolves.toEqual({
|
||||
status: 'checkpoint-unavailable',
|
||||
reason: 'completion-failed'
|
||||
})
|
||||
await expect(migration.byPty.get('pty-sibling')).resolves.toMatchObject({
|
||||
status: 'settled',
|
||||
checkpoint: { id: 'pty-sibling', acceptedSourceEndSu: 4 }
|
||||
})
|
||||
await migration.completion
|
||||
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
|
||||
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-failed')
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ id: 'pty-failed', acceptedSourceEndSu: 0 }),
|
||||
expect.objectContaining({ id: 'pty-sibling', acceptedSourceEndSu: 4 })
|
||||
])
|
||||
)
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
pressureFrames: 0,
|
||||
migratingPtys: 2
|
||||
})
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
|
||||
harness.completions[1]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(
|
||||
harness.intake
|
||||
.getAcceptedSourceCheckpoints(1)
|
||||
.find((checkpoint) => checkpoint.id === 'pty-failed')?.acceptedSourceEndSu
|
||||
).toBe(0)
|
||||
|
||||
const unrelated = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-unrelated',
|
||||
providerGeneration: 2,
|
||||
ptyIncarnation: 'incarnation-unrelated'
|
||||
})
|
||||
)
|
||||
harness.completions[2]!.resolve()
|
||||
await expect(unrelated).resolves.toMatchObject({ providerGeneration: 2 })
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
harness.intake.closeGeneration(1, 'connection_lost')
|
||||
harness.intake.closeGeneration(2, 'connection_lost')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
migratingPtys: 0
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,134 @@
|
||||
import type {
|
||||
SshPtyAcceptedSourceCheckpoint,
|
||||
SshPtyOutputSourceObligations
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
|
||||
export const SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS = 10_000
|
||||
|
||||
export type SshPtyOutputMigrationResult =
|
||||
| Readonly<{ status: 'settled'; checkpoint: SshPtyAcceptedSourceCheckpoint }>
|
||||
| Readonly<{ status: 'checkpoint-unavailable'; reason: 'completion-failed' | 'timeout' }>
|
||||
|
||||
export type SshPtyOutputGenerationMigration = Readonly<{
|
||||
byPty: ReadonlyMap<string, Promise<SshPtyOutputMigrationResult>>
|
||||
completion: Promise<void>
|
||||
}>
|
||||
|
||||
export type SshPtyTrackedModelAdmission = {
|
||||
readonly key: SshPtyModelAdmissionKey
|
||||
started: boolean
|
||||
completion?: Promise<unknown>
|
||||
}
|
||||
|
||||
export class SshPtyOutputModelMigration {
|
||||
private readonly pendingByPty = new Map<string, Set<SshPtyTrackedModelAdmission>>()
|
||||
|
||||
constructor(
|
||||
private readonly admission: SshPtyModelAdmission,
|
||||
private readonly sourceObligations: SshPtyOutputSourceObligations,
|
||||
private readonly resetModel: (providerGeneration: number, ptyId: string) => void
|
||||
) {}
|
||||
|
||||
track(record: SshPtyTrackedModelAdmission): void {
|
||||
const id = migrationKey(record.key)
|
||||
const records = this.pendingByPty.get(id) ?? new Set<SshPtyTrackedModelAdmission>()
|
||||
records.add(record)
|
||||
this.pendingByPty.set(id, records)
|
||||
record.completion?.then(
|
||||
() => this.remove(id, record),
|
||||
() => this.remove(id, record)
|
||||
)
|
||||
}
|
||||
|
||||
beginGeneration(
|
||||
providerGeneration: number,
|
||||
timeoutMs = SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
|
||||
): SshPtyOutputGenerationMigration {
|
||||
const checkpoints = this.sourceObligations.acceptedCheckpoints(providerGeneration)
|
||||
const keys = checkpoints.map((checkpoint) => ({
|
||||
ptyId: checkpoint.id,
|
||||
providerGeneration
|
||||
}))
|
||||
for (const key of keys) {
|
||||
this.admission.beginMigration(key)
|
||||
}
|
||||
const byPty = new Map<string, Promise<SshPtyOutputMigrationResult>>()
|
||||
for (const key of keys) {
|
||||
byPty.set(key.ptyId, this.settlePty(key, timeoutMs))
|
||||
}
|
||||
const completion = Promise.allSettled(byPty.values()).then(() => {})
|
||||
return Object.freeze({ byPty, completion })
|
||||
}
|
||||
|
||||
private async settlePty(
|
||||
key: SshPtyModelAdmissionKey,
|
||||
timeoutMs: number
|
||||
): Promise<SshPtyOutputMigrationResult> {
|
||||
const running = Array.from(this.pendingByPty.get(migrationKey(key)) ?? []).find(
|
||||
(record) => record.started
|
||||
)
|
||||
if (!running?.completion) {
|
||||
return this.settledCheckpoint(key)
|
||||
}
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
const timeout = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), normalizedTimeout(timeoutMs))
|
||||
timer.unref?.()
|
||||
})
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
running.completion.then(
|
||||
() => 'settled' as const,
|
||||
() => 'failed' as const
|
||||
),
|
||||
timeout
|
||||
])
|
||||
if (outcome === 'settled') {
|
||||
return this.settledCheckpoint(key)
|
||||
}
|
||||
const reason = outcome === 'timeout' ? 'timeout' : 'completion-failed'
|
||||
this.resetModel(key.providerGeneration, key.ptyId)
|
||||
this.admission.cancelPty(key, `ssh_model_migration_${reason}`)
|
||||
if (outcome === 'timeout') {
|
||||
await running.completion.catch(() => {})
|
||||
}
|
||||
return Object.freeze({ status: 'checkpoint-unavailable', reason })
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private settledCheckpoint(key: SshPtyModelAdmissionKey): SshPtyOutputMigrationResult {
|
||||
const checkpoint = this.sourceObligations.acceptedCheckpoint(key)
|
||||
if (!checkpoint) {
|
||||
this.resetModel(key.providerGeneration, key.ptyId)
|
||||
return Object.freeze({
|
||||
status: 'checkpoint-unavailable',
|
||||
reason: 'completion-failed'
|
||||
})
|
||||
}
|
||||
return Object.freeze({ status: 'settled', checkpoint })
|
||||
}
|
||||
|
||||
private remove(id: string, record: SshPtyTrackedModelAdmission): void {
|
||||
const records = this.pendingByPty.get(id)
|
||||
records?.delete(record)
|
||||
if (records?.size === 0) {
|
||||
this.pendingByPty.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function migrationKey(key: SshPtyModelAdmissionKey): string {
|
||||
return `${key.providerGeneration}\0${key.ptyId}`
|
||||
}
|
||||
|
||||
function normalizedTimeout(timeoutMs: number): number {
|
||||
return Number.isFinite(timeoutMs) && timeoutMs >= 0
|
||||
? Math.floor(timeoutMs)
|
||||
: SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { RemoteTerminalSourceRangeConsumerHooks } from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { DesktopProjectionSpan } from './ssh-pty-legacy-projection'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtySourceCancellationProof,
|
||||
SshPtySourceCancellationRequest
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
|
||||
import type { SshPtySourceAdmissionReservation } from './ssh-pty-source-obligation-contract'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
export type SshPtyOutputSourceReservation = Readonly<{
|
||||
admission: SshPtySourceAdmissionReservation
|
||||
span: PtySourceSpan
|
||||
}>
|
||||
|
||||
export type SshPtySourceCancellationProofCommit = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
proof: SshPtySourceCancellationProof
|
||||
}>
|
||||
|
||||
export type SshPtyAcceptedSourceCheckpoint = Readonly<{
|
||||
id: string
|
||||
providerGeneration: number
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
deliveryToken: string
|
||||
acceptedSourceEndSu: number
|
||||
}>
|
||||
|
||||
export class SshPtyOutputSourceObligations {
|
||||
private readonly coordinator: SshPtySourceObligationCoordinator
|
||||
private readonly remoteConsumers: SshPtyRemoteSourceRangeConsumers
|
||||
private readonly openedTokens = new Set<string>()
|
||||
private readonly identityByPty = new Map<string, PtySourceDeliveryIdentity>()
|
||||
|
||||
constructor(publish: SshPtyOutputDataEventPublisher | undefined) {
|
||||
this.coordinator = new SshPtySourceObligationCoordinator({
|
||||
onTokenClosed: (identity) => this.removeIdentity(identity),
|
||||
publish:
|
||||
publish ??
|
||||
((_providerGeneration, _batch, onSettled) =>
|
||||
onSettled({ ok: false, error: new Error('SSH PTY source ACK publisher unavailable') }))
|
||||
})
|
||||
this.remoteConsumers = new SshPtyRemoteSourceRangeConsumers(this.coordinator)
|
||||
}
|
||||
|
||||
get remoteHooks(): RemoteTerminalSourceRangeConsumerHooks {
|
||||
return this.remoteConsumers.hooks
|
||||
}
|
||||
|
||||
reserve(
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: DesktopProjectionSpan
|
||||
): SshPtyOutputSourceReservation {
|
||||
const span = this.toSourceSpan(event, projection)
|
||||
const identity = this.sourceIdentity(span)
|
||||
const tokenKey = ptySourceDeliveryKey(identity)
|
||||
if (!this.openedTokens.has(tokenKey)) {
|
||||
this.coordinator.open(identity, span.sourceStartSu)
|
||||
this.openedTokens.add(tokenKey)
|
||||
this.identityByPty.set(this.ptyKey(event), identity)
|
||||
}
|
||||
return Object.freeze({
|
||||
span,
|
||||
admission: this.coordinator.reserve(identity, span, [
|
||||
'model',
|
||||
'desktop',
|
||||
...this.remoteConsumers.requiredConsumers(event.id)
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
commit(
|
||||
reservation: SshPtyOutputSourceReservation,
|
||||
ptyId: string,
|
||||
modelSequenceEnd: number
|
||||
): void {
|
||||
this.coordinator.commit(reservation.admission)
|
||||
this.remoteConsumers.trackSpan(
|
||||
ptyId,
|
||||
reservation.span.spanId,
|
||||
reservation.admission.requiredConsumers,
|
||||
modelSequenceEnd
|
||||
)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtyOutputSourceReservation): boolean {
|
||||
return (
|
||||
this.coordinator.rollback(reservation.admission) ||
|
||||
this.coordinator.rollbackCommitted(reservation.admission)
|
||||
)
|
||||
}
|
||||
|
||||
settleModel(span: PtySourceSpan): void {
|
||||
this.coordinator.settle({
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'model-accepted'
|
||||
})
|
||||
}
|
||||
|
||||
settleDesktop(span: DesktopProjectionSpan, reason: string): void {
|
||||
this.coordinator.settle({
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop',
|
||||
reason
|
||||
})
|
||||
}
|
||||
|
||||
transferDesktop(span: DesktopProjectionSpan, reason: string): void {
|
||||
const transition = {
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop' as const,
|
||||
reason
|
||||
}
|
||||
if (this.coordinator.beginTransfer(transition, 'model')) {
|
||||
this.coordinator.commitTransfer(transition)
|
||||
}
|
||||
}
|
||||
|
||||
sealPty(event: SshPtyOutputExitEvent): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.seal(identity)
|
||||
}
|
||||
}
|
||||
|
||||
markExitPublished(event: SshPtyOutputExitEvent): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.markExitPublished(identity)
|
||||
}
|
||||
}
|
||||
|
||||
whenPtyTerminal(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve()
|
||||
}
|
||||
|
||||
async requestPtyCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
cancel: (request: SshPtySourceCancellationRequest) => Promise<SshPtySourceCancellationProof>
|
||||
): Promise<SshPtySourceCancellationProofCommit | null> {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (!identity) {
|
||||
return null
|
||||
}
|
||||
const request = this.coordinator.beginExitTimeout(identity)
|
||||
const proof = await cancel(request)
|
||||
return Object.freeze({ identity, proof })
|
||||
}
|
||||
|
||||
commitPtyCancellationProof(commit: SshPtySourceCancellationProofCommit): void {
|
||||
this.coordinator.applyCancellationProof(commit.identity, commit.proof)
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (!identity) {
|
||||
return false
|
||||
}
|
||||
this.coordinator.applyCancellationProof(identity, proof)
|
||||
return true
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.applyRecoveryCancellationProof(identity, proof)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.remoteConsumers.closeGeneration(providerGeneration, reason)
|
||||
this.coordinator.closeGeneration(providerGeneration, reason)
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.openedTokens) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.openedTokens.delete(key)
|
||||
}
|
||||
}
|
||||
for (const [key, identity] of this.identityByPty) {
|
||||
if (identity.providerGeneration === providerGeneration) {
|
||||
this.identityByPty.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acceptedCheckpoints(providerGeneration: number): readonly SshPtyAcceptedSourceCheckpoint[] {
|
||||
const checkpoints: SshPtyAcceptedSourceCheckpoint[] = []
|
||||
for (const identity of this.identityByPty.values()) {
|
||||
if (identity.providerGeneration !== providerGeneration) {
|
||||
continue
|
||||
}
|
||||
checkpoints.push(
|
||||
Object.freeze({
|
||||
id: identity.id,
|
||||
providerGeneration: identity.providerGeneration,
|
||||
clientGeneration: identity.clientGeneration,
|
||||
ownerGeneration: identity.ownerGeneration,
|
||||
ptyIncarnation: identity.ptyIncarnation,
|
||||
deliveryToken: identity.deliveryToken,
|
||||
acceptedSourceEndSu: this.coordinator.modelAcceptedEnd(identity)
|
||||
})
|
||||
)
|
||||
}
|
||||
return Object.freeze(checkpoints)
|
||||
}
|
||||
|
||||
acceptedCheckpoint(key: {
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
}): SshPtyAcceptedSourceCheckpoint | null {
|
||||
return (
|
||||
this.acceptedCheckpoints(key.providerGeneration).find(
|
||||
(checkpoint) => checkpoint.id === key.ptyId
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.coordinator.dispose()
|
||||
}
|
||||
|
||||
getDebugSnapshot(): Readonly<{ openedTokens: number; ptyIdentities: number }> {
|
||||
return Object.freeze({
|
||||
openedTokens: this.openedTokens.size,
|
||||
ptyIdentities: this.identityByPty.size
|
||||
})
|
||||
}
|
||||
|
||||
private toSourceSpan(
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: DesktopProjectionSpan
|
||||
): PtySourceSpan {
|
||||
return Object.freeze({
|
||||
id: projection.id,
|
||||
providerGeneration: event.providerGeneration,
|
||||
clientGeneration: projection.clientGeneration,
|
||||
ownerGeneration: projection.ownerGeneration,
|
||||
ptyIncarnation: event.ptyIncarnation,
|
||||
deliveryToken: projection.deliveryToken,
|
||||
spanId: projection.spanId,
|
||||
sourceStartSu: projection.sourceStartSu,
|
||||
sourceEndSu: projection.sourceEndSu,
|
||||
displayStart: projection.displayStart,
|
||||
displayEnd: projection.displayEnd,
|
||||
splittable: projection.splittable,
|
||||
transform: projection.transform,
|
||||
data: event.data
|
||||
})
|
||||
}
|
||||
|
||||
private sourceIdentity(source: PtySourceSpan): PtySourceDeliveryIdentity {
|
||||
return source
|
||||
}
|
||||
|
||||
private ptyKey(event: {
|
||||
id: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}): string {
|
||||
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
|
||||
}
|
||||
|
||||
private removeIdentity(identity: PtySourceDeliveryIdentity): void {
|
||||
this.openedTokens.delete(ptySourceDeliveryKey(identity))
|
||||
for (const [key, candidate] of this.identityByPty) {
|
||||
if (samePtySourceDelivery(candidate, identity)) {
|
||||
this.identityByPty.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type SshPtyOutputDataEventPublisher = NonNullable<
|
||||
SshPtyOutputIntakeDependencies['publishSourceAck']
|
||||
>
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { ProjectionRecord } from './ssh-pty-legacy-projection-record'
|
||||
|
||||
type ProjectionTerminalWaiter = {
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
resolve: () => void
|
||||
}
|
||||
|
||||
function matchesProjection(
|
||||
record: ProjectionRecord | undefined,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): boolean {
|
||||
const identity = record?.semantics.identity
|
||||
return (
|
||||
identity?.providerGeneration === providerGeneration &&
|
||||
identity.ptyIncarnation === ptyIncarnation
|
||||
)
|
||||
}
|
||||
|
||||
export function unpublishedProjectionIds(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
ids: readonly string[],
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): string[] {
|
||||
return ids.filter((id) => {
|
||||
const record = records.get(id)
|
||||
return (
|
||||
record?.state === 'committed' && matchesProjection(record, providerGeneration, ptyIncarnation)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
export function projectionHasOpen(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
idsByPty: ReadonlyMap<string, readonly string[]>,
|
||||
ptyId: string
|
||||
): (providerGeneration: number, ptyIncarnation: string) => boolean {
|
||||
return (providerGeneration, ptyIncarnation) =>
|
||||
(idsByPty.get(ptyId) ?? []).some((id) =>
|
||||
matchesProjection(records.get(id), providerGeneration, ptyIncarnation)
|
||||
)
|
||||
}
|
||||
|
||||
export function resolveProjectionTerminality(
|
||||
terminality: SshPtyProjectionTerminality,
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
idsByPty: ReadonlyMap<string, readonly string[]>,
|
||||
ptyId: string
|
||||
): void {
|
||||
terminality.resolve(ptyId, projectionHasOpen(records, idsByPty, ptyId))
|
||||
}
|
||||
|
||||
export class SshPtyProjectionTerminality {
|
||||
private readonly waitersByPty = new Map<string, ProjectionTerminalWaiter[]>()
|
||||
|
||||
whenTerminal(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
|
||||
): Promise<void> {
|
||||
if (!hasOpen(providerGeneration, ptyIncarnation)) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const waiters = this.waitersByPty.get(ptyId) ?? []
|
||||
waiters.push({ providerGeneration, ptyIncarnation, resolve })
|
||||
this.waitersByPty.set(ptyId, waiters)
|
||||
})
|
||||
}
|
||||
|
||||
resolve(
|
||||
ptyId: string,
|
||||
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
|
||||
): void {
|
||||
const waiters = this.waitersByPty.get(ptyId)
|
||||
if (!waiters) {
|
||||
return
|
||||
}
|
||||
const pending = waiters.filter((waiter) =>
|
||||
hasOpen(waiter.providerGeneration, waiter.ptyIncarnation)
|
||||
)
|
||||
for (const waiter of waiters) {
|
||||
if (!pending.includes(waiter)) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
this.waitersByPty.set(ptyId, pending)
|
||||
} else {
|
||||
this.waitersByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number): void {
|
||||
for (const [ptyId, waiters] of this.waitersByPty) {
|
||||
const pending = waiters.filter((waiter) => waiter.providerGeneration !== providerGeneration)
|
||||
for (const waiter of waiters) {
|
||||
if (waiter.providerGeneration === providerGeneration) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
this.waitersByPty.set(ptyId, pending)
|
||||
} else {
|
||||
this.waitersByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
const identity: PtySourceDeliveryIdentity = {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1'
|
||||
}
|
||||
|
||||
function span(spanId: string, overrides: Partial<PtySourceSpan> = {}): PtySourceSpan {
|
||||
return {
|
||||
...identity,
|
||||
spanId,
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
data: 'data',
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function range(
|
||||
spanId: string,
|
||||
overrides: Partial<TerminalOutputSourceRange> = {}
|
||||
): TerminalOutputSourceRange {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
spanId,
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function createCoordinator(): SshPtySourceObligationCoordinator {
|
||||
return new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
}
|
||||
|
||||
describe('SshPtyRemoteSourceRangeConsumers', () => {
|
||||
it('snapshots membership and settles only the current stream generation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const progress = vi.fn()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger, progress)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
expect(consumers.hooks.attach(stream)).toBe(true)
|
||||
const sourceSpan = span('span-1')
|
||||
const reservation = ledger.reserve(identity, sourceSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', sourceSpan.spanId, reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.settle({ ...stream, streamGeneration: 'stale' }, [range('span-1')])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
consumers.hooks.settle(stream, [range('span-1')])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
|
||||
expect(progress).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it.each(['headless', 'renderer'] as const)(
|
||||
'commits remaining mappings only after a %s snapshot publication',
|
||||
(source) => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
expect(replacement).not.toBeNull()
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring',
|
||||
to: 'remote:snapshot:consumer-1'
|
||||
})
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 3 })).toBe(false)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('transferring')
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 4 })).toBe(true)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferred',
|
||||
to: 'remote:snapshot:consumer-1'
|
||||
})
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual(['remote:consumer-1'])
|
||||
}
|
||||
)
|
||||
|
||||
it('reserves only spans covered by the authoritative snapshot sequence', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const covered = ledger.reserve(identity, span('span-covered'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(covered)
|
||||
consumers.trackSpan('pty-1', 'span-covered', covered.requiredConsumers, 4)
|
||||
const trailingSpan = {
|
||||
...span('span-trailing'),
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
}
|
||||
const trailing = ledger.reserve(identity, trailingSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(trailing)
|
||||
consumers.trackSpan('pty-1', 'span-trailing', trailing.requiredConsumers, 8)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
expect(replacement).not.toBeNull()
|
||||
expect(ledger.obligation('span-covered', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring'
|
||||
})
|
||||
expect(ledger.obligation('span-trailing', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'open'
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls a failed replacement publication back to the live stream obligation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'snapshot-write-failed')).toBe(true)
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'open'
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls back every transfer when replacement reservation fails partway', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const first = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(first)
|
||||
consumers.trackSpan('pty-1', 'span-1', first.requiredConsumers, 4)
|
||||
const secondSpan = span('span-2', {
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
const second = ledger.reserve(identity, secondSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(second)
|
||||
consumers.trackSpan('pty-1', 'span-2', second.requiredConsumers, 8)
|
||||
const beginTransfer = ledger.beginTransfer.bind(ledger)
|
||||
vi.spyOn(ledger, 'beginTransfer').mockImplementation((transition, replacement) => {
|
||||
if (transition.spanId === 'span-2') {
|
||||
throw new Error('injected partial reserve failure')
|
||||
}
|
||||
return beginTransfer(transition, replacement)
|
||||
})
|
||||
|
||||
expect(() => consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')).toThrowError(
|
||||
'injected partial reserve failure'
|
||||
)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
|
||||
})
|
||||
|
||||
it('rejects a replacement whose exact transfer state changed before commit', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const sourceSpan = span('span-1')
|
||||
const admission = ledger.reserve(identity, sourceSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(admission)
|
||||
consumers.trackSpan('pty-1', 'span-1', admission.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')!
|
||||
const transition = {
|
||||
identity: sourceSpan,
|
||||
spanId: sourceSpan.spanId,
|
||||
consumer: 'remote:consumer-1' as const,
|
||||
reason: 'concurrent-replacement'
|
||||
}
|
||||
expect(ledger.rollbackTransfer(transition)).toBe(true)
|
||||
expect(ledger.beginTransfer(transition, 'remote:snapshot:consumer-1')).toBe(true)
|
||||
|
||||
expect(consumers.hooks.commitReplacement(replacement, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
expect(consumers.hooks.rollbackReplacement(replacement, 'commit-rejected')).toBe(false)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring',
|
||||
reason: 'concurrent-replacement'
|
||||
})
|
||||
})
|
||||
|
||||
it('settles a split source span only after its complete ordered range is acknowledged', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.settle(stream, [
|
||||
range('span-1', {
|
||||
sourceEndSu: 2,
|
||||
displayEnd: 2,
|
||||
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
|
||||
consumers.hooks.settle(stream, [
|
||||
range('span-1', {
|
||||
sourceStartSu: 2,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 2,
|
||||
displayEnd: 4,
|
||||
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
|
||||
})
|
||||
|
||||
it('cancels an admitted span on detach without minting a snapshot owner', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.cancel(stream, [], 'stream-detached')
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'canceled',
|
||||
reason: 'stream-detached'
|
||||
})
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
|
||||
it('rolls back a pending replacement before disconnect cancellation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
consumers.hooks.cancel(stream, [], 'connection-closed')
|
||||
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'canceled',
|
||||
reason: 'connection-closed'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects stale stream generations without changing the current obligation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
expect(() =>
|
||||
consumers.hooks.reserveReplacement(
|
||||
{ ...stream, streamGeneration: 'stale' },
|
||||
4,
|
||||
'initial-snapshot'
|
||||
)
|
||||
).toThrow('stale')
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
})
|
||||
|
||||
it('rolls back replacement admission before provider-generation close', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
consumers.closeGeneration(identity.providerGeneration, 'provider-replaced')
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('detaches cleanly after cancellation proof reclaims tracked spans', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
ledger.seal(identity)
|
||||
ledger.beginExitTimeout(identity)
|
||||
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
|
||||
expect(() => consumers.hooks.settle(stream, [range('span-1')])).not.toThrow()
|
||||
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
|
||||
it('rejects and rolls back a replacement after proof reclaims its covered spans', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const recoveredIdentity = {
|
||||
...identity,
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-2'
|
||||
}
|
||||
ledger.open(recoveredIdentity, 4)
|
||||
const recovered = ledger.reserve(
|
||||
recoveredIdentity,
|
||||
span('span-2', {
|
||||
...recoveredIdentity,
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
}),
|
||||
['model', ...consumers.requiredConsumers('pty-1')]
|
||||
)
|
||||
ledger.commit(recovered)
|
||||
consumers.trackSpan('pty-1', 'span-2', recovered.requiredConsumers, 8)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')
|
||||
ledger.seal(identity)
|
||||
ledger.beginExitTimeout(identity)
|
||||
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
|
||||
const queuedOutput = ['covered-output', 'surviving-output']
|
||||
const committed = consumers.hooks.commitReplacement(replacement!, {
|
||||
source: 'headless',
|
||||
seq: 8
|
||||
})
|
||||
if (committed) {
|
||||
queuedOutput.splice(0)
|
||||
}
|
||||
|
||||
expect(committed).toBe(false)
|
||||
expect(queuedOutput).toEqual(['covered-output', 'surviving-output'])
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(true)
|
||||
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(false)
|
||||
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,285 @@
|
||||
import {
|
||||
sameTerminalOutputSourceIdentity,
|
||||
type TerminalOutputSourceRange
|
||||
} from '../../shared/terminal-output-source-range'
|
||||
import type {
|
||||
RemoteTerminalSourceRangeConsumerHooks,
|
||||
RemoteTerminalSourceRangeStreamIdentity
|
||||
} from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract'
|
||||
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
import { SshPtyRemoteSourceRangeReplacements } from './ssh-pty-remote-source-range-replacement'
|
||||
|
||||
function remoteConsumerId(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
): SshPtySourceConsumerId {
|
||||
return `remote:${identity.consumerId}`
|
||||
}
|
||||
|
||||
function uniqueSpanIds(ranges: readonly TerminalOutputSourceRange[]): string[] {
|
||||
return Array.from(new Set(ranges.map((range) => range.spanId)))
|
||||
}
|
||||
|
||||
type RemoteConsumerState = {
|
||||
streamGeneration: string
|
||||
spans: Map<
|
||||
string,
|
||||
Readonly<{
|
||||
identity: PtySourceSpan
|
||||
modelSequenceEnd: number
|
||||
}>
|
||||
>
|
||||
ackedEndBySpan: Map<string, number>
|
||||
}
|
||||
|
||||
export class SshPtyRemoteSourceRangeConsumers {
|
||||
private readonly consumersByPty = new Map<string, Map<string, RemoteConsumerState>>()
|
||||
private readonly replacements: SshPtyRemoteSourceRangeReplacements
|
||||
|
||||
constructor(
|
||||
private readonly coordinator: SshPtySourceObligationCoordinator,
|
||||
private readonly onProgress: (range: TerminalOutputSourceRange) => void = () => {}
|
||||
) {
|
||||
this.replacements = new SshPtyRemoteSourceRangeReplacements(coordinator)
|
||||
}
|
||||
|
||||
readonly hooks: RemoteTerminalSourceRangeConsumerHooks = {
|
||||
attach: (identity) => this.attach(identity),
|
||||
settle: (identity, ranges) => this.settle(identity, ranges),
|
||||
reserveReplacement: (identity, requiredSeq, reason) =>
|
||||
this.reserveReplacement(identity, requiredSeq, reason),
|
||||
commitReplacement: (reservation, publication) =>
|
||||
this.commitReplacement(reservation, publication),
|
||||
rollbackReplacement: (reservation, reason) => this.rollbackReplacement(reservation, reason),
|
||||
cancel: (identity, ranges, reason) => this.cancel(identity, ranges, reason)
|
||||
}
|
||||
|
||||
requiredConsumers(ptyId: string): readonly SshPtySourceConsumerId[] {
|
||||
return Object.freeze(
|
||||
Array.from(this.consumersByPty.get(ptyId)?.keys() ?? []).map(
|
||||
(consumerId) => `remote:${consumerId}` as const
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
trackSpan(
|
||||
ptyId: string,
|
||||
spanId: string,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[],
|
||||
modelSequenceEnd: number
|
||||
): void {
|
||||
if (!Number.isSafeInteger(modelSequenceEnd) || modelSequenceEnd < 0) {
|
||||
throw new Error('ssh_remote_source_range_model_sequence_invalid')
|
||||
}
|
||||
for (const [consumerId, state] of this.consumersByPty.get(ptyId) ?? []) {
|
||||
if (requiredConsumers.includes(`remote:${consumerId}`)) {
|
||||
state.spans.set(
|
||||
spanId,
|
||||
Object.freeze({
|
||||
identity: this.coordinator.spanIdentity(spanId),
|
||||
modelSequenceEnd
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.replacements.closeGeneration(providerGeneration, reason)
|
||||
for (const consumers of this.consumersByPty.values()) {
|
||||
for (const state of consumers.values()) {
|
||||
for (const [spanId, tracked] of state.spans) {
|
||||
if (tracked.identity.providerGeneration === providerGeneration) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private attach(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
|
||||
const consumers =
|
||||
this.consumersByPty.get(identity.ptyId) ?? new Map<string, RemoteConsumerState>()
|
||||
const current = consumers.get(identity.consumerId)
|
||||
if (current && current.streamGeneration !== identity.streamGeneration) {
|
||||
return false
|
||||
}
|
||||
consumers.set(
|
||||
identity.consumerId,
|
||||
current ?? {
|
||||
streamGeneration: identity.streamGeneration,
|
||||
spans: new Map(),
|
||||
ackedEndBySpan: new Map()
|
||||
}
|
||||
)
|
||||
this.consumersByPty.set(identity.ptyId, consumers)
|
||||
return true
|
||||
}
|
||||
|
||||
private settle(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
): void {
|
||||
if (!this.isCurrent(identity)) {
|
||||
return
|
||||
}
|
||||
const state = this.requireState(identity)
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const nextEnds = new Map(state.ackedEndBySpan)
|
||||
const completed = new Set<string>()
|
||||
for (const range of ranges) {
|
||||
const tracked = state.spans.get(range.spanId)
|
||||
if (!tracked) {
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
if (!this.coordinator.hasRetainedSpan(range.spanId)) {
|
||||
state.spans.delete(range.spanId)
|
||||
nextEnds.delete(range.spanId)
|
||||
continue
|
||||
}
|
||||
const currentEnd = nextEnds.get(range.spanId) ?? source.sourceStartSu
|
||||
if (
|
||||
!sameTerminalOutputSourceIdentity(source, range) ||
|
||||
range.sourceStartSu !== currentEnd ||
|
||||
range.sourceEndSu > source.sourceEndSu
|
||||
) {
|
||||
throw new Error('ssh_remote_source_range_settlement_invalid')
|
||||
}
|
||||
nextEnds.set(range.spanId, range.sourceEndSu)
|
||||
if (range.sourceEndSu === source.sourceEndSu) {
|
||||
completed.add(range.spanId)
|
||||
}
|
||||
}
|
||||
state.ackedEndBySpan = nextEnds
|
||||
for (const spanId of completed) {
|
||||
const tracked = state.spans.get(spanId)
|
||||
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
this.coordinator.settle({
|
||||
identity: source,
|
||||
spanId,
|
||||
consumer,
|
||||
reason: 'remote-frame-ack'
|
||||
})
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
for (const range of ranges) {
|
||||
this.onProgress(range)
|
||||
}
|
||||
}
|
||||
|
||||
private reserveReplacement(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
): ReturnType<SshPtyRemoteSourceRangeReplacements['reserve']> {
|
||||
if (!this.isCurrent(identity)) {
|
||||
throw new Error('ssh_remote_source_range_stale_generation')
|
||||
}
|
||||
const state = this.requireState(identity)
|
||||
for (const spanId of state.spans.keys()) {
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
const spanIds = Array.from(state.spans)
|
||||
.filter(([, tracked]) => tracked.modelSequenceEnd <= requiredSeq)
|
||||
.map(([spanId]) => spanId)
|
||||
return this.replacements.reserve(identity, spanIds, requiredSeq, reason)
|
||||
}
|
||||
|
||||
private commitReplacement(
|
||||
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[0],
|
||||
publication: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[1]
|
||||
): boolean {
|
||||
return this.replacements.commit(
|
||||
reservation,
|
||||
publication,
|
||||
this.isCurrent(reservation.identity),
|
||||
(spanIds) => {
|
||||
const state = this.requireState(reservation.identity)
|
||||
for (const spanId of spanIds) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private rollbackReplacement(
|
||||
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['rollback']>[0],
|
||||
reason: string
|
||||
): boolean {
|
||||
return this.replacements.rollback(reservation, reason)
|
||||
}
|
||||
|
||||
private cancel(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[],
|
||||
reason: string
|
||||
): void {
|
||||
if (!this.isCurrent(identity)) {
|
||||
return
|
||||
}
|
||||
this.replacements.rollbackIdentity(identity, `${reason}-replacement-aborted`)
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const state = this.requireState(identity)
|
||||
const spanIds = new Set([
|
||||
...state.spans.keys(),
|
||||
...uniqueSpanIds(ranges).filter((spanId) => state.spans.has(spanId))
|
||||
])
|
||||
for (const spanId of spanIds) {
|
||||
const tracked = state.spans.get(spanId)
|
||||
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
const transition = { identity: source, spanId, consumer, reason }
|
||||
if (this.coordinator.beginTransfer(transition, consumer)) {
|
||||
this.coordinator.cancelTransfer(transition)
|
||||
}
|
||||
}
|
||||
this.detachIdentity(identity)
|
||||
for (const range of ranges) {
|
||||
this.onProgress(range)
|
||||
}
|
||||
}
|
||||
|
||||
private isCurrent(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
|
||||
return (
|
||||
this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)?.streamGeneration ===
|
||||
identity.streamGeneration
|
||||
)
|
||||
}
|
||||
|
||||
private requireState(identity: RemoteTerminalSourceRangeStreamIdentity): RemoteConsumerState {
|
||||
const state = this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)
|
||||
if (!state || state.streamGeneration !== identity.streamGeneration) {
|
||||
throw new Error('ssh_remote_source_range_stale_generation')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
private detachIdentity(identity: RemoteTerminalSourceRangeStreamIdentity): void {
|
||||
const consumers = this.consumersByPty.get(identity.ptyId)
|
||||
if (
|
||||
!consumers ||
|
||||
consumers.get(identity.consumerId)?.streamGeneration !== identity.streamGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
consumers.delete(identity.consumerId)
|
||||
if (consumers.size === 0) {
|
||||
this.consumersByPty.delete(identity.ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import type {
|
||||
RemoteTerminalSourceRangeReplacementPublication,
|
||||
RemoteTerminalSourceRangeReplacementReservation,
|
||||
RemoteTerminalSourceRangeStreamIdentity
|
||||
} from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
|
||||
import type {
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
type ReplacementSpanRecord = Readonly<{
|
||||
source: PtySourceSpan
|
||||
transferState: SshPtySourceObligationState
|
||||
}>
|
||||
|
||||
type ReplacementReservationRecord = {
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation
|
||||
spans: readonly ReplacementSpanRecord[]
|
||||
consumer: SshPtySourceConsumerId
|
||||
replacement: SshPtySourceConsumerId
|
||||
reason: string
|
||||
}
|
||||
|
||||
function remoteConsumerId(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
): SshPtySourceConsumerId {
|
||||
return `remote:${identity.consumerId}`
|
||||
}
|
||||
|
||||
export class SshPtyRemoteSourceRangeReplacements {
|
||||
private readonly reservations = new Map<string, ReplacementReservationRecord>()
|
||||
private nextReservationId = 1
|
||||
|
||||
constructor(private readonly coordinator: SshPtySourceObligationCoordinator) {}
|
||||
|
||||
reserve(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
spanIds: readonly string[],
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
): RemoteTerminalSourceRangeReplacementReservation | null {
|
||||
if (spanIds.length === 0) {
|
||||
return null
|
||||
}
|
||||
if (!Number.isSafeInteger(requiredSeq) || requiredSeq < 0) {
|
||||
throw new Error('ssh_remote_source_range_replacement_sequence_invalid')
|
||||
}
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const replacement = `remote:snapshot:${identity.consumerId}` as const
|
||||
const spans = spanIds.map((spanId) => this.coordinator.spanIdentity(spanId))
|
||||
for (const { spanId } of spans) {
|
||||
if (this.coordinator.obligation(spanId, consumer).state !== 'open') {
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
}
|
||||
const transferred: ReplacementSpanRecord[] = []
|
||||
try {
|
||||
for (const source of spans) {
|
||||
const { spanId } = source
|
||||
const transition = { identity: source, spanId, consumer, reason }
|
||||
if (!this.coordinator.beginTransfer(transition, replacement)) {
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
const transferState = this.coordinator.obligation(spanId, consumer)
|
||||
if (transferState.state !== 'transferring' || transferState.to !== replacement) {
|
||||
this.coordinator.rollbackTransfer(transition)
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
transferred.push(Object.freeze({ source, transferState }))
|
||||
}
|
||||
} catch (error) {
|
||||
for (const span of transferred.toReversed()) {
|
||||
this.rollbackExactSpan(span, consumer, reason)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const reservation = Object.freeze({
|
||||
reservationId: `remote-source-replacement:${this.nextReservationId++}`,
|
||||
identity: Object.freeze({ ...identity }),
|
||||
requiredSeq
|
||||
})
|
||||
this.reservations.set(reservation.reservationId, {
|
||||
reservation,
|
||||
spans: Object.freeze(transferred),
|
||||
consumer,
|
||||
replacement,
|
||||
reason
|
||||
})
|
||||
return reservation
|
||||
}
|
||||
|
||||
commit(
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
publication: RemoteTerminalSourceRangeReplacementPublication,
|
||||
isCurrent: boolean,
|
||||
onCommitted: (spanIds: readonly string[]) => void
|
||||
): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (
|
||||
!record ||
|
||||
record.reservation !== reservation ||
|
||||
!isCurrent ||
|
||||
!Number.isSafeInteger(publication.seq) ||
|
||||
publication.seq < reservation.requiredSeq ||
|
||||
(publication.source !== 'headless' && publication.source !== 'renderer')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
record.spans.some(({ source, transferState }) => {
|
||||
const { spanId } = source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
return true
|
||||
}
|
||||
return this.coordinator.obligation(spanId, record.consumer) !== transferState
|
||||
})
|
||||
) {
|
||||
return false
|
||||
}
|
||||
for (const { source } of record.spans) {
|
||||
const { spanId } = source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
if (
|
||||
!this.coordinator.commitTransfer({ identity: source, spanId, consumer: record.consumer })
|
||||
) {
|
||||
throw new Error('ssh_remote_source_range_replacement_commit_invalid')
|
||||
}
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
onCommitted(record.spans.map(({ source }) => source.spanId))
|
||||
return true
|
||||
}
|
||||
|
||||
rollback(reservation: RemoteTerminalSourceRangeReplacementReservation, reason: string): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation) {
|
||||
return false
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
let rolledBack = true
|
||||
for (const span of record.spans) {
|
||||
const { spanId } = span.source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
rolledBack = this.rollbackExactSpan(span, record.consumer, reason) && rolledBack
|
||||
}
|
||||
return rolledBack
|
||||
}
|
||||
|
||||
rollbackIdentity(identity: RemoteTerminalSourceRangeStreamIdentity, reason: string): void {
|
||||
for (const record of Array.from(this.reservations.values())) {
|
||||
if (
|
||||
record.reservation.identity.ptyId === identity.ptyId &&
|
||||
record.reservation.identity.consumerId === identity.consumerId &&
|
||||
record.reservation.identity.streamGeneration === identity.streamGeneration
|
||||
) {
|
||||
this.rollback(record.reservation, reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
for (const record of Array.from(this.reservations.values())) {
|
||||
if (record.spans.some(({ source }) => source.providerGeneration === providerGeneration)) {
|
||||
this.rollback(record.reservation, `${reason}-replacement-aborted`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private rollbackExactSpan(
|
||||
span: ReplacementSpanRecord,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
const { source, transferState } = span
|
||||
if (
|
||||
!this.coordinator.hasRetainedSpan(source.spanId) ||
|
||||
this.coordinator.obligation(source.spanId, consumer) !== transferState
|
||||
) {
|
||||
return false
|
||||
}
|
||||
return this.coordinator.rollbackTransfer({
|
||||
identity: source,
|
||||
spanId: source.spanId,
|
||||
consumer,
|
||||
reason
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceAckCoalescer } from './ssh-pty-source-ack-coalescer'
|
||||
|
||||
function publication(token: number, endSu: number, settled = vi.fn(), providerGeneration = 1) {
|
||||
const identity = {
|
||||
id: `pty-${token}`,
|
||||
providerGeneration,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: `incarnation-${token}`,
|
||||
deliveryToken: `token-${token}`
|
||||
}
|
||||
return {
|
||||
identity,
|
||||
ack: {
|
||||
id: identity.id,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: identity.deliveryToken,
|
||||
creditedEndSu: endSu
|
||||
},
|
||||
onSettled: settled
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshPtySourceAckCoalescer', () => {
|
||||
it('coalesces cumulative values and advances them only from the write callback', () => {
|
||||
const writes: {
|
||||
batch: PtySourceCreditAckBatch
|
||||
settle: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}[] = []
|
||||
const firstSettled = vi.fn()
|
||||
const latestSettled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, batch, settle) => writes.push({ batch, settle }),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, firstSettled))
|
||||
coalescer.enqueue(publication(1, 20, latestSettled))
|
||||
coalescer.flush()
|
||||
|
||||
expect(writes[0].batch.acknowledgements).toEqual([
|
||||
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 20 })
|
||||
])
|
||||
expect(latestSettled).not.toHaveBeenCalled()
|
||||
writes[0].settle({ ok: true })
|
||||
expect(latestSettled).toHaveBeenCalledWith({ ok: true })
|
||||
expect(firstSettled).toHaveBeenCalledWith({ ok: true })
|
||||
})
|
||||
|
||||
it('limits one batch to 64 tokens and gives the remainder another turn', () => {
|
||||
const batches: PtySourceCreditAckBatch[] = []
|
||||
const scheduled: (() => void)[] = []
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, batch, settle) => {
|
||||
batches.push(batch)
|
||||
settle({ ok: true })
|
||||
},
|
||||
schedule: (callback) => {
|
||||
scheduled.push(callback)
|
||||
return scheduled.length as unknown as ReturnType<typeof setTimeout>
|
||||
},
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
for (let token = 0; token < 70; token++) {
|
||||
coalescer.enqueue(publication(token, 1))
|
||||
}
|
||||
|
||||
coalescer.flush()
|
||||
expect(batches[0].acknowledgements).toHaveLength(64)
|
||||
scheduled.at(-1)!()
|
||||
expect(batches[1].acknowledgements).toHaveLength(6)
|
||||
})
|
||||
|
||||
it('never mixes provider generations in one transport batch', () => {
|
||||
const generations: number[] = []
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (providerGeneration, _batch, settle) => {
|
||||
generations.push(providerGeneration)
|
||||
settle({ ok: true })
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 1, vi.fn(), 1))
|
||||
coalescer.enqueue(publication(2, 1, vi.fn(), 2))
|
||||
|
||||
coalescer.flush()
|
||||
coalescer.flush()
|
||||
|
||||
expect(generations).toEqual([1, 2])
|
||||
})
|
||||
|
||||
it('settles every entry as failed on a synchronous send error', () => {
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: () => {
|
||||
throw new Error('send failed')
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
coalescer.flush()
|
||||
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: 'send failed' })
|
||||
})
|
||||
})
|
||||
|
||||
it('promotes a pending interval flush to immediate at the source threshold', () => {
|
||||
const delays: number[] = []
|
||||
const cancelSchedule = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: vi.fn(),
|
||||
schedule: (_callback, delayMs) => {
|
||||
delays.push(delayMs)
|
||||
return delays.length as unknown as ReturnType<typeof setTimeout>
|
||||
},
|
||||
cancelSchedule
|
||||
})
|
||||
|
||||
coalescer.enqueue(publication(1, 1))
|
||||
coalescer.enqueue(publication(1, 64 * 1024))
|
||||
|
||||
expect(delays).toEqual([8, 0])
|
||||
expect(cancelSchedule).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('fails queued callbacks exactly once during cleanup', () => {
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
|
||||
coalescer.dispose()
|
||||
coalescer.dispose()
|
||||
expect(settled).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('owns an in-flight callback until dispose and ignores its late transport callback', () => {
|
||||
let transportSettle!: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, _batch, settle) => {
|
||||
transportSettle = settle
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
coalescer.flush()
|
||||
|
||||
coalescer.dispose('generation closed')
|
||||
transportSettle({ ok: true })
|
||||
expect(settled).toHaveBeenCalledOnce()
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: 'generation closed' })
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,230 @@
|
||||
import {
|
||||
MAX_PTY_ACK_ENTRIES,
|
||||
type PtySourceCreditAckBatch
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export const SSH_PTY_ACK_FLUSH_MS = 8
|
||||
export const SSH_PTY_ACK_EAGER_ADVANCE_SU = 64 * 1024
|
||||
const ACK_PUBLICATION_WATERMARK_LIMIT = 1024
|
||||
|
||||
type AckSettlement = { ok: true } | { ok: false; error: Error }
|
||||
|
||||
type CoalescedEntry = {
|
||||
publication: SshPtySourceAckPublication
|
||||
members: SshPtySourceAckPublication[]
|
||||
}
|
||||
|
||||
type InFlightBatch = {
|
||||
entries: CoalescedEntry[]
|
||||
settled: boolean
|
||||
}
|
||||
|
||||
export type SshPtySourceAckCoalescerOptions = {
|
||||
publish: (
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: AckSettlement) => void
|
||||
) => void
|
||||
schedule?: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>
|
||||
cancelSchedule?: (timer: ReturnType<typeof setTimeout>) => void
|
||||
onTokenClosed?: (identity: SshPtySourceAckPublication['identity']) => void
|
||||
}
|
||||
|
||||
function ackKey(publication: SshPtySourceAckPublication): string {
|
||||
const { identity } = publication
|
||||
return `${identity.providerGeneration}\0${identity.clientGeneration}\0${identity.ownerGeneration}\0${identity.id}\0${identity.ptyIncarnation}\0${identity.deliveryToken}`
|
||||
}
|
||||
|
||||
export class SshPtySourceAckCoalescer {
|
||||
private readonly pending = new Map<string, CoalescedEntry>()
|
||||
private readonly lastPublishedEndByToken = new Map<string, number>()
|
||||
private readonly schedule: NonNullable<SshPtySourceAckCoalescerOptions['schedule']>
|
||||
private readonly cancelSchedule: NonNullable<SshPtySourceAckCoalescerOptions['cancelSchedule']>
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
private timerDelayMs: number | null = null
|
||||
private inFlight: InFlightBatch | null = null
|
||||
private disposed = false
|
||||
|
||||
constructor(private readonly options: SshPtySourceAckCoalescerOptions) {
|
||||
this.schedule = options.schedule ?? ((callback, delayMs) => setTimeout(callback, delayMs))
|
||||
this.cancelSchedule = options.cancelSchedule ?? clearTimeout
|
||||
}
|
||||
|
||||
enqueue(publication: SshPtySourceAckPublication): void {
|
||||
if (this.disposed) {
|
||||
publication.onSettled({ ok: false, error: new Error('SSH PTY ACK coalescer disposed') })
|
||||
return
|
||||
}
|
||||
const key = ackKey(publication)
|
||||
const current = this.pending.get(key)
|
||||
if (!current) {
|
||||
this.pending.set(key, { publication, members: [publication] })
|
||||
} else {
|
||||
current.members.push(publication)
|
||||
if (publication.ack.creditedEndSu > current.publication.ack.creditedEndSu) {
|
||||
current.publication = publication
|
||||
}
|
||||
}
|
||||
const lastPublished = this.lastPublishedEndByToken.get(key) ?? 0
|
||||
const eager = publication.ack.creditedEndSu - lastPublished >= SSH_PTY_ACK_EAGER_ADVANCE_SU
|
||||
this.requestFlush(eager ? 0 : SSH_PTY_ACK_FLUSH_MS)
|
||||
}
|
||||
|
||||
flush(): void {
|
||||
if (this.disposed || this.inFlight || this.pending.size === 0) {
|
||||
return
|
||||
}
|
||||
if (this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
}
|
||||
const providerGeneration = this.pending.values().next().value!.publication
|
||||
.identity.providerGeneration
|
||||
const selected = Array.from(this.pending.entries())
|
||||
.filter(([, entry]) => entry.publication.identity.providerGeneration === providerGeneration)
|
||||
.slice(0, MAX_PTY_ACK_ENTRIES)
|
||||
for (const [key] of selected) {
|
||||
this.pending.delete(key)
|
||||
}
|
||||
const batch: InFlightBatch = {
|
||||
entries: selected.map(([, entry]) => entry),
|
||||
settled: false
|
||||
}
|
||||
this.inFlight = batch
|
||||
const settle = (result: AckSettlement): void => this.settleBatch(batch, result)
|
||||
try {
|
||||
this.options.publish(
|
||||
providerGeneration,
|
||||
Object.freeze({
|
||||
acknowledgements: Object.freeze(batch.entries.map((entry) => entry.publication.ack))
|
||||
}),
|
||||
settle
|
||||
)
|
||||
} catch (error) {
|
||||
settle({
|
||||
ok: false,
|
||||
error: error instanceof Error ? error : new Error(String(error))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
dispose(reason = 'SSH PTY ACK coalescer disposed'): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
if (this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
}
|
||||
const result = { ok: false as const, error: new Error(reason) }
|
||||
for (const entry of this.pending.values()) {
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
this.pending.clear()
|
||||
if (this.inFlight && !this.inFlight.settled) {
|
||||
const batch = this.inFlight
|
||||
batch.settled = true
|
||||
this.inFlight = null
|
||||
for (const entry of batch.entries) {
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cancelGeneration(providerGeneration: number, reason: string): void {
|
||||
const result = { ok: false as const, error: new Error(reason) }
|
||||
for (const [key, entry] of this.pending) {
|
||||
if (entry.publication.identity.providerGeneration === providerGeneration) {
|
||||
this.pending.delete(key)
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
}
|
||||
const batch = this.inFlight
|
||||
if (batch && !batch.settled) {
|
||||
const retained: CoalescedEntry[] = []
|
||||
for (const entry of batch.entries) {
|
||||
if (entry.publication.identity.providerGeneration === providerGeneration) {
|
||||
this.settleMembers(entry, result)
|
||||
} else {
|
||||
retained.push(entry)
|
||||
}
|
||||
}
|
||||
batch.entries = retained
|
||||
if (retained.length === 0) {
|
||||
batch.settled = true
|
||||
this.inFlight = null
|
||||
}
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.lastPublishedEndByToken.keys()) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.lastPublishedEndByToken.delete(key)
|
||||
}
|
||||
}
|
||||
if (!this.inFlight && this.pending.size === 0 && this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
} else if (!this.inFlight && this.pending.size > 0) {
|
||||
this.requestFlush(0)
|
||||
}
|
||||
}
|
||||
|
||||
get pendingCount(): number {
|
||||
return this.pending.size
|
||||
}
|
||||
|
||||
private settleBatch(batch: InFlightBatch, result: AckSettlement): void {
|
||||
if (batch.settled) {
|
||||
return
|
||||
}
|
||||
batch.settled = true
|
||||
if (this.inFlight === batch) {
|
||||
this.inFlight = null
|
||||
}
|
||||
for (const entry of batch.entries) {
|
||||
if (result.ok) {
|
||||
const key = ackKey(entry.publication)
|
||||
this.lastPublishedEndByToken.delete(key)
|
||||
this.lastPublishedEndByToken.set(key, entry.publication.ack.creditedEndSu)
|
||||
while (this.lastPublishedEndByToken.size > ACK_PUBLICATION_WATERMARK_LIMIT) {
|
||||
this.lastPublishedEndByToken.delete(this.lastPublishedEndByToken.keys().next().value!)
|
||||
}
|
||||
}
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
if (this.pending.size > 0) {
|
||||
this.requestFlush(0)
|
||||
}
|
||||
}
|
||||
|
||||
private settleMembers(entry: CoalescedEntry, result: AckSettlement): void {
|
||||
for (const publication of entry.members.splice(0)) {
|
||||
publication.onSettled(result)
|
||||
}
|
||||
}
|
||||
|
||||
private requestFlush(delayMs: number): void {
|
||||
if (this.inFlight || this.disposed) {
|
||||
return
|
||||
}
|
||||
if (this.timer) {
|
||||
if (this.timerDelayMs !== null && this.timerDelayMs <= delayMs) {
|
||||
return
|
||||
}
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
this.timerDelayMs = delayMs
|
||||
this.timer = this.schedule(() => {
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
this.flush()
|
||||
}, delayMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
|
||||
import { reclaimPublishedSourcePrefix, type TokenRecord } from './ssh-pty-source-obligation-state'
|
||||
|
||||
export function createSshPtySourceAckPublication(
|
||||
token: TokenRecord,
|
||||
endSu: number,
|
||||
spanOwners: Map<string, TokenRecord>,
|
||||
onPublished: () => void
|
||||
): SshPtySourceAckPublication {
|
||||
let settled = false
|
||||
return Object.freeze({
|
||||
identity: token.identity,
|
||||
ack: Object.freeze({
|
||||
id: token.identity.id,
|
||||
clientGeneration: token.identity.clientGeneration,
|
||||
ownerGeneration: token.identity.ownerGeneration,
|
||||
deliveryToken: token.identity.deliveryToken,
|
||||
creditedEndSu: endSu
|
||||
}),
|
||||
onSettled: (result) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
if (!result.ok || token.state === 'closed' || endSu > token.ackQueuedEndSu) {
|
||||
return
|
||||
}
|
||||
token.ackPublishedEndSu = Math.max(token.ackPublishedEndSu, endSu)
|
||||
reclaimPublishedSourcePrefix(token, spanOwners)
|
||||
onPublished()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import {
|
||||
installSshPtySourceAckPublisher,
|
||||
publishSshPtySourceAck
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
describe('SSH PTY intake to relay ACK contract', () => {
|
||||
it('publishes one cumulative relay-ID ACK only after model and desktop settlement', async () => {
|
||||
vi.useFakeTimers()
|
||||
const batches: PtySourceCreditAckBatch[] = []
|
||||
const cleanup = installSshPtySourceAckPublisher(7, (batch, onSettled) => {
|
||||
batches.push(batch)
|
||||
onSettled({ ok: true })
|
||||
})
|
||||
let sequence = 0
|
||||
const intake = new SshPtyOutputIntake({
|
||||
getModelSequence: () => sequence,
|
||||
acceptModel: (event) => {
|
||||
sequence += event.rawLength
|
||||
return { sequence, completion: Promise.resolve() }
|
||||
},
|
||||
project: () => {},
|
||||
prepareExit: () => {},
|
||||
finalizeExit: () => {},
|
||||
publishSourceAck: publishSshPtySourceAck
|
||||
})
|
||||
|
||||
try {
|
||||
const receipt = await intake.acceptData({
|
||||
id: 'ssh:target@@relay-pty-1',
|
||||
data: 'data',
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
source: {
|
||||
relayPtyId: 'relay-pty-1',
|
||||
spanId: 'token-1:0:4',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(8)
|
||||
expect(batches).toHaveLength(0)
|
||||
|
||||
const projectionId = receipt.projection.identity.projectionSemanticsId
|
||||
intake.publishProjectionPrefix([projectionId], 4, 4)
|
||||
intake.settleProjectionPrefix('ssh:target@@relay-pty-1', 4)
|
||||
await vi.advanceTimersByTimeAsync(8)
|
||||
|
||||
expect(batches).toEqual([
|
||||
{
|
||||
acknowledgements: [
|
||||
{
|
||||
id: 'relay-pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
creditedEndSu: 4
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
await intake.acceptExit({
|
||||
id: 'ssh:target@@relay-pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
expect(intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
} finally {
|
||||
intake.dispose()
|
||||
cleanup()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import type {
|
||||
PtySourceCreditAck,
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
|
||||
export type SshPtySourceConsumerId = 'model' | 'desktop' | `remote:${string}`
|
||||
|
||||
export type SshPtySourceObligationState =
|
||||
| Readonly<{ state: 'open' }>
|
||||
| Readonly<{ state: 'transferring'; to: SshPtySourceConsumerId; reason: string }>
|
||||
| Readonly<{ state: 'settled'; reason: string }>
|
||||
| Readonly<{ state: 'transferred'; to: SshPtySourceConsumerId; reason: string }>
|
||||
| Readonly<{ state: 'canceled'; reason: string }>
|
||||
|
||||
export type SshPtySourceAdmissionReservation = Readonly<{
|
||||
reservationId: string
|
||||
span: PtySourceSpan
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
}>
|
||||
|
||||
export type SshPtySourceTokenSnapshot = PtySourceDeliveryIdentity &
|
||||
Readonly<{
|
||||
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
|
||||
receivedEndSu: number
|
||||
obligationsTerminalEndSu: number
|
||||
ackQueuedEndSu: number
|
||||
ackPublishedEndSu: number
|
||||
openSpans: number
|
||||
exitPublished: boolean
|
||||
generationClosed: boolean
|
||||
}>
|
||||
|
||||
export type SshPtySourceAckPublication = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
ack: PtySourceCreditAck
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}>
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
PtySourceCreditAckBatch,
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
const identity: PtySourceDeliveryIdentity = {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1'
|
||||
}
|
||||
|
||||
const span: PtySourceSpan = {
|
||||
...identity,
|
||||
spanId: 'span-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
data: 'data',
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
|
||||
}
|
||||
|
||||
describe('SshPtySourceObligationCoordinator', () => {
|
||||
it('is the single boundary from exact consumer settlement to upstream ACK publication', () => {
|
||||
let written:
|
||||
| {
|
||||
batch: PtySourceCreditAckBatch
|
||||
settle: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}
|
||||
| undefined
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: (_providerGeneration, batch, settle) => {
|
||||
written = { batch, settle }
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
const reservation = coordinator.reserve(identity, span, ['model', 'desktop'])
|
||||
coordinator.commit(reservation)
|
||||
coordinator.settle({
|
||||
identity,
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'emulator-receipt'
|
||||
})
|
||||
coordinator.settle({
|
||||
identity,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop',
|
||||
reason: 'renderer-parse'
|
||||
})
|
||||
coordinator.flushAcknowledgements()
|
||||
|
||||
expect(written?.batch.acknowledgements).toEqual([
|
||||
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 4 })
|
||||
])
|
||||
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(0)
|
||||
written?.settle({ ok: true })
|
||||
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(4)
|
||||
})
|
||||
|
||||
it('rejects an adapter transition carrying a stale full delivery identity', () => {
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.commit(coordinator.reserve(identity, span, ['model']))
|
||||
|
||||
expect(() =>
|
||||
coordinator.settle({
|
||||
identity: { ...identity, ownerGeneration: 99 },
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'stale'
|
||||
})
|
||||
).toThrow('stale')
|
||||
})
|
||||
|
||||
it('keeps later generations publishable after closing one generation', () => {
|
||||
const publish = vi.fn()
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish,
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.closeGeneration(1, 'replaced')
|
||||
const nextIdentity = { ...identity, providerGeneration: 2, deliveryToken: 'token-2' }
|
||||
const nextSpan = { ...span, ...nextIdentity, spanId: 'span-2' }
|
||||
coordinator.open(nextIdentity)
|
||||
coordinator.commit(coordinator.reserve(nextIdentity, nextSpan, ['model']))
|
||||
coordinator.settle({
|
||||
identity: nextIdentity,
|
||||
spanId: nextSpan.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'accepted'
|
||||
})
|
||||
coordinator.flushAcknowledgements()
|
||||
|
||||
expect(publish).toHaveBeenCalledWith(
|
||||
2,
|
||||
{
|
||||
acknowledgements: [expect.objectContaining({ deliveryToken: 'token-2', creditedEndSu: 4 })]
|
||||
},
|
||||
expect.any(Function)
|
||||
)
|
||||
})
|
||||
|
||||
it('generation-closes every retained token on coordinator disposal', () => {
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.commit(coordinator.reserve(identity, span, ['model']))
|
||||
|
||||
coordinator.dispose('provider disposed')
|
||||
|
||||
expect(coordinator.snapshot(identity)).toMatchObject({
|
||||
state: 'closed',
|
||||
generationClosed: true,
|
||||
openSpans: 0
|
||||
})
|
||||
expect(() => coordinator.open({ ...identity, deliveryToken: 'late-token' })).toThrow('disposed')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,267 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
SshPtySourceAckCoalescer,
|
||||
type SshPtySourceAckCoalescerOptions
|
||||
} from './ssh-pty-source-ack-coalescer'
|
||||
import {
|
||||
SshPtySourceObligationLedger,
|
||||
type SshPtySourceAdmissionReservation,
|
||||
type SshPtySourceConsumerId,
|
||||
type SshPtySourceObligationState,
|
||||
type SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-ledger'
|
||||
|
||||
export type SshPtySourceObligationTransition = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
spanId: string
|
||||
consumer: SshPtySourceConsumerId
|
||||
reason: string
|
||||
}>
|
||||
|
||||
type TerminalWaiter = {
|
||||
resolve: () => void
|
||||
reject: (error: Error) => void
|
||||
}
|
||||
|
||||
type TerminalWaiterGroup = {
|
||||
identity: PtySourceDeliveryIdentity
|
||||
waiters: Set<TerminalWaiter>
|
||||
}
|
||||
|
||||
export class SshPtySourceObligationCoordinator {
|
||||
private readonly ledger: SshPtySourceObligationLedger
|
||||
private readonly acknowledgements: SshPtySourceAckCoalescer
|
||||
private readonly terminalWaiters = new Map<string, TerminalWaiterGroup>()
|
||||
private disposed = false
|
||||
|
||||
constructor(options: SshPtySourceAckCoalescerOptions) {
|
||||
this.ledger = new SshPtySourceObligationLedger(options.onTokenClosed)
|
||||
this.acknowledgements = new SshPtySourceAckCoalescer(options)
|
||||
}
|
||||
|
||||
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
|
||||
if (this.disposed) {
|
||||
throw new Error('SSH PTY source obligation coordinator is disposed')
|
||||
}
|
||||
this.ledger.open(identity, checkpointSourceEndSu)
|
||||
}
|
||||
|
||||
reserve(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
span: PtySourceSpan,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
): SshPtySourceAdmissionReservation {
|
||||
return this.ledger.reserve(identity, span, requiredConsumers)
|
||||
}
|
||||
|
||||
commit(reservation: SshPtySourceAdmissionReservation): void {
|
||||
this.ledger.commit(reservation)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
return this.ledger.rollback(reservation)
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const rolledBack = this.ledger.rollbackCommitted(reservation)
|
||||
if (rolledBack) {
|
||||
this.maybeResolveTerminal(reservation.span)
|
||||
}
|
||||
return rolledBack
|
||||
}
|
||||
|
||||
settle(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.settle(transition.spanId, transition.consumer, transition.reason)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
beginTransfer(transition: SshPtySourceObligationTransition, to: SshPtySourceConsumerId): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
return this.ledger.beginTransfer(transition.spanId, transition.consumer, to, transition.reason)
|
||||
}
|
||||
|
||||
commitTransfer(transition: Omit<SshPtySourceObligationTransition, 'reason'>): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.commitTransfer(transition.spanId, transition.consumer)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
cancelTransfer(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.cancelTransfer(
|
||||
transition.spanId,
|
||||
transition.consumer,
|
||||
transition.reason
|
||||
)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
rollbackTransfer(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
return this.ledger.rollbackTransfer(transition.spanId, transition.consumer)
|
||||
}
|
||||
|
||||
seal(identity: PtySourceDeliveryIdentity): void {
|
||||
if (this.ledger.snapshot(identity).state === 'sealed-unsettled') {
|
||||
return
|
||||
}
|
||||
this.ledger.seal(identity)
|
||||
}
|
||||
|
||||
markExitPublished(identity: PtySourceDeliveryIdentity): void {
|
||||
this.queueEligibleAck(identity)
|
||||
this.ledger.markExitPublished(identity)
|
||||
}
|
||||
|
||||
whenTerminal(identity: PtySourceDeliveryIdentity): Promise<void> {
|
||||
if (this.isTerminal(identity)) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
let group = this.terminalWaiters.get(key)
|
||||
if (!group) {
|
||||
group = { identity: Object.freeze({ ...identity }), waiters: new Set() }
|
||||
this.terminalWaiters.set(key, group)
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
group!.waiters.add({ resolve, reject })
|
||||
})
|
||||
}
|
||||
|
||||
beginExitTimeout(identity: PtySourceDeliveryIdentity) {
|
||||
return this.ledger.beginExitTimeout(identity)
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const snapshot = this.ledger.snapshot(identity)
|
||||
if (
|
||||
snapshot.state === 'closed' &&
|
||||
snapshot.receivedEndSu === proof.sentEndSu &&
|
||||
snapshot.ackPublishedEndSu === proof.creditedEndSu
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.ledger.applyCancellationProof(identity, proof)
|
||||
this.rejectWaiters(
|
||||
(group) => samePtySourceDelivery(group.identity, identity),
|
||||
new Error('ssh_source_delivery_canceled')
|
||||
)
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
this.ledger.applyRecoveryCancellationProof(identity, proof)
|
||||
this.rejectWaiters(
|
||||
(group) => samePtySourceDelivery(group.identity, identity),
|
||||
new Error('ssh_source_delivery_canceled')
|
||||
)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): number {
|
||||
this.rejectWaiters(
|
||||
(group) => group.identity.providerGeneration === providerGeneration,
|
||||
new Error(reason)
|
||||
)
|
||||
const closed = this.ledger.closeGeneration(providerGeneration, reason)
|
||||
this.acknowledgements.cancelGeneration(providerGeneration, reason)
|
||||
return closed
|
||||
}
|
||||
|
||||
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
|
||||
return this.ledger.snapshot(identity)
|
||||
}
|
||||
|
||||
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
|
||||
return this.ledger.modelAcceptedEnd(identity)
|
||||
}
|
||||
|
||||
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
|
||||
return this.ledger.obligation(spanId, consumer)
|
||||
}
|
||||
|
||||
spanIdentity(spanId: string): PtySourceSpan {
|
||||
return this.ledger.spanIdentity(spanId)
|
||||
}
|
||||
|
||||
hasRetainedSpan(spanId: string): boolean {
|
||||
return this.ledger.hasRetainedSpan(spanId)
|
||||
}
|
||||
|
||||
flushAcknowledgements(): void {
|
||||
this.acknowledgements.flush()
|
||||
}
|
||||
|
||||
dispose(reason?: string): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
this.rejectWaiters(() => true, new Error(reason ?? 'SSH PTY source obligations disposed'))
|
||||
this.ledger.closeAll(reason ?? 'SSH PTY source obligation coordinator disposed')
|
||||
this.acknowledgements.dispose(reason)
|
||||
}
|
||||
|
||||
private queueEligibleAck(identity: PtySourceDeliveryIdentity): void {
|
||||
const publication = this.ledger.queueAck(identity)
|
||||
if (publication) {
|
||||
this.acknowledgements.enqueue(publication)
|
||||
}
|
||||
this.maybeResolveTerminal(identity)
|
||||
}
|
||||
|
||||
private isTerminal(identity: PtySourceDeliveryIdentity): boolean {
|
||||
const snapshot = this.ledger.snapshot(identity)
|
||||
return (
|
||||
snapshot.obligationsTerminalEndSu === snapshot.receivedEndSu &&
|
||||
snapshot.ackQueuedEndSu === snapshot.receivedEndSu
|
||||
)
|
||||
}
|
||||
|
||||
private maybeResolveTerminal(identity: PtySourceDeliveryIdentity): void {
|
||||
if (!this.isTerminal(identity)) {
|
||||
return
|
||||
}
|
||||
const group = this.terminalWaiters.get(ptySourceDeliveryKey(identity))
|
||||
if (!group) {
|
||||
return
|
||||
}
|
||||
this.terminalWaiters.delete(ptySourceDeliveryKey(identity))
|
||||
for (const waiter of group.waiters) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
private rejectWaiters(predicate: (group: TerminalWaiterGroup) => boolean, error: Error): void {
|
||||
for (const [key, group] of this.terminalWaiters) {
|
||||
if (!predicate(group)) {
|
||||
continue
|
||||
}
|
||||
this.terminalWaiters.delete(key)
|
||||
for (const waiter of group.waiters) {
|
||||
waiter.reject(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private requireSpanIdentity(
|
||||
transition: Pick<SshPtySourceObligationTransition, 'identity' | 'spanId'>
|
||||
): void {
|
||||
if (!samePtySourceDelivery(this.ledger.spanIdentity(transition.spanId), transition.identity)) {
|
||||
throw new Error('SSH PTY source obligation transition has a stale delivery identity')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceObligationLedger } from './ssh-pty-source-obligation-ledger'
|
||||
|
||||
function identity(
|
||||
deliveryToken = 'token-1',
|
||||
overrides: Partial<PtySourceDeliveryIdentity> = {}
|
||||
): PtySourceDeliveryIdentity {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function span(
|
||||
owner: PtySourceDeliveryIdentity,
|
||||
spanId: string,
|
||||
sourceStartSu: number,
|
||||
data: string
|
||||
): PtySourceSpan {
|
||||
return Object.freeze({
|
||||
...owner,
|
||||
spanId,
|
||||
sourceStartSu,
|
||||
sourceEndSu: sourceStartSu + data.length,
|
||||
displayStart: sourceStartSu,
|
||||
displayEnd: sourceStartSu + data.length,
|
||||
data,
|
||||
splittable: true,
|
||||
transform: Object.freeze({
|
||||
transformed: false,
|
||||
rawLengthSu: data.length,
|
||||
scalarSafe: true
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function commitSpan(
|
||||
ledger: SshPtySourceObligationLedger,
|
||||
owner: PtySourceDeliveryIdentity,
|
||||
sourceSpan: PtySourceSpan,
|
||||
consumers: ('model' | 'desktop')[] = ['model', 'desktop']
|
||||
) {
|
||||
const reservation = ledger.reserve(owner, sourceSpan, consumers)
|
||||
ledger.commit(reservation)
|
||||
return reservation
|
||||
}
|
||||
|
||||
describe('SshPtySourceObligationLedger', () => {
|
||||
it('rolls back an uncommitted admission without consuming its source coordinate', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const first = ledger.reserve(owner, span(owner, 'span-1', 0, 'abc'), ['model'])
|
||||
|
||||
expect(ledger.rollback(first)).toBe(true)
|
||||
const retry = ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model'])
|
||||
ledger.commit(retry)
|
||||
expect(ledger.snapshot(owner).receivedEndSu).toBe(3)
|
||||
})
|
||||
|
||||
it('rolls back a committed tail while every obligation is still open', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const reservation = commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
|
||||
|
||||
expect(ledger.rollbackCommitted(reservation)).toBe(true)
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ receivedEndSu: 0, openSpans: 0 })
|
||||
expect(() => ledger.spanIdentity('span-1')).toThrow('Unknown or reclaimed')
|
||||
expect(
|
||||
ledger.commit(ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model']))
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps terminal, queued, and successfully published ACK ends independent', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 0,
|
||||
ackQueuedEndSu: 0,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
ledger.settle('span-1', 'desktop', 'renderer-parse')
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 4,
|
||||
ackQueuedEndSu: 0,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
const publication = ledger.queueAck(owner)!
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 4,
|
||||
ackQueuedEndSu: 4,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
publication.onSettled({ ok: false, error: new Error('write callback failed') })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackQueuedEndSu: 4, ackPublishedEndSu: 0 })
|
||||
ledger.retryQueuedAck(owner)!.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 0 })
|
||||
})
|
||||
|
||||
it('checkpoints only the contiguous model-settled prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
|
||||
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
|
||||
ledger.settle('span-2', 'model', 'out-of-order')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
|
||||
})
|
||||
|
||||
it('continues checkpoints from the reclaimed ACK-published prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
ledger.settle('span-1', 'desktop', 'renderer-parse')
|
||||
ledger.queueAck(owner)!.onSettled({ ok: true })
|
||||
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 1 })
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(4)
|
||||
ledger.settle('span-2', 'model', 'emulator-receipt')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
|
||||
})
|
||||
|
||||
it('requires an exact transfer fence before a desktop obligation becomes terminal', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
|
||||
expect(ledger.beginTransfer('span-1', 'desktop', 'model', 'renderer-send-failed')).toBe(true)
|
||||
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(0)
|
||||
expect(ledger.commitTransfer('span-1', 'desktop')).toBe(true)
|
||||
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(3)
|
||||
expect(ledger.obligation('span-1', 'desktop')).toMatchObject({
|
||||
state: 'transferred',
|
||||
to: 'model'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps sealed exit state until final ACK publication succeeds', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
expect(() => ledger.markExitPublished(owner)).toThrow('terminal ACK')
|
||||
ledger.settle('tail', 'model', 'emulator-receipt')
|
||||
const publication = ledger.queueAck(owner)!
|
||||
ledger.markExitPublished(owner)
|
||||
expect(ledger.snapshot(owner).state).toBe('sealed-unsettled')
|
||||
publication.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner).state).toBe('closed')
|
||||
})
|
||||
|
||||
it('publishes token cancellation intent before accepting timeout cleanup proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })).toThrow()
|
||||
expect(ledger.beginExitTimeout(owner)).toEqual({
|
||||
id: owner.id,
|
||||
deliveryToken: owner.deliveryToken,
|
||||
clientGeneration: owner.clientGeneration,
|
||||
ownerGeneration: owner.ownerGeneration
|
||||
})
|
||||
ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('ignores a late successful write callback after generation-close proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'), ['model'])
|
||||
ledger.settle('span-1', 'model', 'accepted')
|
||||
const publication = ledger.queueAck(owner)!
|
||||
|
||||
expect(ledger.closeGeneration(1, 'provider-closed')).toBe(1)
|
||||
publication.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
state: 'closed',
|
||||
generationClosed: true,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
expect(() => ledger.obligation('span-1', 'model')).toThrow('reclaimed')
|
||||
})
|
||||
|
||||
it('rejects stale generations, tokens, and non-contiguous source spans', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
expect(() =>
|
||||
ledger.reserve(identity('stale'), span(identity('stale'), 'stale', 0, 'x'), ['model'])
|
||||
).toThrow('stale')
|
||||
expect(() => ledger.reserve(owner, span(owner, 'gap', 1, 'x'), ['model'])).toThrow(
|
||||
'non-contiguous'
|
||||
)
|
||||
})
|
||||
|
||||
it('requires cancellation proof to match the exact local received and published ends', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
ledger.beginExitTimeout(owner)
|
||||
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 3, creditedEndSu: 0 })).toThrow(
|
||||
'invalid'
|
||||
)
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 1 })).toThrow(
|
||||
'invalid'
|
||||
)
|
||||
})
|
||||
|
||||
it('applies recovery cancellation proof over a locally admitted prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner, 4)
|
||||
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
|
||||
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 12, creditedEndSu: 4 })
|
||||
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('rejects recovery cancellation proof that misses local intake state', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner, 4)
|
||||
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
|
||||
|
||||
expect(() =>
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 7, creditedEndSu: 4 })
|
||||
).toThrow('invalid')
|
||||
expect(() =>
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 8, creditedEndSu: 5 })
|
||||
).toThrow('invalid')
|
||||
})
|
||||
|
||||
it('bounds closed-token tombstones and removes committed reservation indexes', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owners = Array.from({ length: 300 }, (_, index) =>
|
||||
identity(`token-${index}`, {
|
||||
id: `pty-${index}`,
|
||||
ptyIncarnation: `incarnation-${index}`
|
||||
})
|
||||
)
|
||||
for (const [index, owner] of owners.entries()) {
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, `span-${index}`, 0, 'x'), ['model'])
|
||||
ledger.closeGeneration(1, 'generation-closed')
|
||||
}
|
||||
|
||||
expect(() => ledger.snapshot(owners[0])).toThrow('stale')
|
||||
expect(ledger.snapshot(owners.at(-1)!)).toMatchObject({ state: 'closed' })
|
||||
})
|
||||
|
||||
it('reclaims uncommitted reservations on exact cancellation proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const reservation = ledger.reserve(owner, span(owner, 'pending', 0, 'x'), ['model'])
|
||||
ledger.seal(owner)
|
||||
ledger.beginExitTimeout(owner)
|
||||
|
||||
ledger.applyCancellationProof(owner, { sentEndSu: 0, creditedEndSu: 0 })
|
||||
|
||||
expect(ledger.rollback(reservation)).toBe(false)
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('rejects reopening a recently closed one-use token', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
ledger.closeGeneration(owner.providerGeneration, 'closed')
|
||||
|
||||
expect(() => ledger.open(owner)).toThrow('already used')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,329 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
assertNonNegativeSafeInteger,
|
||||
assertPtySourceIdentity,
|
||||
assertPtySourceSpan
|
||||
} from '../../shared/pty-source-credit-validation'
|
||||
import type {
|
||||
SshPtySourceAckPublication,
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import { createSshPtySourceAckPublication } from './ssh-pty-source-ack-publication'
|
||||
import {
|
||||
beginSourceExitTimeout,
|
||||
cancelOpenSourceObligations,
|
||||
closeAllSourceTokens,
|
||||
closeSourceGeneration,
|
||||
closeSourceToken,
|
||||
createSourceSpanRecord,
|
||||
createSourceToken,
|
||||
markSourceExitPublished,
|
||||
requireSourceSpan,
|
||||
rollbackCommittedSourceSpan,
|
||||
sealSourceToken,
|
||||
snapshotSourceToken,
|
||||
type ReservationRecord,
|
||||
type TokenRecord
|
||||
} from './ssh-pty-source-obligation-state'
|
||||
import {
|
||||
applySourceRecoveryCancellationProof,
|
||||
cancelSourceObligationTransfer,
|
||||
commitSourceObligationTransfer,
|
||||
modelAcceptedSourceEnd,
|
||||
rollbackSourceObligationTransfer,
|
||||
transitionOpenSourceObligation
|
||||
} from './ssh-pty-source-obligation-transitions'
|
||||
|
||||
export type {
|
||||
SshPtySourceAckPublication,
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export class SshPtySourceObligationLedger {
|
||||
private readonly tokens = new Map<string, TokenRecord>()
|
||||
private readonly closedSnapshots = new Map<string, SshPtySourceTokenSnapshot>()
|
||||
private readonly reservations = new Map<string, ReservationRecord>()
|
||||
private readonly spanOwners = new Map<string, TokenRecord>()
|
||||
private nextReservationId = 1
|
||||
|
||||
constructor(
|
||||
private readonly onTokenClosed: (identity: PtySourceDeliveryIdentity) => void = () => {}
|
||||
) {}
|
||||
|
||||
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
|
||||
assertPtySourceIdentity(identity)
|
||||
assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu')
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
if (this.tokens.has(key) || this.closedSnapshots.has(key)) {
|
||||
throw new Error('SSH PTY source token was already used')
|
||||
}
|
||||
this.tokens.set(key, createSourceToken(identity, checkpointSourceEndSu))
|
||||
}
|
||||
|
||||
reserve(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
span: PtySourceSpan,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
): SshPtySourceAdmissionReservation {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.state !== 'active') {
|
||||
throw new Error('SSH PTY source token no longer admits data')
|
||||
}
|
||||
assertPtySourceSpan(span)
|
||||
if (
|
||||
!samePtySourceDelivery(token.identity, span) ||
|
||||
span.sourceStartSu !== token.receivedEndSu ||
|
||||
this.spanOwners.has(span.spanId)
|
||||
) {
|
||||
throw new Error('SSH PTY source span is stale, duplicate, or non-contiguous')
|
||||
}
|
||||
const uniqueConsumers = Array.from(new Set(requiredConsumers))
|
||||
if (!uniqueConsumers.includes('model')) {
|
||||
throw new Error('SSH PTY source span requires the terminal model obligation')
|
||||
}
|
||||
const reservation = Object.freeze({
|
||||
reservationId: `ssh-source-admission:${this.nextReservationId++}`,
|
||||
span,
|
||||
requiredConsumers: Object.freeze(uniqueConsumers)
|
||||
})
|
||||
this.reservations.set(reservation.reservationId, {
|
||||
reservation,
|
||||
state: 'reserved'
|
||||
})
|
||||
return reservation
|
||||
}
|
||||
|
||||
commit(reservation: SshPtySourceAdmissionReservation): void {
|
||||
const record = this.requireReservation(reservation)
|
||||
if (record.state !== 'reserved') {
|
||||
throw new Error('SSH PTY source admission reservation is not pending')
|
||||
}
|
||||
const token = this.requireToken(reservation.span)
|
||||
if (token.state !== 'active' || token.receivedEndSu !== reservation.span.sourceStartSu) {
|
||||
throw new Error('SSH PTY source admission reservation became stale')
|
||||
}
|
||||
const spanRecord = createSourceSpanRecord(reservation.span, reservation.requiredConsumers)
|
||||
token.spans.push(spanRecord)
|
||||
token.receivedEndSu = reservation.span.sourceEndSu
|
||||
this.spanOwners.set(reservation.span.spanId, token)
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation || record.state !== 'reserved') {
|
||||
return false
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
return true
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const token = this.tokens.get(ptySourceDeliveryKey(reservation.span))
|
||||
if (!token || !samePtySourceDelivery(token.identity, reservation.span)) {
|
||||
return false
|
||||
}
|
||||
return rollbackCommittedSourceSpan(token, reservation, this.spanOwners)
|
||||
}
|
||||
|
||||
settle(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
|
||||
return this.transitionOpen(spanId, consumer, Object.freeze({ state: 'settled', reason }))
|
||||
}
|
||||
|
||||
beginTransfer(
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
to: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
return this.transitionOpen(
|
||||
spanId,
|
||||
consumer,
|
||||
Object.freeze({ state: 'transferring', to, reason })
|
||||
)
|
||||
}
|
||||
|
||||
commitTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
|
||||
return commitSourceObligationTransfer(this.spanOwners, spanId, consumer)
|
||||
}
|
||||
|
||||
cancelTransfer(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
|
||||
return cancelSourceObligationTransfer(this.spanOwners, spanId, consumer, reason)
|
||||
}
|
||||
|
||||
rollbackTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
|
||||
return rollbackSourceObligationTransfer(this.spanOwners, spanId, consumer)
|
||||
}
|
||||
|
||||
queueAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.obligationsTerminalEndSu <= token.ackQueuedEndSu) {
|
||||
return null
|
||||
}
|
||||
const endSu = token.obligationsTerminalEndSu
|
||||
token.ackQueuedEndSu = endSu
|
||||
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
|
||||
this.maybeClose(token)
|
||||
)
|
||||
}
|
||||
|
||||
retryQueuedAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.ackQueuedEndSu <= token.ackPublishedEndSu) {
|
||||
return null
|
||||
}
|
||||
const endSu = token.ackQueuedEndSu
|
||||
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
|
||||
this.maybeClose(token)
|
||||
)
|
||||
}
|
||||
|
||||
seal(identity: PtySourceDeliveryIdentity): void {
|
||||
sealSourceToken(this.requireToken(identity))
|
||||
}
|
||||
|
||||
markExitPublished(identity: PtySourceDeliveryIdentity): void {
|
||||
const token = this.requireToken(identity)
|
||||
markSourceExitPublished(token)
|
||||
this.maybeClose(token)
|
||||
}
|
||||
|
||||
beginExitTimeout(identity: PtySourceDeliveryIdentity): Readonly<{
|
||||
id: string
|
||||
deliveryToken: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
}> {
|
||||
return beginSourceExitTimeout(this.requireToken(identity))
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const token = this.requireToken(identity)
|
||||
if (
|
||||
token.state !== 'canceling' ||
|
||||
proof.sentEndSu !== token.receivedEndSu ||
|
||||
proof.creditedEndSu !== token.ackPublishedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source cancellation proof is stale or invalid')
|
||||
}
|
||||
cancelOpenSourceObligations(token, 'relay-cancellation-proof')
|
||||
this.closeToken(token)
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const token = this.requireToken(identity)
|
||||
applySourceRecoveryCancellationProof(token, proof)
|
||||
this.closeToken(token)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): number {
|
||||
return closeSourceGeneration(
|
||||
this.tokens,
|
||||
this.reservations,
|
||||
providerGeneration,
|
||||
reason,
|
||||
(token) => this.closeToken(token)
|
||||
)
|
||||
}
|
||||
|
||||
closeAll(reason: string): number {
|
||||
return closeAllSourceTokens(this.tokens, this.reservations, reason, (token) =>
|
||||
this.closeToken(token)
|
||||
)
|
||||
}
|
||||
|
||||
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
const token = this.tokens.get(key)
|
||||
if (token && samePtySourceDelivery(token.identity, identity)) {
|
||||
return snapshotSourceToken(token)
|
||||
}
|
||||
const closed = this.closedSnapshots.get(key)
|
||||
if (closed && samePtySourceDelivery(closed, identity)) {
|
||||
return closed
|
||||
}
|
||||
throw new Error('Unknown or stale SSH PTY source token')
|
||||
}
|
||||
|
||||
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
|
||||
return modelAcceptedSourceEnd(this.requireToken(identity))
|
||||
}
|
||||
|
||||
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
|
||||
const obligation = requireSourceSpan(this.spanOwners, spanId).span.obligations.get(consumer)
|
||||
if (!obligation) {
|
||||
throw new Error('SSH PTY source consumer obligation does not exist')
|
||||
}
|
||||
return obligation
|
||||
}
|
||||
|
||||
spanIdentity(spanId: string): PtySourceSpan {
|
||||
return requireSourceSpan(this.spanOwners, spanId).span.span
|
||||
}
|
||||
|
||||
hasRetainedSpan(spanId: string): boolean {
|
||||
return this.spanOwners.has(spanId)
|
||||
}
|
||||
|
||||
private transitionOpen(
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
next: SshPtySourceObligationState
|
||||
): boolean {
|
||||
return transitionOpenSourceObligation(this.spanOwners, spanId, consumer, next)
|
||||
}
|
||||
|
||||
private maybeClose(token: TokenRecord): void {
|
||||
if (
|
||||
token.state === 'sealed-unsettled' &&
|
||||
token.exitPublished &&
|
||||
token.ackPublishedEndSu === token.receivedEndSu
|
||||
) {
|
||||
this.closeToken(token)
|
||||
}
|
||||
}
|
||||
|
||||
private closeToken(token: TokenRecord): void {
|
||||
closeSourceToken(
|
||||
token,
|
||||
this.tokens,
|
||||
this.closedSnapshots,
|
||||
this.reservations,
|
||||
this.spanOwners,
|
||||
this.onTokenClosed
|
||||
)
|
||||
}
|
||||
|
||||
private requireReservation(reservation: SshPtySourceAdmissionReservation): ReservationRecord {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation) {
|
||||
throw new Error('Unknown SSH PTY source admission reservation')
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
private requireToken(identity: PtySourceDeliveryIdentity): TokenRecord {
|
||||
const token = this.tokens.get(ptySourceDeliveryKey(identity))
|
||||
if (!token || !samePtySourceDelivery(token.identity, identity)) {
|
||||
throw new Error('Unknown or stale SSH PTY source token')
|
||||
}
|
||||
return token
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type {
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export const CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT = 256
|
||||
|
||||
export type SpanRecord = {
|
||||
span: PtySourceSpan
|
||||
obligations: Map<SshPtySourceConsumerId, SshPtySourceObligationState>
|
||||
}
|
||||
|
||||
export type ReservationRecord = {
|
||||
reservation: SshPtySourceAdmissionReservation
|
||||
state: 'reserved' | 'committed' | 'rolled-back'
|
||||
}
|
||||
|
||||
export type TokenRecord = {
|
||||
identity: PtySourceDeliveryIdentity
|
||||
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
|
||||
checkpointSourceEndSu: number
|
||||
receivedEndSu: number
|
||||
obligationsTerminalEndSu: number
|
||||
ackQueuedEndSu: number
|
||||
ackPublishedEndSu: number
|
||||
spans: SpanRecord[]
|
||||
exitPublished: boolean
|
||||
generationClosed: boolean
|
||||
}
|
||||
|
||||
export function createSourceToken(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
checkpointSourceEndSu: number
|
||||
): TokenRecord {
|
||||
return {
|
||||
identity: Object.freeze({ ...identity }),
|
||||
state: 'active',
|
||||
checkpointSourceEndSu,
|
||||
receivedEndSu: checkpointSourceEndSu,
|
||||
obligationsTerminalEndSu: checkpointSourceEndSu,
|
||||
ackQueuedEndSu: checkpointSourceEndSu,
|
||||
ackPublishedEndSu: checkpointSourceEndSu,
|
||||
spans: [],
|
||||
exitPublished: false,
|
||||
generationClosed: false
|
||||
}
|
||||
}
|
||||
|
||||
export function createSourceSpanRecord(
|
||||
span: PtySourceSpan,
|
||||
consumers: readonly SshPtySourceConsumerId[]
|
||||
): SpanRecord {
|
||||
return {
|
||||
span,
|
||||
obligations: new Map(
|
||||
consumers.map((consumer) => [consumer, Object.freeze({ state: 'open' as const })])
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function sealSourceToken(token: TokenRecord): void {
|
||||
if (token.state !== 'active') {
|
||||
throw new Error('SSH PTY source token cannot be sealed from its current state')
|
||||
}
|
||||
token.state = 'sealed-unsettled'
|
||||
}
|
||||
|
||||
export function markSourceExitPublished(token: TokenRecord): void {
|
||||
if (token.state !== 'sealed-unsettled') {
|
||||
throw new Error('SSH PTY source exit publication requires a sealed token')
|
||||
}
|
||||
if (
|
||||
token.obligationsTerminalEndSu !== token.receivedEndSu ||
|
||||
token.ackQueuedEndSu !== token.receivedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source exit cannot publish before terminal ACK queueing')
|
||||
}
|
||||
token.exitPublished = true
|
||||
}
|
||||
|
||||
export function beginSourceExitTimeout(token: TokenRecord): Readonly<{
|
||||
id: string
|
||||
deliveryToken: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
}> {
|
||||
if (token.state !== 'sealed-unsettled') {
|
||||
throw new Error('SSH PTY source exit timeout requires a sealed token')
|
||||
}
|
||||
token.state = 'canceling'
|
||||
return Object.freeze({
|
||||
id: token.identity.id,
|
||||
deliveryToken: token.identity.deliveryToken,
|
||||
clientGeneration: token.identity.clientGeneration,
|
||||
ownerGeneration: token.identity.ownerGeneration
|
||||
})
|
||||
}
|
||||
|
||||
function obligationIsTerminal(obligation: SshPtySourceObligationState): boolean {
|
||||
return (
|
||||
obligation.state === 'settled' ||
|
||||
obligation.state === 'transferred' ||
|
||||
obligation.state === 'canceled'
|
||||
)
|
||||
}
|
||||
|
||||
export function snapshotSourceToken(token: TokenRecord): SshPtySourceTokenSnapshot {
|
||||
return Object.freeze({
|
||||
...token.identity,
|
||||
state: token.state,
|
||||
receivedEndSu: token.receivedEndSu,
|
||||
obligationsTerminalEndSu: token.obligationsTerminalEndSu,
|
||||
ackQueuedEndSu: token.ackQueuedEndSu,
|
||||
ackPublishedEndSu: token.ackPublishedEndSu,
|
||||
openSpans: token.spans.length,
|
||||
exitPublished: token.exitPublished,
|
||||
generationClosed: token.generationClosed
|
||||
})
|
||||
}
|
||||
|
||||
export function advanceSourceTerminalEnd(token: TokenRecord): void {
|
||||
let endSu = token.obligationsTerminalEndSu
|
||||
for (const record of token.spans) {
|
||||
if (record.span.sourceEndSu <= endSu) {
|
||||
continue
|
||||
}
|
||||
if (
|
||||
record.span.sourceStartSu !== endSu ||
|
||||
!Array.from(record.obligations.values()).every(obligationIsTerminal)
|
||||
) {
|
||||
break
|
||||
}
|
||||
endSu = record.span.sourceEndSu
|
||||
}
|
||||
token.obligationsTerminalEndSu = endSu
|
||||
}
|
||||
|
||||
export function cancelOpenSourceObligations(token: TokenRecord, reason: string): void {
|
||||
for (const record of token.spans) {
|
||||
for (const [consumer, obligation] of record.obligations) {
|
||||
if (obligation.state === 'open' || obligation.state === 'transferring') {
|
||||
record.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
|
||||
}
|
||||
}
|
||||
}
|
||||
advanceSourceTerminalEnd(token)
|
||||
}
|
||||
|
||||
export function reclaimPublishedSourcePrefix(
|
||||
token: TokenRecord,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): void {
|
||||
while (token.spans[0]?.span.sourceEndSu <= token.ackPublishedEndSu) {
|
||||
const record = token.spans.shift()!
|
||||
spanOwners.delete(record.span.spanId)
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseSourceTokenSpans(
|
||||
token: TokenRecord,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): void {
|
||||
for (const record of token.spans) {
|
||||
spanOwners.delete(record.span.spanId)
|
||||
}
|
||||
token.spans = []
|
||||
}
|
||||
|
||||
export function releaseSourceTokenReservations(
|
||||
token: TokenRecord,
|
||||
reservations: Map<string, ReservationRecord>
|
||||
): void {
|
||||
for (const [reservationId, record] of reservations) {
|
||||
if (samePtySourceDelivery(record.reservation.span, token.identity)) {
|
||||
record.state = 'rolled-back'
|
||||
reservations.delete(reservationId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function rollbackCommittedSourceSpan(
|
||||
token: TokenRecord,
|
||||
reservation: SshPtySourceAdmissionReservation,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): boolean {
|
||||
const last = token.spans.at(-1)
|
||||
if (
|
||||
last?.span !== reservation.span ||
|
||||
token.receivedEndSu !== reservation.span.sourceEndSu ||
|
||||
token.obligationsTerminalEndSu > reservation.span.sourceStartSu ||
|
||||
token.ackQueuedEndSu > reservation.span.sourceStartSu ||
|
||||
Array.from(last.obligations.values()).some((obligation) => obligation.state !== 'open')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
token.spans.pop()
|
||||
token.receivedEndSu = reservation.span.sourceStartSu
|
||||
spanOwners.delete(reservation.span.spanId)
|
||||
return true
|
||||
}
|
||||
|
||||
export function requireSourceSpan(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string
|
||||
): { token: TokenRecord; span: SpanRecord } {
|
||||
const token = spanOwners.get(spanId)
|
||||
const span = token?.spans.find((candidate) => candidate.span.spanId === spanId)
|
||||
if (!token || !span) {
|
||||
throw new Error('Unknown or reclaimed SSH PTY source span')
|
||||
}
|
||||
return { token, span }
|
||||
}
|
||||
|
||||
export function closeSourceGeneration(
|
||||
tokens: Map<string, TokenRecord>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
providerGeneration: number,
|
||||
reason: string,
|
||||
closeToken: (token: TokenRecord) => void
|
||||
): number {
|
||||
let closed = 0
|
||||
for (const token of Array.from(tokens.values())) {
|
||||
if (token.identity.providerGeneration !== providerGeneration || token.state === 'closed') {
|
||||
continue
|
||||
}
|
||||
token.generationClosed = true
|
||||
cancelOpenSourceObligations(token, reason)
|
||||
closeToken(token)
|
||||
closed++
|
||||
}
|
||||
for (const [id, record] of reservations) {
|
||||
if (
|
||||
record.state === 'reserved' &&
|
||||
record.reservation.span.providerGeneration === providerGeneration
|
||||
) {
|
||||
record.state = 'rolled-back'
|
||||
reservations.delete(id)
|
||||
}
|
||||
}
|
||||
return closed
|
||||
}
|
||||
|
||||
export function closeAllSourceTokens(
|
||||
tokens: Map<string, TokenRecord>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
reason: string,
|
||||
closeToken: (token: TokenRecord) => void
|
||||
): number {
|
||||
let closed = 0
|
||||
const generations = new Set(
|
||||
Array.from(tokens.values(), (token) => token.identity.providerGeneration)
|
||||
)
|
||||
for (const providerGeneration of generations) {
|
||||
closed += closeSourceGeneration(tokens, reservations, providerGeneration, reason, closeToken)
|
||||
}
|
||||
return closed
|
||||
}
|
||||
|
||||
export function closeSourceToken(
|
||||
token: TokenRecord,
|
||||
tokens: Map<string, TokenRecord>,
|
||||
closedSnapshots: Map<string, SshPtySourceTokenSnapshot>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
spanOwners: Map<string, TokenRecord>,
|
||||
onTokenClosed: (identity: PtySourceDeliveryIdentity) => void
|
||||
): void {
|
||||
token.state = 'closed'
|
||||
releaseSourceTokenSpans(token, spanOwners)
|
||||
releaseSourceTokenReservations(token, reservations)
|
||||
const key = ptySourceDeliveryKey(token.identity)
|
||||
tokens.delete(key)
|
||||
closedSnapshots.set(key, snapshotSourceToken(token))
|
||||
while (closedSnapshots.size > CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT) {
|
||||
closedSnapshots.delete(closedSnapshots.keys().next().value!)
|
||||
}
|
||||
onTokenClosed(token.identity)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type {
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import {
|
||||
advanceSourceTerminalEnd,
|
||||
cancelOpenSourceObligations,
|
||||
requireSourceSpan,
|
||||
type TokenRecord
|
||||
} from './ssh-pty-source-obligation-state'
|
||||
|
||||
export function applySourceRecoveryCancellationProof(
|
||||
token: TokenRecord,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
if (
|
||||
token.state !== 'active' ||
|
||||
proof.sentEndSu < token.receivedEndSu ||
|
||||
proof.creditedEndSu !== token.ackPublishedEndSu ||
|
||||
proof.creditedEndSu > token.receivedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source recovery cancellation proof is stale or invalid')
|
||||
}
|
||||
cancelOpenSourceObligations(token, 'relay-recovery-cancellation-proof')
|
||||
}
|
||||
|
||||
export function transitionOpenSourceObligation(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
next: SshPtySourceObligationState
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'open') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, next)
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function commitSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
const current = span.obligations.get(consumer)
|
||||
if (current?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(
|
||||
consumer,
|
||||
Object.freeze({ state: 'transferred', to: current.to, reason: current.reason })
|
||||
)
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function cancelSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function rollbackSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId
|
||||
): boolean {
|
||||
const { span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, Object.freeze({ state: 'open' }))
|
||||
return true
|
||||
}
|
||||
|
||||
export function modelAcceptedSourceEnd(token: TokenRecord): number {
|
||||
let acceptedEndSu = token.ackPublishedEndSu
|
||||
for (const record of token.spans) {
|
||||
if (
|
||||
record.span.sourceStartSu !== acceptedEndSu ||
|
||||
record.obligations.get('model')?.state !== 'settled'
|
||||
) {
|
||||
break
|
||||
}
|
||||
acceptedEndSu = record.span.sourceEndSu
|
||||
}
|
||||
return acceptedEndSu
|
||||
}
|
||||
+132
-45
@@ -11,6 +11,8 @@ const {
|
||||
mockConnectionManager,
|
||||
mockDeployAndLaunchRelay,
|
||||
mockForceStopRelayForTarget,
|
||||
mockAcceptSshPtyOutputData,
|
||||
mockAcceptSshPtyOutputExit,
|
||||
mockMux,
|
||||
mockPtyProvider,
|
||||
mockFsProvider,
|
||||
@@ -49,10 +51,13 @@ const {
|
||||
},
|
||||
mockDeployAndLaunchRelay: vi.fn(),
|
||||
mockForceStopRelayForTarget: vi.fn(),
|
||||
mockAcceptSshPtyOutputData: vi.fn().mockResolvedValue({}),
|
||||
mockAcceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined),
|
||||
mockMux: {
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false),
|
||||
onNotification: vi.fn(),
|
||||
onNotificationByMethod: vi.fn().mockReturnValue(() => {}),
|
||||
onRequest: vi.fn().mockReturnValue(() => {}),
|
||||
onDispose: vi.fn().mockReturnValue(() => {}),
|
||||
request: vi.fn().mockResolvedValue({}),
|
||||
@@ -65,7 +70,8 @@ const {
|
||||
onReplay: vi.fn(),
|
||||
attach: vi.fn(),
|
||||
attachForReconnect: vi.fn().mockResolvedValue({}),
|
||||
shutdown: vi.fn()
|
||||
shutdown: vi.fn(),
|
||||
providerGeneration: 0
|
||||
},
|
||||
mockFsProvider: {},
|
||||
mockGitProvider: {},
|
||||
@@ -100,6 +106,25 @@ vi.mock('electron', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-pty-output-intake-registry', () => ({
|
||||
acceptSshPtyOutputData: mockAcceptSshPtyOutputData,
|
||||
acceptSshPtyOutputExit: mockAcceptSshPtyOutputExit,
|
||||
allocateSshPtyProviderGeneration: (() => {
|
||||
let generation = 0
|
||||
return () => ++generation
|
||||
})(),
|
||||
beginSshPtyOutputGenerationMigration: vi.fn(() => ({
|
||||
byPty: new Map(),
|
||||
completion: Promise.resolve()
|
||||
})),
|
||||
applySshPtySourceCancellationProof: vi.fn().mockReturnValue(false),
|
||||
applySshPtySourceRecoveryCancellationProof: vi.fn().mockReturnValue(false),
|
||||
closeSshPtyOutputGeneration: vi.fn(),
|
||||
getSshPtyAcceptedSourceCheckpoints: vi.fn().mockReturnValue([]),
|
||||
installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}),
|
||||
installSshPtySourceCancellationPublisher: vi.fn().mockReturnValue(() => {})
|
||||
}))
|
||||
|
||||
vi.mock('../ssh/ssh-connection-store', () => ({
|
||||
SshConnectionStore: class MockSshConnectionStore {
|
||||
constructor() {
|
||||
@@ -142,7 +167,8 @@ vi.mock('../providers/ssh-pty-provider', () => ({
|
||||
isSshPtyNotFoundError: (err: unknown) =>
|
||||
(err instanceof Error ? err.message : String(err)).includes('not found'),
|
||||
SshPtyProvider: class MockSshPtyProvider {
|
||||
constructor() {
|
||||
constructor(_targetId: unknown, _mux: unknown, _env: unknown, providerGeneration: number) {
|
||||
mockPtyProvider.providerGeneration = providerGeneration
|
||||
return mockPtyProvider
|
||||
}
|
||||
}
|
||||
@@ -222,6 +248,9 @@ import {
|
||||
type SshConnectionState,
|
||||
type SshTarget
|
||||
} from '../../shared/ssh-types'
|
||||
import { PTY_CONSUMER_SESSION_PROTOCOL_VERSION } from '../../shared/pty-consumer-session'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import {
|
||||
clearProviderPtyState,
|
||||
deletePtyOwnership,
|
||||
@@ -231,6 +260,16 @@ import {
|
||||
import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation'
|
||||
|
||||
describe('SSH IPC handlers', () => {
|
||||
const relayBuildId = '0.1.0+ipc-test'
|
||||
const ipcTestSource = {
|
||||
relayPtyId: 'remote-pty',
|
||||
spanId: 'ipc-test-delivery:0:5',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: 'ipc-test-delivery',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
} as const
|
||||
const handlers = new Map<string, (_event: unknown, args: unknown) => unknown>()
|
||||
const mockStore = {
|
||||
getRepos: () => [],
|
||||
@@ -272,7 +311,8 @@ describe('SSH IPC handlers', () => {
|
||||
const relayLostStabilizedMs = 5_000
|
||||
const createRelayLaunchResult = () => ({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
platform: 'linux-x64'
|
||||
platform: 'linux-x64',
|
||||
serverBuildId: relayBuildId
|
||||
})
|
||||
const getLatestRelayDisposeCallback = (): RelayDisposeCallback => {
|
||||
const calls = mockMux.onDispose.mock.calls
|
||||
@@ -322,21 +362,43 @@ describe('SSH IPC handlers', () => {
|
||||
mockConnectionManager.setCallbacks.mockReset()
|
||||
mockConnectionManager.callbacksRef.current = null
|
||||
mockForceStopRelayForTarget.mockReset().mockResolvedValue(undefined)
|
||||
mockAcceptSshPtyOutputData.mockReset().mockResolvedValue({})
|
||||
mockAcceptSshPtyOutputExit.mockReset().mockResolvedValue(undefined)
|
||||
|
||||
mockDeployAndLaunchRelay.mockReset().mockResolvedValue({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
platform: 'linux-x64'
|
||||
platform: 'linux-x64',
|
||||
serverBuildId: relayBuildId
|
||||
})
|
||||
mockMux.dispose.mockReset()
|
||||
mockMux.isDisposed.mockReset().mockReturnValue(false)
|
||||
mockMux.onNotification.mockReset()
|
||||
mockMux.onNotificationByMethod.mockReset().mockReturnValue(() => {})
|
||||
mockMux.onDispose.mockReset().mockReturnValue(() => {})
|
||||
mockMux.request.mockReset().mockImplementation((method: string) =>
|
||||
Promise.resolve(
|
||||
method === 'pty.openClient'
|
||||
? {
|
||||
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
||||
serverBuildId: relayBuildId,
|
||||
clientGeneration: 1,
|
||||
role: 'session-owner',
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'ipc-test-owner',
|
||||
capabilities: {
|
||||
outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
|
||||
}
|
||||
}
|
||||
: {}
|
||||
)
|
||||
)
|
||||
mockMux.probeLiveness.mockReset().mockResolvedValue(false)
|
||||
mockPtyProvider.onData.mockReset()
|
||||
mockPtyProvider.onExit.mockReset()
|
||||
mockPtyProvider.onReplay.mockReset()
|
||||
mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({})
|
||||
mockPtyProvider.shutdown.mockReset()
|
||||
mockPtyProvider.providerGeneration = 0
|
||||
mockRegisterSshGitProvider.mockReset()
|
||||
mockPortForwardManager.addForward.mockReset()
|
||||
mockPortForwardManager.updateForward.mockReset()
|
||||
@@ -565,6 +627,7 @@ describe('SSH IPC handlers', () => {
|
||||
}
|
||||
mockDeployAndLaunchRelay.mockResolvedValueOnce({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
serverBuildId: relayBuildId,
|
||||
hostPlatform
|
||||
})
|
||||
mockSshStore.getTarget.mockReturnValue(target)
|
||||
@@ -1050,7 +1113,7 @@ describe('SSH IPC handlers', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('forwards remote PTY events into the runtime', async () => {
|
||||
it('forwards remote PTY events through the output intake authority', async () => {
|
||||
const runtime = {
|
||||
onPtyData: vi.fn(),
|
||||
onPtyExit: vi.fn()
|
||||
@@ -1073,24 +1136,47 @@ describe('SSH IPC handlers', () => {
|
||||
})
|
||||
|
||||
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; data: string }) => void)
|
||||
| undefined
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
|
||||
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; code: number }) => void)
|
||||
| ((payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void)
|
||||
| undefined
|
||||
|
||||
onData?.({ id: 'remote-pty', data: 'hello' })
|
||||
onExit?.({ id: 'remote-pty', code: 7 })
|
||||
onData?.({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
source: ipcTestSource
|
||||
})
|
||||
onExit?.({
|
||||
id: 'remote-pty',
|
||||
code: 7,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
|
||||
expect(runtime.onPtyData).toHaveBeenCalledWith(
|
||||
'remote-pty',
|
||||
'hello',
|
||||
expect.any(Number),
|
||||
'hello'.length,
|
||||
undefined
|
||||
)
|
||||
expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 7, undefined)
|
||||
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
rawLength: 'hello'.length,
|
||||
transformed: false,
|
||||
source: ipcTestSource
|
||||
})
|
||||
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith({
|
||||
id: 'remote-pty',
|
||||
code: 7,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
expect(runtime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(runtime.onPtyExit).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('mirrors SSH state broadcasts onto the runtime client-event stream', async () => {
|
||||
@@ -1380,7 +1466,7 @@ describe('SSH IPC handlers', () => {
|
||||
expect(replacementConnectionManager.disconnect).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refreshes live session callbacks to the newest window, store, and runtime', async () => {
|
||||
it('refreshes live session callbacks to the newest window and output authorities', async () => {
|
||||
const firstWindow = createMockWindow()
|
||||
const secondWindow = createMockWindow()
|
||||
const firstRuntime = {
|
||||
@@ -1411,11 +1497,14 @@ describe('SSH IPC handlers', () => {
|
||||
})
|
||||
|
||||
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; data: string }) => void)
|
||||
| undefined
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
|
||||
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; code: number }) => void)
|
||||
| ((payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void)
|
||||
| undefined
|
||||
const onDetectedPorts = mockPortScannerCallbacks.get('ssh-1') as
|
||||
| ((targetId: string, ports: unknown[], platform: string) => void)
|
||||
@@ -1434,8 +1523,19 @@ describe('SSH IPC handlers', () => {
|
||||
error: 'network down',
|
||||
reconnectAttempt: 0
|
||||
})
|
||||
onData?.({ id: 'remote-pty', data: 'hello' })
|
||||
onExit?.({ id: 'remote-pty', code: 9 })
|
||||
onData?.({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
source: ipcTestSource
|
||||
})
|
||||
onExit?.({
|
||||
id: 'remote-pty',
|
||||
code: 9,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
onDetectedPorts?.(
|
||||
'ssh-1',
|
||||
[{ host: '127.0.0.1', port: 3000, pid: 12, processName: 'node' }],
|
||||
@@ -1454,33 +1554,20 @@ describe('SSH IPC handlers', () => {
|
||||
connectionGeneration: 1
|
||||
}
|
||||
})
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith(
|
||||
'pty:data',
|
||||
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'remote-pty', data: 'hello' })
|
||||
)
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith('pty:exit', {
|
||||
id: 'remote-pty',
|
||||
code: 9
|
||||
})
|
||||
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'remote-pty', code: 9 })
|
||||
)
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith('ssh:detected-ports-changed', {
|
||||
targetId: 'ssh-1',
|
||||
ports: expect.arrayContaining([expect.objectContaining({ port: 3000 })])
|
||||
})
|
||||
expect(secondRuntime.onPtyData).toHaveBeenCalledWith(
|
||||
'remote-pty',
|
||||
'hello',
|
||||
expect.any(Number),
|
||||
'hello'.length,
|
||||
undefined
|
||||
)
|
||||
expect(secondRuntime.onPtyExit).toHaveBeenCalledWith('remote-pty', 9, undefined)
|
||||
expect(secondRuntime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(secondRuntime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(firstRuntime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(firstRuntime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'remote-pty',
|
||||
'terminated'
|
||||
)
|
||||
})
|
||||
|
||||
it('re-registers without replacing managers when no targets are connected', () => {
|
||||
|
||||
@@ -106,7 +106,8 @@ describe('tcc prompt notice threshold', () => {
|
||||
const [, contents] = writeFileAtomically.mock.calls.at(-1) as [string, string]
|
||||
expect(JSON.parse(contents)).toMatchObject({
|
||||
promptCount: TCC_PROMPT_NOTICE_THRESHOLD,
|
||||
notified: true
|
||||
notified: true,
|
||||
acknowledgedAfterClose: true
|
||||
})
|
||||
})
|
||||
|
||||
@@ -142,7 +143,11 @@ describe('tcc prompt notice threshold', () => {
|
||||
|
||||
expect(consumePendingTccPromptNotice(2)).toBeNull()
|
||||
const [, contents] = writeFileAtomically.mock.calls.at(-1) as [string, string]
|
||||
expect(JSON.parse(contents)).toMatchObject({ dismissed: true, notified: true })
|
||||
expect(JSON.parse(contents)).toMatchObject({
|
||||
dismissed: true,
|
||||
notified: true,
|
||||
acknowledgedAfterClose: true
|
||||
})
|
||||
})
|
||||
|
||||
it('routes a later prompt to the replacement main window', () => {
|
||||
@@ -297,11 +302,43 @@ describe('tcc prompt notice threshold', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('delivers a tally persisted below the old threshold without respawning the watcher', () => {
|
||||
it.each([
|
||||
{ promptCount: 2, notified: false },
|
||||
{ promptCount: 1, notified: true }
|
||||
])('requires a fresh detection for a legacy tally: $promptCount/$notified', (persisted) => {
|
||||
const platform = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' })
|
||||
readTallyFile.mockReturnValue(JSON.stringify({ ...persisted, dismissed: false }))
|
||||
try {
|
||||
const mainWindow = createWindowStub()
|
||||
initTccPromptNotice(mainWindow as never)
|
||||
|
||||
expect(watchStart).toHaveBeenCalledOnce()
|
||||
expect(mainWindow.webContents.send).not.toHaveBeenCalled()
|
||||
expect(consumePendingTccPromptNotice(1)).toBeNull()
|
||||
|
||||
watchOptions[0].onPrompt()
|
||||
|
||||
expect(mainWindow.webContents.send).toHaveBeenCalledWith('macosTccPrompts:threshold', {
|
||||
promptCount: 1
|
||||
})
|
||||
expect(watchStop).toHaveBeenCalledOnce()
|
||||
expect(consumePendingTccPromptNotice(1)).toEqual({ claimId: 1, promptCount: 1 })
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', platform!)
|
||||
}
|
||||
})
|
||||
|
||||
it('replays an unclosed notice from the new delivery contract', () => {
|
||||
const platform = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' })
|
||||
readTallyFile.mockReturnValue(
|
||||
JSON.stringify({ promptCount: 2, notified: false, dismissed: false })
|
||||
JSON.stringify({
|
||||
promptCount: 1,
|
||||
notified: false,
|
||||
dismissed: false,
|
||||
acknowledgedAfterClose: false
|
||||
})
|
||||
)
|
||||
try {
|
||||
const mainWindow = createWindowStub()
|
||||
@@ -309,10 +346,32 @@ describe('tcc prompt notice threshold', () => {
|
||||
|
||||
expect(watchStart).not.toHaveBeenCalled()
|
||||
expect(mainWindow.webContents.send).toHaveBeenCalledWith('macosTccPrompts:threshold', {
|
||||
promptCount: 2
|
||||
promptCount: 1
|
||||
})
|
||||
expect(mainWindow.once).not.toHaveBeenCalled()
|
||||
expect(consumePendingTccPromptNotice(1)).toEqual({ claimId: 1, promptCount: 2 })
|
||||
expect(consumePendingTccPromptNotice(1)).toEqual({ claimId: 1, promptCount: 1 })
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', platform!)
|
||||
}
|
||||
})
|
||||
|
||||
it('does not replay a notice acknowledged after close', () => {
|
||||
const platform = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' })
|
||||
readTallyFile.mockReturnValue(
|
||||
JSON.stringify({
|
||||
promptCount: 1,
|
||||
notified: true,
|
||||
dismissed: false,
|
||||
acknowledgedAfterClose: true
|
||||
})
|
||||
)
|
||||
try {
|
||||
const mainWindow = createWindowStub()
|
||||
initTccPromptNotice(mainWindow as never)
|
||||
|
||||
expect(watchStart).not.toHaveBeenCalled()
|
||||
expect(mainWindow.webContents.send).not.toHaveBeenCalled()
|
||||
expect(consumePendingTccPromptNotice(1)).toBeNull()
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', platform!)
|
||||
}
|
||||
|
||||
@@ -31,9 +31,15 @@ type TccPromptTally = {
|
||||
promptCount: number
|
||||
notified: boolean
|
||||
dismissed: boolean
|
||||
acknowledgedAfterClose: boolean
|
||||
}
|
||||
|
||||
const EMPTY_TALLY: TccPromptTally = { promptCount: 0, notified: false, dismissed: false }
|
||||
const EMPTY_TALLY: TccPromptTally = {
|
||||
promptCount: 0,
|
||||
notified: false,
|
||||
dismissed: false,
|
||||
acknowledgedAfterClose: false
|
||||
}
|
||||
|
||||
let tally: TccPromptTally = { ...EMPTY_TALLY }
|
||||
let mainWindowRef: BrowserWindow | null = null
|
||||
@@ -50,10 +56,17 @@ function tallyPath(): string {
|
||||
function loadTally(): TccPromptTally {
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(tallyPath(), 'utf-8')) as Partial<TccPromptTally>
|
||||
const dismissed = parsed.dismissed === true
|
||||
if (!dismissed && typeof parsed.acknowledgedAfterClose !== 'boolean') {
|
||||
// Why: a legacy tally only proves a past prompt, not that Full Disk Access is still missing.
|
||||
return { ...EMPTY_TALLY }
|
||||
}
|
||||
const acknowledgedAfterClose = parsed.acknowledgedAfterClose === true
|
||||
return {
|
||||
promptCount: typeof parsed.promptCount === 'number' ? parsed.promptCount : 0,
|
||||
notified: parsed.notified === true,
|
||||
dismissed: parsed.dismissed === true
|
||||
notified: parsed.notified === true && (dismissed || acknowledgedAfterClose),
|
||||
dismissed,
|
||||
acknowledgedAfterClose
|
||||
}
|
||||
} catch {
|
||||
return { ...EMPTY_TALLY }
|
||||
@@ -108,7 +121,7 @@ export function acknowledgePendingTccPromptNotice(ownerToken: number, claimId: n
|
||||
return
|
||||
}
|
||||
pendingClaim = null
|
||||
tally = { ...tally, notified: true }
|
||||
tally = { ...tally, notified: true, acknowledgedAfterClose: true }
|
||||
saveTally()
|
||||
}
|
||||
|
||||
@@ -124,7 +137,7 @@ export function releasePendingTccPromptNotice(ownerToken: number, claimId?: numb
|
||||
/** Permanently stops the notice for this user; the watcher shuts down with it. */
|
||||
export function dismissTccPromptNotice(): void {
|
||||
pendingClaim = null
|
||||
tally = { ...tally, dismissed: true, notified: true }
|
||||
tally = { ...tally, dismissed: true, notified: true, acknowledgedAfterClose: true }
|
||||
saveTally()
|
||||
stopTccPromptNotice()
|
||||
}
|
||||
|
||||
@@ -1548,6 +1548,26 @@ describe('Store', () => {
|
||||
expect(updatedTarget).not.toHaveProperty('systemSshConnectionReuse')
|
||||
})
|
||||
|
||||
it('drops retired per-target SSH terminal source-credit selections', async () => {
|
||||
const store = await createStore()
|
||||
store.addSshTarget({
|
||||
id: 'ssh-source-credit-on',
|
||||
label: 'Noisy build host',
|
||||
host: 'build.example.com',
|
||||
port: 22,
|
||||
username: 'dev',
|
||||
experimentalPtySourceCreditV1: true
|
||||
} as never)
|
||||
|
||||
expect(store.getSshTarget('ssh-source-credit-on')).not.toHaveProperty(
|
||||
'experimentalPtySourceCreditV1'
|
||||
)
|
||||
store.flush()
|
||||
const persisted = readDataFile() as { sshTargets?: Record<string, unknown>[] }
|
||||
const target = persisted.sshTargets?.find((entry) => entry.id === 'ssh-source-credit-on')
|
||||
expect(target).not.toHaveProperty('experimentalPtySourceCreditV1')
|
||||
})
|
||||
|
||||
it('upserts ~/.ssh/config through the real store: rotated port updates in place and persists', async () => {
|
||||
loadUserSshConfigMock.mockReturnValue([{ host: 'cluster' }])
|
||||
const candidate = (port: number, id: string) => [
|
||||
@@ -1563,7 +1583,6 @@ describe('Store', () => {
|
||||
expect(inserted).toHaveLength(1)
|
||||
expect(inserted[0]?.source).toBe('ssh-config')
|
||||
expect(inserted[0]?.port).toBe(2200)
|
||||
|
||||
// Rotated port: upsert updates the same target in place and normalizeSshTarget must keep `source` (no false re-derive into a permanently-dirty state).
|
||||
sshConfigHostsToTargetsMock.mockReturnValue(candidate(2222, 'ssh-cfg-2'))
|
||||
const changed = sshStore.importFromSshConfig()
|
||||
|
||||
@@ -1113,6 +1113,7 @@ function backfillLegacyAutomationContexts(
|
||||
type LegacySshTarget = SshTarget & {
|
||||
remoteWorkspaceSyncEnabled?: unknown
|
||||
remoteWorkspaceSyncGracePeriodSeconds?: unknown
|
||||
experimentalPtySourceCreditV1?: unknown
|
||||
}
|
||||
|
||||
// Why: old targets predate configHost; default to label-based lookup so imported SSH aliases still resolve via ssh -G.
|
||||
@@ -1127,6 +1128,7 @@ function normalizeSshTarget(t: SshTarget): SshTarget {
|
||||
delete target.remoteWorkspaceSyncGracePeriodSeconds
|
||||
delete target.relayGracePeriodSeconds
|
||||
delete target.systemSshConnectionReuse
|
||||
delete target.experimentalPtySourceCreditV1
|
||||
// Why: prefer the synced grace over stale relayGracePeriodSeconds so a user's "unlimited" (0) survives migration.
|
||||
const relayGracePeriodSeconds =
|
||||
legacySyncEnabled === true && typeof legacyGracePeriodSeconds === 'number'
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user