test(terminal): record Fish scanner reliability evidence

This commit is contained in:
Jinwoo-H
2026-08-12 22:21:23 -04:00
parent fd7567ee75
commit 4099540e67
3 changed files with 30 additions and 6 deletions
+14 -4
View File
@@ -20,7 +20,7 @@
"surfaces": ["queued terminal startup commands", "zsh, bash, and Fish startup output"],
"platforms": ["macos", "linux"],
"providers": ["local", "local-daemon", "ssh-daemon", "ssh-relay", "paired-runtime"],
"coveredPlatforms": ["macos"],
"coveredPlatforms": ["macos", "linux"],
"coveredProviders": ["local", "local-daemon", "ssh-relay"],
"coverageNotes": "Real macOS node-pty integration covers zsh 5.9 .zshenv/.zprofile and bash 3.2 profile replacement shells, including exec -a names, same-shell and child-process silent reads, a blocking zle-line-init hook, non-shell readline replacements, and a non-shell image renamed zsh. A real Fish PTY proves marker stripping is byte-exact and does not render bracketed-paste control text. An Ubuntu 20.04 container confirms compatible zsh/bash termios. Deterministic owner contracts cover local, daemon, and SSH relay identity/readiness composition plus provider- and renderer-owned delivery. Renderer-owned relay fallback publishes the existing ready marker, so mixed client/host versions need no new wire capability. Live SSH, paired-runtime, and WSL remain gaps.",
"motivatingLinks": [
@@ -31,7 +31,8 @@
"oracle": "Split Fish output immediately after the ready marker and after the following ESC introducer; local and daemon owners must emit the exact ESC [ ? 2004 h bytes and headless xterm must not render literal control text. Run the real generated wrappers under node-pty with disposable zsh, bash, and Fish startup files. Plain and exec -a replacement shells must receive one queued command at their first completed line editor without waiting for the 15-second timeout. The identical oracle must time out without the startup identity and prompt fallback. Same-shell reads, child-process reads, a blocking zle-line-init hook, sqlite3, and sqlite3 renamed zsh must not trigger prompt fallback. Relay owner coverage must use the same lossless scanner composition and release through the same prompt contract.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/main/line-editor-ready-output-scanner.test.ts src/main/shell-startup-identity-scanner.test.ts src/main/shell-startup-output-scanner.test.ts src/main/shell-prompt-readiness-probe.test.ts src/shared/pty-slave-line-discipline-echo.test.ts src/shared/shell-process-readiness.test.ts src/main/daemon/session.test.ts src/main/daemon/shell-ready.test.ts src/main/providers/local-pty-shell-ready.test.ts src/main/providers/local-pty-provider.test.ts src/relay/pty-shell-launch.test.ts src/relay/pty-handler.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/relay/pty-handler.test.ts"
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/relay/pty-handler.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 src/main/daemon/shell-ready.test.ts src/main/shell-startup-output-scanner.test.ts"
],
"testFiles": [
"src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts",
@@ -89,11 +90,20 @@
}
],
"evidenceRuns": [
{
"date": "2026-08-12",
"runner": "manual",
"platform": "linux",
"command": "pnpm exec vitest run --config config/vitest.config.ts --maxWorkers=1 src/main/daemon/shell-ready.test.ts src/main/shell-startup-output-scanner.test.ts",
"result": "passed",
"durationSeconds": 7,
"summary": "A clean Debian 12 arm64 container with real Fish 3.6 and node-pty passed exact marker-stripped output plus headless-xterm rendering."
},
{
"date": "2026-08-12",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/main/line-editor-ready-output-scanner.test.ts src/main/shell-startup-identity-scanner.test.ts src/main/shell-prompt-readiness-probe.test.ts src/shared/pty-slave-line-discipline-echo.test.ts src/shared/shell-process-readiness.test.ts src/main/daemon/session.test.ts src/main/daemon/shell-ready.test.ts src/main/providers/local-pty-shell-ready.test.ts src/main/providers/local-pty-provider.test.ts src/relay/pty-shell-launch.test.ts src/relay/pty-handler.test.ts",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/repro-13767-shell-ready-marker-lost-to-exec.test.ts src/main/line-editor-ready-output-scanner.test.ts src/main/shell-startup-identity-scanner.test.ts src/main/shell-startup-output-scanner.test.ts src/main/shell-prompt-readiness-probe.test.ts src/shared/pty-slave-line-discipline-echo.test.ts src/shared/shell-process-readiness.test.ts src/main/daemon/session.test.ts src/main/daemon/shell-ready.test.ts src/main/providers/local-pty-shell-ready.test.ts src/main/providers/local-pty-provider.test.ts src/relay/pty-shell-launch.test.ts src/relay/pty-handler.test.ts",
"result": "passed",
"durationSeconds": 4,
"summary": "All real exec, silent-read, line-init-hook, renamed-image, disabled-bracketed-paste, and relay-owner oracles plus identity, foreground, executable-image, protocol, and line-editor contracts passed."
@@ -109,7 +119,7 @@
},
"redGreenEvidence": {
"status": "complete",
"evidence": "Oracle SHA-256 b13b08006a38b2adc2acb51ae9342909514e55ab4822fac4fbb76cd49b344cc6 is byte-identical across runs. Latest main 09ec516ae5 and the candidate with its production implementation reverted both failed exactly the three zsh .zprofile/.zshenv and bash exec cases by deterministic 5-second timeout; the zle-line-init oracle also exposed unsafe baseline delivery, while all ten other safety controls passed. The candidate passed all 14 on macOS."
"evidence": "The Fish chunk oracle patch SHA-256 fd49a4d3d91bd5f0d10c1b1f44f4cfbd92cef9155a5505b415c8680184cdc765 is byte-identical across runs. Scan target e4c278eb52, latest main d9b390f4fd, and candidate 1f9715c87b with the fix commit reverted each failed the ESC-introducer split in both local and daemon owners while passing both marker-only splits. Candidate 1f9715c87b passed all four cases plus the independent headless-xterm render oracle. The original exec-readiness oracle SHA-256 b13b08006a38b2adc2acb51ae9342909514e55ab4822fac4fbb76cd49b344cc6 remains 14/14 green."
},
"performanceBudget": {
"required": true,
@@ -37,6 +37,17 @@ describe('shell startup output scanner', () => {
})
})
it('counts an identity-held byte as post-marker output', () => {
const state = createShellStartupOutputScanState()
expect(scanShellStartupOutput(state, `${READY_MARKER}\x1b`)).toEqual({
output: '\x1b',
shellPid: null,
ready: true,
postMarkerBytesObserved: true
})
})
it('drains every incomplete scanner prefix in byte order', () => {
const state = createShellStartupOutputScanState()
expect(scanShellStartupOutput(state, '\x1b]777;orca-shell-st').output).toBe('')
+5 -2
View File
@@ -46,15 +46,18 @@ export function scanShellStartupOutput(
const readiness = scanForShellReady(state.ready, data)
let output = readiness.output
let postMarkerBytesObserved = readiness.postMarkerBytesObserved
if (readiness.matched && state.identity) {
output += drainShellStartupIdentityHeldBytes(state.identity)
const heldOutput = drainShellStartupIdentityHeldBytes(state.identity)
output += heldOutput
postMarkerBytesObserved ||= heldOutput.length > 0
state.identity = null
}
return {
output,
shellPid,
ready: readiness.matched,
postMarkerBytesObserved: readiness.postMarkerBytesObserved
postMarkerBytesObserved
}
}