diff --git a/config/scripts/ssh-relay-runtime-artifact.test.mjs b/config/scripts/ssh-relay-runtime-artifact.test.mjs index d09198ed799..e13ae1be529 100644 --- a/config/scripts/ssh-relay-runtime-artifact.test.mjs +++ b/config/scripts/ssh-relay-runtime-artifact.test.mjs @@ -10,7 +10,10 @@ import { inspectSshRelayRuntimeArchive } from './ssh-relay-runtime-archive.mjs' import { computeSshRelayRuntimeContentId } from './ssh-relay-runtime-identity.mjs' -import { verifyRuntimeTree } from './verify-ssh-relay-runtime.mjs' +import { + formatSshRelayRuntimeSmokeFailure, + verifyRuntimeTree +} from './verify-ssh-relay-runtime.mjs' const temporaryDirectories = [] afterEach(async () => { @@ -179,6 +182,25 @@ describe('SSH relay runtime artifact', () => { } }) + it('preserves bounded bundled-smoke child failure diagnostics', () => { + const message = formatSshRelayRuntimeSmokeFailure({ + code: 'ETIMEDOUT', + killed: true, + signal: 'SIGTERM', + message: 'Command timed out', + stdout: 'partial smoke output', + stderr: `${'x'.repeat(70 * 1024)}Bundled runtime smoke failed: watcher create exceeded 15000 ms` + }) + + expect(message).toContain('timeoutMs=45000') + expect(message).toContain('code="ETIMEDOUT"') + expect(message).toContain('killed=true') + expect(message).toContain('signal="SIGTERM"') + expect(message).toContain('partial smoke output') + expect(message).toContain('truncated') + expect(message).toContain('watcher create exceeded 15000 ms') + }) + it.skipIf(process.platform === 'win32')( 'creates deterministic archives and rehashes every entry', async () => { diff --git a/config/scripts/verify-ssh-relay-runtime.mjs b/config/scripts/verify-ssh-relay-runtime.mjs index 5179d0fc034..f849367cbba 100644 --- a/config/scripts/verify-ssh-relay-runtime.mjs +++ b/config/scripts/verify-ssh-relay-runtime.mjs @@ -10,6 +10,31 @@ import { computeSshRelayRuntimeContentId } from './ssh-relay-runtime-identity.mj const execFileAsync = promisify(execFile) const scriptDirectory = import.meta.dirname const MAX_SMOKE_OUTPUT_BYTES = 4 * 1024 * 1024 +const MAX_SMOKE_FAILURE_DETAIL_BYTES = 64 * 1024 +const SMOKE_TIMEOUT_MS = 45_000 + +function boundedSshRelayRuntimeSmokeDetail(value) { + const text = typeof value === 'string' ? value : Buffer.isBuffer(value) ? value.toString() : '' + const bytes = Buffer.from(text) + if (bytes.length <= MAX_SMOKE_FAILURE_DETAIL_BYTES) { + return text + } + const retained = bytes.subarray(bytes.length - MAX_SMOKE_FAILURE_DETAIL_BYTES).toString() + return `[truncated ${bytes.length - MAX_SMOKE_FAILURE_DETAIL_BYTES} bytes]\n${retained}` +} + +export function formatSshRelayRuntimeSmokeFailure(error) { + return [ + 'Bundled runtime smoke command failed:', + `timeoutMs=${SMOKE_TIMEOUT_MS}`, + `code=${JSON.stringify(error?.code ?? null)}`, + `killed=${JSON.stringify(error?.killed ?? false)}`, + `signal=${JSON.stringify(error?.signal ?? null)}`, + `message=${JSON.stringify(error?.message ?? String(error))}`, + `stdout=${JSON.stringify(boundedSshRelayRuntimeSmokeDetail(error?.stdout))}`, + `stderr=${JSON.stringify(boundedSshRelayRuntimeSmokeDetail(error?.stderr))}` + ].join(' ') +} function parseArguments(argv) { const result = {} @@ -110,13 +135,19 @@ export async function verifyRuntimeTree(runtimeDirectory, identity) { async function runBundledSmoke(runtimeDirectory, identity) { const nodePath = join(runtimeDirectory, 'bin', identity.os === 'win32' ? 'node.exe' : 'node') const childPath = resolve(scriptDirectory, 'ssh-relay-runtime-smoke-child.cjs') - const result = await execFileAsync(nodePath, [childPath, runtimeDirectory], { - cwd: runtimeDirectory, - encoding: 'utf8', - maxBuffer: MAX_SMOKE_OUTPUT_BYTES, - timeout: 45_000, - windowsHide: true - }) + let result + try { + result = await execFileAsync(nodePath, [childPath, runtimeDirectory], { + cwd: runtimeDirectory, + encoding: 'utf8', + maxBuffer: MAX_SMOKE_OUTPUT_BYTES, + timeout: SMOKE_TIMEOUT_MS, + windowsHide: true + }) + } catch (error) { + // The child classifies PTY/watcher failures; retain its bounded tail to avoid blind CI retries. + throw new Error(formatSshRelayRuntimeSmokeFailure(error), { cause: error }) + } const smoke = JSON.parse(result.stdout) if (smoke.nodeVersion !== `v${identity.nodeVersion}`) { throw new Error(`Bundled smoke used unexpected Node version: ${smoke.nodeVersion}`) diff --git a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md index 35c08e48eef..37907dcd5e9 100644 --- a/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md +++ b/docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md @@ -2,7 +2,7 @@ Date created: 2026-07-14
Last updated: 2026-07-14
-Current phase: Milestone 3 / Work Package 2 target-native runtime assembly — four POSIX native-runner artifact builds are CI-green; Windows contract tests are native-runner green but signature-tool path portability, runtime/ZIP, reproducibility, oldest-baseline, native-trust, cross-family remote, and measured-baseline gates remain open; no bundled-runtime path is enabled
+Current phase: Milestone 3 / Work Package 2 target-native runtime assembly — four POSIX native-runner artifact builds are CI-green; Windows signed-input verification and runtime/ZIP assembly reach native smoke on both architectures, but bounded smoke diagnostics/execution, reproducibility, oldest-baseline, native-trust, cross-family remote, and measured-baseline gates remain open; no bundled-runtime path is enabled
Primary design: [SSH relay GitHub Release plan](./2026-07-14-ssh-relay-github-release-plan.html)
Motivating issues: [#8450](https://github.com/stablyai/orca/issues/8450), [#1693](https://github.com/stablyai/orca/issues/1693) @@ -61,11 +61,11 @@ same change as the work it records. smoke, SBOM, and provenance only. The Windows x64/arm64 artifact candidate and portability corrections are implemented through `b6903b220` in stacked draft PR [#8741](https://github.com/stablyai/orca/pull/8741). Both native architectures passed all seven - contract suites at the preceding head, then failed closed before extraction/build because Git for - Windows `gpgv` rejected an absolute drive-letter keyring path (E-M3-WINDOWS-CI-RED-003). The - cwd-relative correction is locally green under E-M3-WINDOWS-GPG-PATH-LOCAL-001 and awaits native - CI. This package may produce test artifacts but must not publish, resolve, transfer, install, - launch, or enable them. + contract suites, exact signed-input verification, native compilation, and bounded runtime/ZIP + assembly at exact CI head `42aa02fa9` (E-M3-WINDOWS-CI-RED-004). Both then time out during the + bundled smoke, while the parent currently suppresses bounded child stderr needed to classify the + failing PTY or watcher substage. This package may produce test artifacts but must not publish, + resolve, transfer, install, launch, or enable them. - Active evidence gate: the immutable Node v24.18.0 contract, pinned release key, bounded verifier, and artifact-only CLI are locally green under E-M3-NODE-RED-001 and E-M3-NODE-PROVENANCE-001. E-M3-RUNTIME-LOCAL-001 additionally proves one unpublished Linux arm64 glibc assembly, exact-tree @@ -90,7 +90,10 @@ same change as the work it records. Windows shebang parsing, authenticated-key checkout bytes, and POSIX-only mode assertions. At the corrected exact head, E-M3-WINDOWS-CI-RED-003 proves 20 passing and one intentionally skipped test on each native architecture, exact input download, and a common fail-closed Git-for-Windows - `gpgv` drive-letter keyring incompatibility. No Windows executable evidence is claimed. + `gpgv` drive-letter keyring incompatibility. E-M3-WINDOWS-CI-RED-004 proves that correction with + 21 passing and one intentionally skipped test, signed input acceptance, native compilation, and + runtime/ZIP assembly on each architecture; both 45-second bundled smoke commands then time out + without propagating their already bounded child stderr. No Windows executable cell is claimed. - Production behavior: unchanged; Orca embeds relay JavaScript and installs `node-pty` plus `@parcel/watcher` with remote npm. - New runtime assets published: none. @@ -103,13 +106,12 @@ same change as the work it records. - Rollout control: existing per-SSH-target configuration; legacy is the default and the bundled runtime is an explicit per-target Beta opt-in under E-M1-ROLLOUT-DECISION-001. - Legacy fallback removal: not authorized. -- Next required action: pass only cwd-relative verified-copy paths to `gpg`/`gpgv`, add a - discriminating command-boundary regression test, and collect exact Windows x64/arm64 plus POSIX - regression evidence from the corrected draft-PR head. A - same-head/same-runner clean-rebuild identity oracle for native build reproducibility remains a - separate open gate. Keep cross-family remote infrastructure, signing/trust, and measured legacy - baseline gates open; do not introduce publication, resolver, transfer, rollout, tuple enablement, - or default behavior. +- Next required action: preserve the smoke command's bounded stdout/stderr and timeout metadata in + the parent failure, add a discriminating diagnostic test, and rerun Windows x64 first to classify + the PTY/watcher failure before changing smoke behavior. A same-head/same-runner clean-rebuild + identity oracle for native build reproducibility remains a separate open gate. Keep cross-family + remote infrastructure, signing/trust, and measured legacy baseline gates open; do not introduce + publication, resolver, transfer, rollout, tuple enablement, or default behavior. ## Non-Negotiable Invariants @@ -2709,6 +2711,136 @@ or unexpected token`; the first logs did not identify a source location. - Follow-up: run the exact correction on Windows x64/arm64 and all four POSIX artifact jobs, then record any next discriminating native boundary without broadening this work package. +### E-M3-WINDOWS-CI-RED-004 — Native Windows assembly reached an opaque bounded smoke timeout + +- Date: 2026-07-14 +- Commit SHA / PR: exact workflow head `42aa02fa9eccf81e40d80f87461ddd232a1cda1f`, containing + implementation commit `b6903b220`; draft PR + [#8741](https://github.com/stablyai/orca/pull/8741) +- Run: [SSH Relay Runtime Artifacts 29340686444](https://github.com/stablyai/orca/actions/runs/29340686444), + conclusion `failure`; all four POSIX regression jobs passed and uploaded only unpublished + seven-day Actions artifacts. +- Native Windows jobs: + - x64 job `87111302431`: `windows-2022`, resolved `win22` image `20260706.237.1`, GitHub-hosted + X64, Windows Server 2022 Datacenter build 20348, Node v24.18.0; failed after 4m37s. + - arm64 job `87111302486`: `windows-11-arm`, resolved `win11-arm64` image `20260706.102.1`, + GitHub-hosted ARM64, Windows 10 Enterprise build 26200, Node v24.18.0; failed after 7m9s. +- Remote and transport: no SSH remote. Exact inputs came from `nodejs.org/dist/v24.18.0`; no release + publication or Windows artifact upload occurred. +- Exact commands: + + ```sh + gh run view 29340686444 --repo stablyai/orca \ + --json databaseId,headSha,status,conclusion,url,createdAt,updatedAt,jobs + gh api repos/stablyai/orca/actions/jobs/87111302431/logs | \ + rg -n -C 25 "SSH relay runtime verification failed|durationMs|contentId|Process completed with exit code" + gh api repos/stablyai/orca/actions/jobs/87111302486/logs | \ + rg -n -C 25 "SSH relay runtime verification failed|durationMs|contentId|Process completed with exit code" + gh api --paginate repos/stablyai/orca/actions/runs/29340686444/artifacts + ``` + +- Result: FAIL at the native bundled smoke on both architectures. Each runner passed all seven + contract suites with 21 passing and one intentionally skipped test, authenticated the exact Node + checksum/signature plus ZIP/headers/import library, compiled the patched native modules, assembled + and reinspected a 60-entry/42-file deterministic-format ZIP, and entered verification's smoke only + after archive/tree integrity checks. X64 produced a 37,212,065-byte ZIP expanding to 97,248,414 + bytes, content ID `90823b2c6bf7dad748a4399fe74fdfa8be82ef4a00d727e3a6be173f274fd43f`, + archive SHA-256 `ee7634dea179026b3ba8232294016d939e32edc34e1fd3697dd9e11722285d6a`, + and 146,511.617 ms build duration. Arm64 produced a 33,261,531-byte ZIP expanding to 86,189,740 + bytes, content ID `09ec2772bb8c47ee240fd226963b7e251a8060e7f84321cb24d2668bf993e2ca`, + archive SHA-256 `5cb50521ec88d491f1a4183fc13f0b0f688e242e290038c4d34e0125d8e8b2c2`, + and 176,108.728 ms build duration. +- Failure boundary: each bundled-Node smoke exceeded the parent command's 45-second timeout. The + child already bounds PTY and watcher waits at 15 seconds and writes its classified failure to + stderr, but `verify-ssh-relay-runtime.mjs` currently reports only the generic `execFile` stack. + The logs therefore cannot distinguish PTY, watcher, cleanup-handle, or combined failure. Upload + remained skipped. Smoke RSS, channel/file counts, and cancellation settlement were not emitted. +- Oracle proved: the GPG relative-path correction works through real Git-for-Windows GPG on native + x64 and arm64; signed-input gating, offline native build inputs, patched native compilation, + bundled Node v24.18.0 staging, ZIP construction/inspection, and pre-execution archive/tree gates + all precede the failing smoke. The four POSIX jobs remain green on the exact same source head. +- Does not prove: successful Windows PTY/ConPTY/watcher execution, which smoke substage failed, + prompt child-process settlement, ZIP clean-rebuild identity, durable Windows artifact bytes, + native trust/signing, oldest baseline, SSH, or any enabled tuple. +- Checklist items satisfied: native GPG path portability and pre-smoke Windows assembly progression + only. Windows per-tuple cells remain unchecked because executable smoke and artifact retention did + not pass. +- Follow-up: preserve bounded child stdout, stderr, timeout, exit, and signal details in the parent + diagnostic; add a failure-path unit test; rerun x64 first and change no PTY/watcher behavior until + the classified child failure is visible. + +### E-M3-WINDOWS-SMOKE-DIAGNOSTIC-RED-001 — Bounded child-detail propagation red gate + +- Date: 2026-07-14 +- Commit SHA / PR: uncommitted purpose-named regression test on exact head + `42aa02fa9eccf81e40d80f87461ddd232a1cda1f`; draft PR + [#8741](https://github.com/stablyai/orca/pull/8741) +- Runner: macOS 26.2 arm64; Node v26.0.0 and pnpm 10.24.0 +- Remote and transport: none; synthetic child-process error object only +- Exact command: + `pnpm exec vitest run --config config/vitest.config.ts config/scripts/ssh-relay-runtime-artifact.test.mjs` +- Result: expected FAIL; the new diagnostic test failed and three existing tests passed in 239 ms + because `formatSshRelayRuntimeSmokeFailure` did not exist. The required oracle supplies timeout, + exit/signal state, partial stdout, and a greater-than-limit stderr whose final classified failure + must survive explicit truncation. +- Oracle proved: existing parent diagnostics cannot satisfy the bounded failure-detail contract and + the new test discriminates both missing metadata and unbounded/error-tail-losing implementations. +- Does not prove: child execution, Windows behavior, the actual PTY/watcher failure, cleanup + settlement, or any tuple support. +- Checklist items satisfied: red half of the smoke-diagnostic correction only. +- Follow-up: implement a 64-KiB tail-preserving formatter around the already 4-MiB-bounded child + process, wrap `execFile` failure with it, and rerun focused/static gates before native x64 CI. + +### E-M3-WINDOWS-SMOKE-DIAGNOSTIC-LOCAL-001 — Bounded child-detail propagation correction + +- Date: 2026-07-14 +- Commit SHA / PR: uncommitted correction on exact base + `42aa02fa9eccf81e40d80f87461ddd232a1cda1f`; draft PR + [#8741](https://github.com/stablyai/orca/pull/8741) +- Runner: macOS 26.2 arm64; Node v26.0.0 and pnpm 10.24.0. Exact-head Node 24 CI remains + authoritative. +- Remote and transport: none; synthetic child-process error plus local artifact contracts +- Exact commands: + + ```sh + pnpm exec vitest run --config config/vitest.config.ts \ + config/scripts/ssh-relay-runtime-artifact.test.mjs + pnpm exec vitest run --config config/vitest.config.ts \ + config/scripts/ssh-relay-node-release-verification.test.mjs \ + config/scripts/ssh-relay-node-tar-inspection.test.mjs \ + config/scripts/ssh-relay-node-zip-inspection.test.mjs \ + config/scripts/ssh-relay-runtime-artifact.test.mjs \ + config/scripts/ssh-relay-runtime-windows-tree.test.mjs \ + config/scripts/ssh-relay-runtime-zip.test.mjs \ + config/scripts/ssh-relay-runtime-workflow.test.mjs + pnpm run typecheck + pnpm exec oxlint \ + config/scripts/verify-ssh-relay-runtime.mjs \ + config/scripts/ssh-relay-runtime-artifact.test.mjs + pnpm exec oxlint + pnpm run check:max-lines-ratchet + GOMAXPROCS=2 pnpm run lint + pnpm exec oxfmt --check \ + docs/reference/plans/2026-07-14-ssh-relay-github-release-implementation-checklist.md \ + docs/reference/plans/2026-07-14-ssh-relay-github-release-plan.html + git diff --check + ``` + +- Result: PASS. The focused artifact suite passed 4/4 tests in 194 ms; all seven artifact suites + passed 23/23 tests in 293 ms. Typecheck, focused and full oxlint, max-lines ratchet, full + lint/reliability/localization gates, plan formatting, and diff whitespace passed. Full lint + reported only existing warnings outside this package and used the recorded `GOMAXPROCS=2` + stability constraint. +- Oracle proved: any failed bundled-smoke child now propagates the declared 45-second parent timeout, + exit code, kill state, signal, message, stdout, and stderr; each stream retains at most its final + 64 KiB with an explicit omitted-byte count. The real command remains capped at 4 MiB and no + success-path parsing or smoke behavior changed. +- Does not prove: real child failure propagation on Windows, which PTY/watcher stage fails, + process-tree settlement after timeout, successful native smoke, or any tuple support. +- Checklist items satisfied: local green half of the bounded smoke-diagnostic correction only. +- Follow-up: rerun native Windows x64, capture the classified child tail, and make no execution-path + change until that evidence identifies the actual failure. + ### E-M3-RUNTIME-LOCAL-001 — First target-native Linux arm64 runtime artifact - Date: 2026-07-14 @@ -2944,18 +3076,18 @@ or unexpected token`; the first logs did not identify a source location. No product gap is accepted merely because it appears in this list. Each entry requires explicit owner and promotion condition. -| Gap | Current behavior | Risk | Owner | Promotion/removal condition | Status | -| ------------------------------------------ | ----------------------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------ | ------------ | -| Bundled runtime only partially implemented | Four unpublished POSIX native artifact proofs; no production consumer | #8450/#1693 environment failures remain | Codex implementation owner | Complete Work Packages 2–7 plus Milestones 3–14 | Open | -| No bundled tuple enabled | Every target's default and effective mode remains legacy | No bundled support claim can be made | Codex implementation owner | Complete target-native build/trust and both required live-evidence layers | Open | -| Windows runtime/zip builder absent | Windows targets remain entirely on the legacy path | POSIX proof cannot establish Windows behavior | Codex implementation owner | Bounded deterministic zip, Node/native build, signing, smoke, and live gates | Open | -| Native clean-rebuild identity unproved | One successful artifact per corrected-head hosted runner | Toolchain drift may change native content IDs | Codex implementation owner | Same-head, same-runner clean builds match or a reviewed reproducibility policy | Open | -| Cross-family Layer B remotes unavailable | GitHub native runner labels exist; no approved reachable target pool | Client/remote integration gaps may escape | Repository release administrator + implementation owner | Approve provider/snapshots/credentials/egress/teardown/cost owner | BLOCKED | -| Musl has no accepted official Node binary | Musl is deliberately legacy-only | Unofficial binary would break provenance trust | Codex implementation owner | Orca-owned target-native source build, signing, provenance, and live gates | ACCEPTED GAP | -| Native arm64 live matrices incomplete | Hosted Linux/Windows arm64 labels exist; full SSH/runtime cells do not | Cross-build or unit tests may hide native bugs | Codex implementation owner | Full native archive, trust, SFTP/system-SSH, RPC, and baseline evidence | Open | -| Legacy performance baseline unmeasured | Numeric budgets exist; paired cold/warm measurements do not | Regression thresholds lack a measured baseline | Codex implementation owner | Purpose-built paired harness with ten samples on pinned runner classes | Open | -| Manifest signing environment unprovisioned | Ed25519/key-rotation policy exists; no protected runtime signing secret | Runtime assets cannot be safely published | Repository release administrator | Protected environment, reviewers, two test keys, rehearsals, and access audit | BLOCKED | -| Bootstrap primitives lack full live proof | POSIX/Windows contracts exist; bounded SSH implementations do not | Hidden dependency or transfer corruption | Codex implementation owner | Purpose-named full-size SFTP/POSIX/Windows system-SSH live suites | Open | +| Gap | Current behavior | Risk | Owner | Promotion/removal condition | Status | +| ------------------------------------------ | ----------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------ | ------------ | +| Bundled runtime only partially implemented | Four unpublished POSIX native artifact proofs; no production consumer | #8450/#1693 environment failures remain | Codex implementation owner | Complete Work Packages 2–7 plus Milestones 3–14 | Open | +| No bundled tuple enabled | Every target's default and effective mode remains legacy | No bundled support claim can be made | Codex implementation owner | Complete target-native build/trust and both required live-evidence layers | Open | +| Windows runtime smoke incomplete | Native candidate ZIPs build, but smoke fails before artifact retention | Pre-smoke proof cannot establish relay behavior | Codex implementation owner | Classified passing PTY/watcher smoke, reproducibility, trust, and live gates | Open | +| Native clean-rebuild identity unproved | One successful artifact per corrected-head hosted runner | Toolchain drift may change native content IDs | Codex implementation owner | Same-head, same-runner clean builds match or a reviewed reproducibility policy | Open | +| Cross-family Layer B remotes unavailable | GitHub native runner labels exist; no approved reachable target pool | Client/remote integration gaps may escape | Repository release administrator + implementation owner | Approve provider/snapshots/credentials/egress/teardown/cost owner | BLOCKED | +| Musl has no accepted official Node binary | Musl is deliberately legacy-only | Unofficial binary would break provenance trust | Codex implementation owner | Orca-owned target-native source build, signing, provenance, and live gates | ACCEPTED GAP | +| Native arm64 live matrices incomplete | Hosted Linux/Windows arm64 labels exist; full SSH/runtime cells do not | Cross-build or unit tests may hide native bugs | Codex implementation owner | Full native archive, trust, SFTP/system-SSH, RPC, and baseline evidence | Open | +| Legacy performance baseline unmeasured | Numeric budgets exist; paired cold/warm measurements do not | Regression thresholds lack a measured baseline | Codex implementation owner | Purpose-built paired harness with ten samples on pinned runner classes | Open | +| Manifest signing environment unprovisioned | Ed25519/key-rotation policy exists; no protected runtime signing secret | Runtime assets cannot be safely published | Repository release administrator | Protected environment, reviewers, two test keys, rehearsals, and access audit | BLOCKED | +| Bootstrap primitives lack full live proof | POSIX/Windows contracts exist; bounded SSH implementations do not | Hidden dependency or transfer corruption | Codex implementation owner | Purpose-named full-size SFTP/POSIX/Windows system-SSH live suites | Open | ## Final Definition of Done