test: enable localhost SSH terminal and hook journey in CI (#19097)

* test: run localhost SSH terminal and hooks in CI

* test: isolate localhost SSH session fixtures across repetitions

* test: route remote agent hook source changes to localhost journey

* test: record localhost SSH reliability evidence and remaining gaps

* test: route the real SSH session hook authority
This commit is contained in:
Neil
2026-09-06 10:05:26 -07:00
committed by GitHub
parent b459b8f16d
commit 4d9e963ffd
7 changed files with 227 additions and 3 deletions
+65
View File
@@ -229,6 +229,7 @@ jobs:
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
. != "tests/e2e/local-ssh-browser-routing.spec.ts" and
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
. != "tests/e2e/ssh-localhost.spec.ts" and
. != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
@@ -375,3 +376,67 @@ jobs:
ORCA_BACKGROUND_LAUNCH: '1'
ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
ssh-localhost:
name: localhost SSH terminal and hooks
needs: [build, prepare-native-cache]
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts')
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install SSH server and headless tools
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: electron
- uses: actions/download-artifact@v8
with:
name: e2e-build-out
path: out/
- name: Start isolated localhost SSH server
shell: bash
run: |
# Bare shells install Pi extensions only for an existing agent home.
mkdir -p "$HOME/.pi/agent"
fixture="$RUNNER_TEMP/orca-localhost-sshd"
mkdir -p "$fixture"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key"
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key"
cat > "$fixture/sshd_config" <<EOF
Port 22222
ListenAddress 127.0.0.1
HostKey $fixture/host_key
PidFile $fixture/sshd.pid
AuthorizedKeysFile $fixture/client_key.pub
StrictModes no
PasswordAuthentication no
KbdInteractiveAuthentication no
UsePAM yes
AllowUsers $(id -un)
Subsystem sftp internal-sftp
EOF
sudo mkdir -p /run/sshd
sudo /usr/sbin/sshd -f "$fixture/sshd_config" -E "$fixture/sshd.log"
ssh -i "$fixture/client_key" -p 22222 -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 127.0.0.1 true || { sudo cat "$fixture/sshd.log"; exit 1; }
{
echo "ORCA_E2E_SSH_PORT=22222"
echo "ORCA_E2E_SSH_USER=$(id -un)"
echo "ORCA_E2E_SSH_IDENTITY_FILE=$fixture/client_key"
} >> "$GITHUB_ENV"
- name: Run localhost SSH terminal and hook journey
env:
SKIP_BUILD: '1'
ORCA_E2E_SSH_LOCALHOST: '1'
ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1
- uses: actions/upload-artifact@v7
if: failure()
with:
name: localhost-ssh-traces
path: test-results/
retention-days: 7
if-no-files-found: ignore
+93
View File
@@ -10,6 +10,99 @@
}
},
"gates": [
{
"id": "ssh.localhost-terminal-agent-hooks",
"title": "Localhost SSH terminal and agent hooks reach the owning pane",
"maturity": "experimental",
"protection": "partial",
"owner": "terminal-runtime",
"layer": "electron-ssh-e2e",
"surfaces": ["SSH terminal", "remote agent status", "remote plugin installation"],
"platforms": ["macos", "linux", "windows"],
"providers": ["ssh"],
"coveredPlatforms": ["linux"],
"coveredProviders": ["ssh"],
"coverageNotes": "Ubuntu CI loopback sshd shares the runner filesystem. Fresh per-test repositories isolate retained relay workspace snapshots; existing Pi home supplies the documented bare-shell plugin prerequisite.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19097"],
"invariant": "A localhost SSH terminal executes on the SSH host and routes authenticated hook status to its owning pane without treating idle keyboard input as agent interruption.",
"oracle": "Require terminal output markers, exported hook identity, actual OpenCode/Pi plugin files, and matching pane/worktree/connection hook events; Ctrl-C and Escape in an idle shell must not interrupt a hook-owned agent.",
"commands": [
"gh run view 34045578306 --log",
"gh run view 34045975180 --log",
"gh run view 34046230389 --log",
"ORCA_E2E_SSH_LOCALHOST=1 ORCA_FEATURE_REMOTE_AGENT_HOOKS=1 pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1",
"node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/ssh-localhost-e2e-routing.test.mjs"
],
"testFiles": [
"tests/e2e/ssh-localhost.spec.ts",
"config/scripts/ssh-localhost-e2e-routing.test.mjs"
],
"assertionRefs": [
{
"file": "tests/e2e/ssh-localhost.spec.ts",
"assertions": ["routes a terminal and agent-hook status over localhost SSH"]
},
{
"file": "config/scripts/ssh-localhost-e2e-routing.test.mjs",
"assertions": ["selects the localhost journey for its remote hook authorities"]
}
],
"evidenceRuns": [
{
"date": "2026-09-06",
"runner": "ci",
"platform": "linux",
"command": "gh run view 34045578306 --log",
"result": "failed",
"summary": "Shared repository:2passed1failed, active pane PTY binding timed out amid old SSH target ownership conflicts.",
"durationSeconds": 150
},
{
"date": "2026-09-06",
"runner": "ci",
"platform": "linux",
"command": "gh run view 34045975180 --log",
"result": "passed",
"durationSeconds": 114,
"summary": "Fresh per-test repository:3passed,0skips0retries; original assertions retained."
},
{
"date": "2026-09-06",
"runner": "ci",
"platform": "linux",
"command": "gh run view 34046230389 --log",
"result": "passed",
"summary": "Normal selective workflow with isolated repository executed the localhost journey successfully; generic lane filtered it out.",
"durationSeconds": 36.4
}
],
"runtimeBudget": {
"p95Seconds": 1200,
"scope": "CI job timeout; measured p95 not established"
},
"flakeHistory": {
"status": "soaking",
"evidence": "Single baseline passed, shared-path repetitions exposed state leakage; isolated-path3/3 and normal workflow passed. Long-term history missing."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Same original scenario failed across shared-path repetitions and passed with unique paths; no application fault-mutation proof."
},
"performanceBudget": {
"required": false,
"evidence": "Functional terminal and hook routing coverage, not a performance oracle."
},
"promotionCriteria": [
"Collect repeated scheduled Linux runs without unexplained failures.",
"Preserve all original terminal, environment, plugin-file, and hook-status assertions."
],
"knownGaps": [
"Different client profiles reopening one existing remote workspace can encounter old target-qualified PTY IDs; the fixture isolation does not fix that application behavior.",
"No macOS/Windows, remote network failure, folder-only, packaged, or mixed-version claim.",
"PR E2E is not part of required verify while broader reliability remains unresolved."
],
"demotionRule": "Keep experimental on unexplained failures; do not mask them with retries, skips, or longer timeouts."
},
{
"id": "terminal-output.prestarted-shell-snapshot-adoption",
"title": "Prestarted shell adoption paints covered output once",
+9
View File
@@ -13,6 +13,15 @@ const NATIVE_IME_HARNESS =
/^(?:config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/
export const PR_E2E_SOURCE_ROUTES = [
{
id: 'ssh.localhost-agent-hooks',
specs: ['tests/e2e/ssh-localhost.spec.ts'],
matches: (file) =>
isProductSource(file) &&
/^src\/(?:relay\/(?:agent-hook|relay-agent-hook-runtime|plugin-overlay)|main\/(?:agent-hooks\/|ssh\/ssh-relay-session\.ts$)|shared\/agent-hook)/.test(
file
)
},
{
id: 'browser-network.ssh-docker-route',
specs: ['tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts'],
@@ -13,6 +13,7 @@ const EXPECTED_MATRIX = {
'.github/workflows/e2e.yml#e2e': { contents: 'read' },
'.github/workflows/e2e.yml#prepare-native-cache': { contents: 'read' },
'.github/workflows/e2e.yml#ssh-browser-network-route': { contents: 'read' },
'.github/workflows/e2e.yml#ssh-localhost': { contents: 'read' },
'.github/workflows/e2e.yml#ssh-docker-watcher-isolation': { contents: 'read' },
'.github/workflows/homebrew-bump.yml#bump-cask': { contents: 'read' },
'.github/workflows/release-mac-build.yml#build-mac': { contents: 'write' },
+1 -2
View File
@@ -48,10 +48,9 @@ if (runtime.status !== 0) {
// - E2E does not gate merges: `verify.needs` in pr.yml omits `e2e` while the suite is red on
// main. Nothing in this lane blocks a PR yet. pr.yml's Require-successful-checks comment
// has the exact wiring to flip it, and the gate contract asserts the current state.
// - Three specs are gated on env vars no workflow sets, so they run nowhere
// - Two specs are gated on env vars no workflow sets, so they run nowhere
// and are not Docker-gated, which puts them outside this file's contract:
// nested-runtime-ssh-lifecycle, nested-runtime-ssh-routing (ORCA_E2E_NESTED_RUNTIME_SSH)
// ssh-localhost (ORCA_E2E_SSH_LOCALHOST)
// The nested-runtime runner remains unused by CI.
const result = spawnSync(
pnpm,
@@ -0,0 +1,52 @@
import { existsSync, readFileSync } from 'node:fs'
import { resolve } from 'node:path'
import { parse } from 'yaml'
import { expect, it } from 'vitest'
import { selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
const workflow = parse(
readFileSync(resolve(import.meta.dirname, '../../.github/workflows/e2e.yml'), 'utf8')
)
it('gives the localhost SSH journey its same-filesystem server and agent prerequisite', () => {
const spec = 'tests/e2e/ssh-localhost.spec.ts'
const job = workflow.jobs['ssh-localhost']
expect(job.if).toContain("inputs.test_files == ''")
expect(job.if).toContain(spec)
expect(job['runs-on']).toBe('ubuntu-latest')
expect(job.needs).toEqual(['build', 'prepare-native-cache'])
const setup = job.steps.find((step) => step.name === 'Start isolated localhost SSH server')
expect(setup.run).toContain('ListenAddress 127.0.0.1')
expect(setup.run).toContain('PasswordAuthentication no')
expect(setup.run).toContain('UsePAM yes')
expect(setup.run).toContain('mkdir -p "$HOME/.pi/agent"')
for (const key of ['ORCA_E2E_SSH_PORT', 'ORCA_E2E_SSH_USER', 'ORCA_E2E_SSH_IDENTITY_FILE']) {
expect(setup.run).toContain(key)
}
const run = job.steps.find((step) => step.name === 'Run localhost SSH terminal and hook journey')
expect(run.env.ORCA_E2E_SSH_LOCALHOST).toBe('1')
expect(run.env.ORCA_FEATURE_REMOTE_AGENT_HOOKS).toBe('1')
expect(run.run).toContain(spec)
expect(run.run).toContain('--project=electron-headless')
expect(run.run).not.toContain('--retries')
expect(run['continue-on-error']).toBeUndefined()
expect(
workflow.jobs['changed-e2e'].steps.find((step) => step.name === 'Run changed E2E specs').run
).toContain(`. != "${spec}"`)
})
it('selects the localhost journey for its remote hook authorities', () => {
const spec = 'tests/e2e/ssh-localhost.spec.ts'
for (const file of [
'src/relay/relay-agent-hook-runtime.ts',
'src/relay/agent-hook-server.ts',
'src/relay/plugin-overlay.ts',
'src/main/agent-hooks/server.ts',
'src/main/ssh/ssh-relay-session.ts',
'src/shared/agent-hook-relay.ts'
]) {
expect(existsSync(resolve(import.meta.dirname, '../..', file)), file).toBe(true)
expect(selectPrE2eSpecs([file])).toContain(spec)
}
expect(selectPrE2eSpecs(['src/renderer/src/components/Unrelated.tsx'])).not.toContain(spec)
})
+6 -1
View File
@@ -1,4 +1,6 @@
import os from 'node:os'
import { createSeededTestRepo } from './helpers/seeded-test-repo'
import { cleanupTestRepository } from './global-teardown'
import type { Page } from '@stablyai/playwright-test'
import { test, expect } from './helpers/orca-app'
@@ -151,9 +153,12 @@ test.describe('Localhost SSH', () => {
test('routes a terminal and agent-hook status over localhost SSH', async ({
orcaPage,
testRepoPath
registerPostElectronShutdownCleanup
}) => {
test.slow()
// The relay persists workspace sessions by path across fresh client profiles.
const testRepoPath = createSeededTestRepo({ publishPath: false })
registerPostElectronShutdownCleanup(async () => cleanupTestRepository(testRepoPath))
await waitForSessionReady(orcaPage)
await waitForActiveWorktree(orcaPage)