diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e000cdab4ba..f75d7ba00bb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -229,6 +229,7 @@ jobs: . != "tests/e2e/paired-startup-exec-readiness.spec.ts" and . != "tests/e2e/local-ssh-browser-routing.spec.ts" and . != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and + . != "tests/e2e/ssh-localhost.spec.ts" and . != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and . != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and . != "tests/e2e/terminal-ibus-hangul-native.spec.ts" @@ -375,3 +376,67 @@ jobs: ORCA_BACKGROUND_LAUNCH: '1' ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1' run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts + + ssh-localhost: + name: localhost SSH terminal and hooks + needs: [build, prepare-native-cache] + if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts') + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + - name: Install SSH server and headless tools + run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: electron + - uses: actions/download-artifact@v8 + with: + name: e2e-build-out + path: out/ + - name: Start isolated localhost SSH server + shell: bash + run: | + # Bare shells install Pi extensions only for an existing agent home. + mkdir -p "$HOME/.pi/agent" + fixture="$RUNNER_TEMP/orca-localhost-sshd" + mkdir -p "$fixture" + ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key" + ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key" + cat > "$fixture/sshd_config" <> "$GITHUB_ENV" + - name: Run localhost SSH terminal and hook journey + env: + SKIP_BUILD: '1' + ORCA_E2E_SSH_LOCALHOST: '1' + ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1' + ORCA_E2E_FORWARD_APP_LOGS: '1' + run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1 + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: localhost-ssh-traces + path: test-results/ + retention-days: 7 + if-no-files-found: ignore diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index b794519c9b7..21209935826 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,99 @@ } }, "gates": [ + { + "id": "ssh.localhost-terminal-agent-hooks", + "title": "Localhost SSH terminal and agent hooks reach the owning pane", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "electron-ssh-e2e", + "surfaces": ["SSH terminal", "remote agent status", "remote plugin installation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["ssh"], + "coveredPlatforms": ["linux"], + "coveredProviders": ["ssh"], + "coverageNotes": "Ubuntu CI loopback sshd shares the runner filesystem. Fresh per-test repositories isolate retained relay workspace snapshots; existing Pi home supplies the documented bare-shell plugin prerequisite.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/19097"], + "invariant": "A localhost SSH terminal executes on the SSH host and routes authenticated hook status to its owning pane without treating idle keyboard input as agent interruption.", + "oracle": "Require terminal output markers, exported hook identity, actual OpenCode/Pi plugin files, and matching pane/worktree/connection hook events; Ctrl-C and Escape in an idle shell must not interrupt a hook-owned agent.", + "commands": [ + "gh run view 34045578306 --log", + "gh run view 34045975180 --log", + "gh run view 34046230389 --log", + "ORCA_E2E_SSH_LOCALHOST=1 ORCA_FEATURE_REMOTE_AGENT_HOOKS=1 pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1", + "node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/ssh-localhost-e2e-routing.test.mjs" + ], + "testFiles": [ + "tests/e2e/ssh-localhost.spec.ts", + "config/scripts/ssh-localhost-e2e-routing.test.mjs" + ], + "assertionRefs": [ + { + "file": "tests/e2e/ssh-localhost.spec.ts", + "assertions": ["routes a terminal and agent-hook status over localhost SSH"] + }, + { + "file": "config/scripts/ssh-localhost-e2e-routing.test.mjs", + "assertions": ["selects the localhost journey for its remote hook authorities"] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-06", + "runner": "ci", + "platform": "linux", + "command": "gh run view 34045578306 --log", + "result": "failed", + "summary": "Shared repository:2passed1failed, active pane PTY binding timed out amid old SSH target ownership conflicts.", + "durationSeconds": 150 + }, + { + "date": "2026-09-06", + "runner": "ci", + "platform": "linux", + "command": "gh run view 34045975180 --log", + "result": "passed", + "durationSeconds": 114, + "summary": "Fresh per-test repository:3passed,0skips0retries; original assertions retained." + }, + { + "date": "2026-09-06", + "runner": "ci", + "platform": "linux", + "command": "gh run view 34046230389 --log", + "result": "passed", + "summary": "Normal selective workflow with isolated repository executed the localhost journey successfully; generic lane filtered it out.", + "durationSeconds": 36.4 + } + ], + "runtimeBudget": { + "p95Seconds": 1200, + "scope": "CI job timeout; measured p95 not established" + }, + "flakeHistory": { + "status": "soaking", + "evidence": "Single baseline passed, shared-path repetitions exposed state leakage; isolated-path3/3 and normal workflow passed. Long-term history missing." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "Same original scenario failed across shared-path repetitions and passed with unique paths; no application fault-mutation proof." + }, + "performanceBudget": { + "required": false, + "evidence": "Functional terminal and hook routing coverage, not a performance oracle." + }, + "promotionCriteria": [ + "Collect repeated scheduled Linux runs without unexplained failures.", + "Preserve all original terminal, environment, plugin-file, and hook-status assertions." + ], + "knownGaps": [ + "Different client profiles reopening one existing remote workspace can encounter old target-qualified PTY IDs; the fixture isolation does not fix that application behavior.", + "No macOS/Windows, remote network failure, folder-only, packaged, or mixed-version claim.", + "PR E2E is not part of required verify while broader reliability remains unresolved." + ], + "demotionRule": "Keep experimental on unexplained failures; do not mask them with retries, skips, or longer timeouts." + }, { "id": "terminal-output.prestarted-shell-snapshot-adoption", "title": "Prestarted shell adoption paints covered output once", diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index 4b1c1930892..18c6b032788 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -13,6 +13,15 @@ const NATIVE_IME_HARNESS = /^(?:config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/ export const PR_E2E_SOURCE_ROUTES = [ + { + id: 'ssh.localhost-agent-hooks', + specs: ['tests/e2e/ssh-localhost.spec.ts'], + matches: (file) => + isProductSource(file) && + /^src\/(?:relay\/(?:agent-hook|relay-agent-hook-runtime|plugin-overlay)|main\/(?:agent-hooks\/|ssh\/ssh-relay-session\.ts$)|shared\/agent-hook)/.test( + file + ) + }, { id: 'browser-network.ssh-docker-route', specs: ['tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts'], diff --git a/config/scripts/release-cut-token-permissions.test.mjs b/config/scripts/release-cut-token-permissions.test.mjs index 85f36dea3a0..0fc1e5f8448 100644 --- a/config/scripts/release-cut-token-permissions.test.mjs +++ b/config/scripts/release-cut-token-permissions.test.mjs @@ -13,6 +13,7 @@ const EXPECTED_MATRIX = { '.github/workflows/e2e.yml#e2e': { contents: 'read' }, '.github/workflows/e2e.yml#prepare-native-cache': { contents: 'read' }, '.github/workflows/e2e.yml#ssh-browser-network-route': { contents: 'read' }, + '.github/workflows/e2e.yml#ssh-localhost': { contents: 'read' }, '.github/workflows/e2e.yml#ssh-docker-watcher-isolation': { contents: 'read' }, '.github/workflows/homebrew-bump.yml#bump-cask': { contents: 'read' }, '.github/workflows/release-mac-build.yml#build-mac': { contents: 'write' }, diff --git a/config/scripts/run-ssh-docker-e2e.mjs b/config/scripts/run-ssh-docker-e2e.mjs index 4b9d51137f1..b88bde609bb 100644 --- a/config/scripts/run-ssh-docker-e2e.mjs +++ b/config/scripts/run-ssh-docker-e2e.mjs @@ -48,10 +48,9 @@ if (runtime.status !== 0) { // - E2E does not gate merges: `verify.needs` in pr.yml omits `e2e` while the suite is red on // main. Nothing in this lane blocks a PR yet. pr.yml's Require-successful-checks comment // has the exact wiring to flip it, and the gate contract asserts the current state. -// - Three specs are gated on env vars no workflow sets, so they run nowhere +// - Two specs are gated on env vars no workflow sets, so they run nowhere // and are not Docker-gated, which puts them outside this file's contract: // nested-runtime-ssh-lifecycle, nested-runtime-ssh-routing (ORCA_E2E_NESTED_RUNTIME_SSH) -// ssh-localhost (ORCA_E2E_SSH_LOCALHOST) // The nested-runtime runner remains unused by CI. const result = spawnSync( pnpm, diff --git a/config/scripts/ssh-localhost-e2e-routing.test.mjs b/config/scripts/ssh-localhost-e2e-routing.test.mjs new file mode 100644 index 00000000000..b400e86153c --- /dev/null +++ b/config/scripts/ssh-localhost-e2e-routing.test.mjs @@ -0,0 +1,52 @@ +import { existsSync, readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { parse } from 'yaml' +import { expect, it } from 'vitest' +import { selectPrE2eSpecs } from './pr-e2e-source-routing.mjs' + +const workflow = parse( + readFileSync(resolve(import.meta.dirname, '../../.github/workflows/e2e.yml'), 'utf8') +) + +it('gives the localhost SSH journey its same-filesystem server and agent prerequisite', () => { + const spec = 'tests/e2e/ssh-localhost.spec.ts' + const job = workflow.jobs['ssh-localhost'] + expect(job.if).toContain("inputs.test_files == ''") + expect(job.if).toContain(spec) + expect(job['runs-on']).toBe('ubuntu-latest') + expect(job.needs).toEqual(['build', 'prepare-native-cache']) + const setup = job.steps.find((step) => step.name === 'Start isolated localhost SSH server') + expect(setup.run).toContain('ListenAddress 127.0.0.1') + expect(setup.run).toContain('PasswordAuthentication no') + expect(setup.run).toContain('UsePAM yes') + expect(setup.run).toContain('mkdir -p "$HOME/.pi/agent"') + for (const key of ['ORCA_E2E_SSH_PORT', 'ORCA_E2E_SSH_USER', 'ORCA_E2E_SSH_IDENTITY_FILE']) { + expect(setup.run).toContain(key) + } + const run = job.steps.find((step) => step.name === 'Run localhost SSH terminal and hook journey') + expect(run.env.ORCA_E2E_SSH_LOCALHOST).toBe('1') + expect(run.env.ORCA_FEATURE_REMOTE_AGENT_HOOKS).toBe('1') + expect(run.run).toContain(spec) + expect(run.run).toContain('--project=electron-headless') + expect(run.run).not.toContain('--retries') + expect(run['continue-on-error']).toBeUndefined() + expect( + workflow.jobs['changed-e2e'].steps.find((step) => step.name === 'Run changed E2E specs').run + ).toContain(`. != "${spec}"`) +}) + +it('selects the localhost journey for its remote hook authorities', () => { + const spec = 'tests/e2e/ssh-localhost.spec.ts' + for (const file of [ + 'src/relay/relay-agent-hook-runtime.ts', + 'src/relay/agent-hook-server.ts', + 'src/relay/plugin-overlay.ts', + 'src/main/agent-hooks/server.ts', + 'src/main/ssh/ssh-relay-session.ts', + 'src/shared/agent-hook-relay.ts' + ]) { + expect(existsSync(resolve(import.meta.dirname, '../..', file)), file).toBe(true) + expect(selectPrE2eSpecs([file])).toContain(spec) + } + expect(selectPrE2eSpecs(['src/renderer/src/components/Unrelated.tsx'])).not.toContain(spec) +}) diff --git a/tests/e2e/ssh-localhost.spec.ts b/tests/e2e/ssh-localhost.spec.ts index 1117fcb9409..afb3e780b95 100644 --- a/tests/e2e/ssh-localhost.spec.ts +++ b/tests/e2e/ssh-localhost.spec.ts @@ -1,4 +1,6 @@ import os from 'node:os' +import { createSeededTestRepo } from './helpers/seeded-test-repo' +import { cleanupTestRepository } from './global-teardown' import type { Page } from '@stablyai/playwright-test' import { test, expect } from './helpers/orca-app' @@ -151,9 +153,12 @@ test.describe('Localhost SSH', () => { test('routes a terminal and agent-hook status over localhost SSH', async ({ orcaPage, - testRepoPath + registerPostElectronShutdownCleanup }) => { test.slow() + // The relay persists workspace sessions by path across fresh client profiles. + const testRepoPath = createSeededTestRepo({ publishPath: false }) + registerPostElectronShutdownCleanup(async () => cleanupTestRepository(testRepoPath)) await waitForSessionReady(orcaPage) await waitForActiveWorktree(orcaPage)