mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
test: enable localhost SSH terminal and hook journey in CI (#19097)
* test: run localhost SSH terminal and hooks in CI * test: isolate localhost SSH session fixtures across repetitions * test: route remote agent hook source changes to localhost journey * test: record localhost SSH reliability evidence and remaining gaps * test: route the real SSH session hook authority
This commit is contained in:
@@ -229,6 +229,7 @@ jobs:
|
||||
. != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
|
||||
. != "tests/e2e/local-ssh-browser-routing.spec.ts" and
|
||||
. != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
|
||||
. != "tests/e2e/ssh-localhost.spec.ts" and
|
||||
. != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and
|
||||
. != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
|
||||
. != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
|
||||
@@ -375,3 +376,67 @@ jobs:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
|
||||
run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
|
||||
|
||||
ssh-localhost:
|
||||
name: localhost SSH terminal and hooks
|
||||
needs: [build, prepare-native-cache]
|
||||
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
- name: Install SSH server and headless tools
|
||||
run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: electron
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: e2e-build-out
|
||||
path: out/
|
||||
- name: Start isolated localhost SSH server
|
||||
shell: bash
|
||||
run: |
|
||||
# Bare shells install Pi extensions only for an existing agent home.
|
||||
mkdir -p "$HOME/.pi/agent"
|
||||
fixture="$RUNNER_TEMP/orca-localhost-sshd"
|
||||
mkdir -p "$fixture"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key"
|
||||
ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key"
|
||||
cat > "$fixture/sshd_config" <<EOF
|
||||
Port 22222
|
||||
ListenAddress 127.0.0.1
|
||||
HostKey $fixture/host_key
|
||||
PidFile $fixture/sshd.pid
|
||||
AuthorizedKeysFile $fixture/client_key.pub
|
||||
StrictModes no
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
UsePAM yes
|
||||
AllowUsers $(id -un)
|
||||
Subsystem sftp internal-sftp
|
||||
EOF
|
||||
sudo mkdir -p /run/sshd
|
||||
sudo /usr/sbin/sshd -f "$fixture/sshd_config" -E "$fixture/sshd.log"
|
||||
ssh -i "$fixture/client_key" -p 22222 -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 127.0.0.1 true || { sudo cat "$fixture/sshd.log"; exit 1; }
|
||||
{
|
||||
echo "ORCA_E2E_SSH_PORT=22222"
|
||||
echo "ORCA_E2E_SSH_USER=$(id -un)"
|
||||
echo "ORCA_E2E_SSH_IDENTITY_FILE=$fixture/client_key"
|
||||
} >> "$GITHUB_ENV"
|
||||
- name: Run localhost SSH terminal and hook journey
|
||||
env:
|
||||
SKIP_BUILD: '1'
|
||||
ORCA_E2E_SSH_LOCALHOST: '1'
|
||||
ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1'
|
||||
ORCA_E2E_FORWARD_APP_LOGS: '1'
|
||||
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: failure()
|
||||
with:
|
||||
name: localhost-ssh-traces
|
||||
path: test-results/
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -10,6 +10,99 @@
|
||||
}
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"id": "ssh.localhost-terminal-agent-hooks",
|
||||
"title": "Localhost SSH terminal and agent hooks reach the owning pane",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "terminal-runtime",
|
||||
"layer": "electron-ssh-e2e",
|
||||
"surfaces": ["SSH terminal", "remote agent status", "remote plugin installation"],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["ssh"],
|
||||
"coveredPlatforms": ["linux"],
|
||||
"coveredProviders": ["ssh"],
|
||||
"coverageNotes": "Ubuntu CI loopback sshd shares the runner filesystem. Fresh per-test repositories isolate retained relay workspace snapshots; existing Pi home supplies the documented bare-shell plugin prerequisite.",
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19097"],
|
||||
"invariant": "A localhost SSH terminal executes on the SSH host and routes authenticated hook status to its owning pane without treating idle keyboard input as agent interruption.",
|
||||
"oracle": "Require terminal output markers, exported hook identity, actual OpenCode/Pi plugin files, and matching pane/worktree/connection hook events; Ctrl-C and Escape in an idle shell must not interrupt a hook-owned agent.",
|
||||
"commands": [
|
||||
"gh run view 34045578306 --log",
|
||||
"gh run view 34045975180 --log",
|
||||
"gh run view 34046230389 --log",
|
||||
"ORCA_E2E_SSH_LOCALHOST=1 ORCA_FEATURE_REMOTE_AGENT_HOOKS=1 pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1",
|
||||
"node_modules/.bin/vitest run --config config/vitest.config.ts config/scripts/ssh-localhost-e2e-routing.test.mjs"
|
||||
],
|
||||
"testFiles": [
|
||||
"tests/e2e/ssh-localhost.spec.ts",
|
||||
"config/scripts/ssh-localhost-e2e-routing.test.mjs"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "tests/e2e/ssh-localhost.spec.ts",
|
||||
"assertions": ["routes a terminal and agent-hook status over localhost SSH"]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/ssh-localhost-e2e-routing.test.mjs",
|
||||
"assertions": ["selects the localhost journey for its remote hook authorities"]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-09-06",
|
||||
"runner": "ci",
|
||||
"platform": "linux",
|
||||
"command": "gh run view 34045578306 --log",
|
||||
"result": "failed",
|
||||
"summary": "Shared repository:2passed1failed, active pane PTY binding timed out amid old SSH target ownership conflicts.",
|
||||
"durationSeconds": 150
|
||||
},
|
||||
{
|
||||
"date": "2026-09-06",
|
||||
"runner": "ci",
|
||||
"platform": "linux",
|
||||
"command": "gh run view 34045975180 --log",
|
||||
"result": "passed",
|
||||
"durationSeconds": 114,
|
||||
"summary": "Fresh per-test repository:3passed,0skips0retries; original assertions retained."
|
||||
},
|
||||
{
|
||||
"date": "2026-09-06",
|
||||
"runner": "ci",
|
||||
"platform": "linux",
|
||||
"command": "gh run view 34046230389 --log",
|
||||
"result": "passed",
|
||||
"summary": "Normal selective workflow with isolated repository executed the localhost journey successfully; generic lane filtered it out.",
|
||||
"durationSeconds": 36.4
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 1200,
|
||||
"scope": "CI job timeout; measured p95 not established"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "soaking",
|
||||
"evidence": "Single baseline passed, shared-path repetitions exposed state leakage; isolated-path3/3 and normal workflow passed. Long-term history missing."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "Same original scenario failed across shared-path repetitions and passed with unique paths; no application fault-mutation proof."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": false,
|
||||
"evidence": "Functional terminal and hook routing coverage, not a performance oracle."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect repeated scheduled Linux runs without unexplained failures.",
|
||||
"Preserve all original terminal, environment, plugin-file, and hook-status assertions."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Different client profiles reopening one existing remote workspace can encounter old target-qualified PTY IDs; the fixture isolation does not fix that application behavior.",
|
||||
"No macOS/Windows, remote network failure, folder-only, packaged, or mixed-version claim.",
|
||||
"PR E2E is not part of required verify while broader reliability remains unresolved."
|
||||
],
|
||||
"demotionRule": "Keep experimental on unexplained failures; do not mask them with retries, skips, or longer timeouts."
|
||||
},
|
||||
{
|
||||
"id": "terminal-output.prestarted-shell-snapshot-adoption",
|
||||
"title": "Prestarted shell adoption paints covered output once",
|
||||
|
||||
@@ -13,6 +13,15 @@ const NATIVE_IME_HARNESS =
|
||||
/^(?:config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/
|
||||
|
||||
export const PR_E2E_SOURCE_ROUTES = [
|
||||
{
|
||||
id: 'ssh.localhost-agent-hooks',
|
||||
specs: ['tests/e2e/ssh-localhost.spec.ts'],
|
||||
matches: (file) =>
|
||||
isProductSource(file) &&
|
||||
/^src\/(?:relay\/(?:agent-hook|relay-agent-hook-runtime|plugin-overlay)|main\/(?:agent-hooks\/|ssh\/ssh-relay-session\.ts$)|shared\/agent-hook)/.test(
|
||||
file
|
||||
)
|
||||
},
|
||||
{
|
||||
id: 'browser-network.ssh-docker-route',
|
||||
specs: ['tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts'],
|
||||
|
||||
@@ -13,6 +13,7 @@ const EXPECTED_MATRIX = {
|
||||
'.github/workflows/e2e.yml#e2e': { contents: 'read' },
|
||||
'.github/workflows/e2e.yml#prepare-native-cache': { contents: 'read' },
|
||||
'.github/workflows/e2e.yml#ssh-browser-network-route': { contents: 'read' },
|
||||
'.github/workflows/e2e.yml#ssh-localhost': { contents: 'read' },
|
||||
'.github/workflows/e2e.yml#ssh-docker-watcher-isolation': { contents: 'read' },
|
||||
'.github/workflows/homebrew-bump.yml#bump-cask': { contents: 'read' },
|
||||
'.github/workflows/release-mac-build.yml#build-mac': { contents: 'write' },
|
||||
|
||||
@@ -48,10 +48,9 @@ if (runtime.status !== 0) {
|
||||
// - E2E does not gate merges: `verify.needs` in pr.yml omits `e2e` while the suite is red on
|
||||
// main. Nothing in this lane blocks a PR yet. pr.yml's Require-successful-checks comment
|
||||
// has the exact wiring to flip it, and the gate contract asserts the current state.
|
||||
// - Three specs are gated on env vars no workflow sets, so they run nowhere
|
||||
// - Two specs are gated on env vars no workflow sets, so they run nowhere
|
||||
// and are not Docker-gated, which puts them outside this file's contract:
|
||||
// nested-runtime-ssh-lifecycle, nested-runtime-ssh-routing (ORCA_E2E_NESTED_RUNTIME_SSH)
|
||||
// ssh-localhost (ORCA_E2E_SSH_LOCALHOST)
|
||||
// The nested-runtime runner remains unused by CI.
|
||||
const result = spawnSync(
|
||||
pnpm,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
import { parse } from 'yaml'
|
||||
import { expect, it } from 'vitest'
|
||||
import { selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
|
||||
|
||||
const workflow = parse(
|
||||
readFileSync(resolve(import.meta.dirname, '../../.github/workflows/e2e.yml'), 'utf8')
|
||||
)
|
||||
|
||||
it('gives the localhost SSH journey its same-filesystem server and agent prerequisite', () => {
|
||||
const spec = 'tests/e2e/ssh-localhost.spec.ts'
|
||||
const job = workflow.jobs['ssh-localhost']
|
||||
expect(job.if).toContain("inputs.test_files == ''")
|
||||
expect(job.if).toContain(spec)
|
||||
expect(job['runs-on']).toBe('ubuntu-latest')
|
||||
expect(job.needs).toEqual(['build', 'prepare-native-cache'])
|
||||
const setup = job.steps.find((step) => step.name === 'Start isolated localhost SSH server')
|
||||
expect(setup.run).toContain('ListenAddress 127.0.0.1')
|
||||
expect(setup.run).toContain('PasswordAuthentication no')
|
||||
expect(setup.run).toContain('UsePAM yes')
|
||||
expect(setup.run).toContain('mkdir -p "$HOME/.pi/agent"')
|
||||
for (const key of ['ORCA_E2E_SSH_PORT', 'ORCA_E2E_SSH_USER', 'ORCA_E2E_SSH_IDENTITY_FILE']) {
|
||||
expect(setup.run).toContain(key)
|
||||
}
|
||||
const run = job.steps.find((step) => step.name === 'Run localhost SSH terminal and hook journey')
|
||||
expect(run.env.ORCA_E2E_SSH_LOCALHOST).toBe('1')
|
||||
expect(run.env.ORCA_FEATURE_REMOTE_AGENT_HOOKS).toBe('1')
|
||||
expect(run.run).toContain(spec)
|
||||
expect(run.run).toContain('--project=electron-headless')
|
||||
expect(run.run).not.toContain('--retries')
|
||||
expect(run['continue-on-error']).toBeUndefined()
|
||||
expect(
|
||||
workflow.jobs['changed-e2e'].steps.find((step) => step.name === 'Run changed E2E specs').run
|
||||
).toContain(`. != "${spec}"`)
|
||||
})
|
||||
|
||||
it('selects the localhost journey for its remote hook authorities', () => {
|
||||
const spec = 'tests/e2e/ssh-localhost.spec.ts'
|
||||
for (const file of [
|
||||
'src/relay/relay-agent-hook-runtime.ts',
|
||||
'src/relay/agent-hook-server.ts',
|
||||
'src/relay/plugin-overlay.ts',
|
||||
'src/main/agent-hooks/server.ts',
|
||||
'src/main/ssh/ssh-relay-session.ts',
|
||||
'src/shared/agent-hook-relay.ts'
|
||||
]) {
|
||||
expect(existsSync(resolve(import.meta.dirname, '../..', file)), file).toBe(true)
|
||||
expect(selectPrE2eSpecs([file])).toContain(spec)
|
||||
}
|
||||
expect(selectPrE2eSpecs(['src/renderer/src/components/Unrelated.tsx'])).not.toContain(spec)
|
||||
})
|
||||
@@ -1,4 +1,6 @@
|
||||
import os from 'node:os'
|
||||
import { createSeededTestRepo } from './helpers/seeded-test-repo'
|
||||
import { cleanupTestRepository } from './global-teardown'
|
||||
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
@@ -151,9 +153,12 @@ test.describe('Localhost SSH', () => {
|
||||
|
||||
test('routes a terminal and agent-hook status over localhost SSH', async ({
|
||||
orcaPage,
|
||||
testRepoPath
|
||||
registerPostElectronShutdownCleanup
|
||||
}) => {
|
||||
test.slow()
|
||||
// The relay persists workspace sessions by path across fresh client profiles.
|
||||
const testRepoPath = createSeededTestRepo({ publishPath: false })
|
||||
registerPostElectronShutdownCleanup(async () => cleanupTestRepository(testRepoPath))
|
||||
await waitForSessionReady(orcaPage)
|
||||
await waitForActiveWorktree(orcaPage)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user