diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 29996c70361..e6b16fab7a7 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -9,6 +9,12 @@ on: - 'config/scripts/build-windows-cli-launcher.mjs' - 'config/scripts/build-windows-cli-launcher.test.mjs' - 'config/scripts/computer-e2e-workflow.test.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-processes.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs' - 'config/scripts/computer-use-modifier-safety.test.mjs' - 'config/scripts/computer-use-skill-guidance.test.mjs' - 'config/scripts/computer-use-smoke.mjs' @@ -59,6 +65,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: actions/setup-node@v6 with: node-version-file: package.json @@ -85,6 +93,8 @@ jobs: config/scripts/build-windows-cli-launcher.test.mjs src/main/ssh/ssh-remote-cli-launcher.test.ts config/scripts/computer-e2e-workflow.test.mjs + config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs + config/scripts/macos-computer-helper-owner-loss-processes.test.mjs config/scripts/computer-use-modifier-safety.test.mjs config/scripts/computer-use-skill-guidance.test.mjs config/scripts/computer-use-smoke.test.mjs @@ -140,6 +150,32 @@ jobs: ORCA_COMPUTER_E2E: '1' run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts + mac-native-owner-smoke: + if: github.event_name == 'pull_request' + runs-on: macos-15 + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version-file: package.json + - uses: pnpm/action-setup@v6 + with: + run_install: false + - run: pnpm install --frozen-lockfile + - name: Owner-loss benchmark process cleanup + run: >- + pnpm vitest run + config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs + config/scripts/macos-computer-helper-owner-loss-processes.test.mjs + - name: Authenticated helper owner-loss smoke + run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1 + - name: Swift tests and signed universal helper verification + run: pnpm verify:computer-native + mac: # macOS Accessibility and Screen Recording require user-granted TCC entries. # Keep this on manual/scheduled permission-bearing runners instead of PR CI. diff --git a/config/scripts/computer-e2e-workflow.test.mjs b/config/scripts/computer-e2e-workflow.test.mjs index d6db7c5bbe9..a4230668bc7 100644 --- a/config/scripts/computer-e2e-workflow.test.mjs +++ b/config/scripts/computer-e2e-workflow.test.mjs @@ -47,6 +47,7 @@ describe('computer-use e2e workflow', () => { expect(triggerPaths).toEqual( expect.arrayContaining([ 'config/scripts/computer-e2e-workflow.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs', 'config/scripts/computer-use-modifier-safety.test.mjs', 'config/scripts/computer-use-skill-guidance.test.mjs', 'config/scripts/computer-use-smoke.mjs', @@ -70,9 +71,14 @@ describe('computer-use e2e workflow', () => { const nativeSmokeRuns = workflow.jobs['native-smoke'].steps .map((step) => step.run) .filter((run) => typeof run === 'string') + const checkout = workflow.jobs['native-smoke'].steps.find( + (step) => step.uses === 'actions/checkout@v6' + ) const regressionRun = nativeSmokeRuns.find((run) => run.includes('pnpm vitest run')) const expectedRegressionFiles = [ 'config/scripts/computer-e2e-workflow.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs', 'config/scripts/computer-use-modifier-safety.test.mjs', 'config/scripts/computer-use-skill-guidance.test.mjs', 'config/scripts/computer-use-smoke.test.mjs', @@ -106,12 +112,68 @@ describe('computer-use e2e workflow', () => { 'src/shared/remote-runtime-client.test.ts' ] + expect(checkout.with['persist-credentials']).toBe(false) expect(regressionRun).toBeTruthy() for (const file of expectedRegressionFiles) { expect(regressionRun).toContain(file) } }) + it('builds and tests the macOS helper on pull requests without TCC e2e', () => { + const workflow = parse( + readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') + ) + const job = workflow.jobs['mac-native-owner-smoke'] + const runs = job.steps.map((step) => step.run).filter((run) => typeof run === 'string') + const checkout = job.steps.find((step) => step.uses === 'actions/checkout@v6') + + expect(job.if).toBe("github.event_name == 'pull_request'") + expect(job['runs-on']).toBe('macos-15') + expect(checkout.with['persist-credentials']).toBe(false) + expect(runs).toContain('pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1') + const cleanupRun = runs.find((run) => + run.includes('config/scripts/macos-computer-helper-owner-loss-processes.test.mjs') + ) + expect(cleanupRun).toContain( + 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs' + ) + expect(runs).toContain('pnpm verify:computer-native') + expect(runs.join('\n')).not.toContain('test:e2e:computer') + expect(workflow.on.pull_request.paths).toEqual( + expect.arrayContaining([ + 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs', + 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs', + 'config/scripts/macos-computer-helper-owner-loss-processes.mjs', + 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs' + ]) + ) + }) + + it('runs deterministic macOS owner-loss benchmark cleanup coverage', () => { + const benchmark = readFileSync( + join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'), + 'utf8' + ) + const cleanup = readFileSync( + join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'), + 'utf8' + ) + + expect(benchmark).toContain('spawnBenchmarkProcess(executable, [launcherDir]') + expect(benchmark).toContain("stdio: ['ignore', stdoutDescriptor, stderrDescriptor]") + expect(benchmark).toContain('cleanupOwnerLossTrial({') + const parseIndex = benchmark.indexOf('parseBenchmarkTrialResult(serializedResult)') + const cleanupIndex = benchmark.indexOf('cleanupOwnerLossTrial({') + expect(parseIndex).toBeGreaterThanOrEqual(0) + expect(cleanupIndex).toBeGreaterThanOrEqual(0) + expect(parseIndex).toBeLessThan(cleanupIndex) + expect(benchmark).toContain('trialCleanupSha256: artifactSha256(trialCleanupPath)') + expect(cleanup).toContain('killRecordedAndMatchingProcesses(options.recordPath') + expect(cleanup).toContain("signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL'") + }) + it('boots the built daemon under plain Node in the PR native-smoke job after the main build', () => { const workflow = parse( readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8') diff --git a/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs b/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs new file mode 100644 index 00000000000..8724cf8faf4 --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs @@ -0,0 +1,614 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto' +import { execFileSync, fork } from 'node:child_process' +import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs' +import net from 'node:net' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { + median, + percentile, + processSnapshot, + sampleProcess +} from './macos-computer-helper-owner-loss-metrics.mjs' +import { + benchmarkTrialNeedsCleanup, + parseBenchmarkTrialResult, + processIdentityIsCurrent, + signalProcessIdentity, + spawnBenchmarkProcess, + throwBenchmarkTrialFailures, + writeProcessRecord +} from './macos-computer-helper-owner-loss-processes.mjs' +import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs' + +const INTERNAL_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_INTERNAL' +const EXPECTATION_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_EXPECTATION' +const HELPER_RECORD_PATH_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_HELPER_RECORD_PATH' +const RESULT_PATH_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_RESULT_PATH' +const ACTIVE_REQUEST_COUNT = 100_000 +const DEFAULT_TRIALS = 3 +const OWNER_HOLD_MS = 31_000 +const PROCESS_EXIT_TIMEOUT_MS = 5_000 +const RETAIN_PROOF_MS = 3_000 +const TRIAL_TIMEOUT_MS = OWNER_HOLD_MS + 4 * PROCESS_EXIT_TIMEOUT_MS + 120_000 +const MIB = 1024 * 1024 +const scriptPath = import.meta.filename +const repoRoot = path.resolve(import.meta.dirname, '..', '..') +const metricsPath = path.join(import.meta.dirname, 'macos-computer-helper-owner-loss-metrics.mjs') +const processCleanupPath = path.join( + import.meta.dirname, + 'macos-computer-helper-owner-loss-processes.mjs' +) +const trialCleanupPath = path.join( + import.meta.dirname, + 'macos-computer-helper-owner-loss-trial-cleanup.mjs' +) +const sidecarPath = path.join(repoRoot, 'out', 'main', 'computer-sidecar.js') +const helperAppPath = path.join( + repoRoot, + 'native', + 'computer-use-macos', + '.build', + 'release', + 'Orca Computer Use.app' +) +const helperPath = path.join(helperAppPath, 'Contents', 'MacOS', 'orca-computer-use-macos') + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function isProcessAlive(pid) { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +async function waitForProcessExit(identity, timeoutMs) { + const startedAt = performance.now() + while (performance.now() - startedAt < timeoutMs) { + if (!processIdentityIsCurrent(identity)) { + return performance.now() - startedAt + } + await sleep(50) + } + return null +} + +async function stopProcess(identity) { + if (!identity || !processIdentityIsCurrent(identity)) { + return + } + signalProcessIdentity(identity, helperPath, 'SIGTERM') + if ((await waitForProcessExit(identity, 2_000)) !== null) { + return + } + signalProcessIdentity(identity, helperPath, 'SIGKILL') + await waitForProcessExit(identity, 2_000) +} + +function startSidecar() { + const errors = [] + const child = fork(sidecarPath, [], { + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: '1', + ORCA_COMPUTER_SIDECAR: '1', + ORCA_COMPUTER_MACOS_HELPER_APP_PATH: helperAppPath + } + }) + child.on('error', (error) => errors.push(error.stack ?? error.message)) + child.stderr?.on('data', (chunk) => errors.push(String(chunk))) + return { child, errors } +} + +function requestSidecar(sidecar, id, method) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup() + reject(new Error(`Sidecar ${method} request timed out: ${sidecar.errors.join('')}`)) + }, 10_000) + const onMessage = (message) => { + if (message?.id !== id) { + return + } + cleanup() + if (message.ok) { + resolve(message.result) + } else { + reject(new Error(`Sidecar ${method} failed: ${JSON.stringify(message.error)}`)) + } + } + const onExit = (code, signal) => { + cleanup() + reject( + new Error( + `Sidecar exited during ${method}: ${JSON.stringify({ code, signal, stderr: sidecar.errors.join('') })}` + ) + ) + } + const onError = (error) => { + cleanup() + reject(new Error(`Sidecar ${method} process error: ${error.message}`)) + } + const cleanup = () => { + clearTimeout(timeout) + sidecar.child.off('message', onMessage) + sidecar.child.off('exit', onExit) + sidecar.child.off('error', onError) + } + sidecar.child.on('message', onMessage) + sidecar.child.once('exit', onExit) + sidecar.child.once('error', onError) + try { + sidecar.child.send({ id, method, params: {} }) + } catch (error) { + onError(error) + } + }) +} + +async function waitForHelper(sidecarPid) { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + const output = execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,command='], { + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }) + for (const line of output.split('\n')) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/) + if ( + match && + Number(match[2]) === sidecarPid && + Number(match[3]) === Number(match[1]) && + match[4].includes(helperPath) && + match[4].includes(' --agent ') + ) { + return { pid: Number(match[1]), pgid: Number(match[3]), command: match[4] } + } + } + await sleep(50) + } + throw new Error(`Could not find helper owned by sidecar ${sidecarPid}`) +} + +function socketPathFromCommand(command) { + const match = command.match(/ --agent (.+?) --token-file /) + if (!match) { + throw new Error(`Could not read helper socket path from command: ${command}`) + } + return match[1] +} + +function connectInvalidPeer(socketPath) { + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath) + let accepted = false + const timeout = setTimeout(() => { + socket.destroy() + reject(new Error('Invalid-peer connection timed out')) + }, 5_000) + let buffer = '' + const cleanup = () => { + clearTimeout(timeout) + socket.off('error', onError) + socket.off('data', onData) + } + const onError = (error) => { + if (accepted) { + return + } + cleanup() + reject(error) + } + const onData = (chunk) => { + buffer += chunk + const newline = buffer.indexOf('\n') + if (newline < 0) { + return + } + let response + try { + response = JSON.parse(buffer.slice(0, newline)) + } catch (error) { + cleanup() + socket.destroy() + reject(error) + return + } + if (response.ok !== false || response.error?.code !== 'permission_denied') { + cleanup() + socket.destroy() + reject(new Error(`Invalid peer was not rejected: ${JSON.stringify(response)}`)) + return + } + clearTimeout(timeout) + socket.off('data', onData) + accepted = true + resolve(socket) + } + socket.setEncoding('utf8') + socket.on('error', onError) + socket.on('data', onData) + socket.once('connect', () => { + socket.write( + `${JSON.stringify({ id: 991, method: 'handshake', params: {}, token: 'invalid' })}\n` + ) + }) + }) +} + +async function waitForChildExit(child, timeoutMs) { + if (child.exitCode !== null || child.signalCode !== null) { + return true + } + return await Promise.race([ + new Promise((resolve) => child.once('exit', () => resolve(true))), + sleep(timeoutMs).then(() => false) + ]) +} + +async function startAuthenticatedSession() { + const sidecar = startSidecar() + let helper + try { + const capabilitiesResult = requestSidecar(sidecar, 1, 'capabilities').then( + (capabilities) => ({ capabilities }), + (error) => ({ error }) + ) + helper = await waitForHelper(sidecar.child.pid) + const helperRecordPath = process.env[HELPER_RECORD_PATH_ENV] + if (!helperRecordPath) { + throw new Error('Missing helper process record path') + } + writeProcessRecord(helperRecordPath, helper) + const { capabilities, error } = await capabilitiesResult + if (error) { + throw error + } + if (capabilities?.protocolVersion !== 1) { + throw new Error(`Unexpected helper handshake: ${JSON.stringify(capabilities)}`) + } + return { authenticated: capabilities.protocolVersion === 1, sidecar, helper } + } catch (error) { + sidecar.child.kill('SIGKILL') + await stopProcess(helper) + throw error + } +} + +async function exerciseActiveRequests(sidecar) { + const latencies = [] + const startedAt = performance.now() + for (let index = 0; index < ACTIVE_REQUEST_COUNT; index += 1) { + const requestStartedAt = performance.now() + const result = await requestSidecar(sidecar, 10_000 + index, 'listApps') + if (!Array.isArray(result?.apps)) { + throw new Error(`Unexpected listApps response: ${JSON.stringify(result)}`) + } + latencies.push(performance.now() - requestStartedAt) + } + const totalMs = performance.now() - startedAt + return { + totalMs, + requestsPerSecond: (ACTIVE_REQUEST_COUNT * 1_000) / totalMs, + medianLatencyMs: median(latencies), + p95LatencyMs: percentile(latencies, 0.95), + maxLatencyMs: Math.max(...latencies) + } +} + +async function verifyGracefulClose() { + const { sidecar, helper } = await startAuthenticatedSession() + try { + const startedAt = performance.now() + sidecar.child.disconnect() + if (!(await waitForChildExit(sidecar.child, PROCESS_EXIT_TIMEOUT_MS))) { + throw new Error('Sidecar did not exit after graceful IPC close') + } + const helperExitMs = await waitForProcessExit(helper, PROCESS_EXIT_TIMEOUT_MS) + if (helperExitMs === null) { + throw new Error('Helper did not exit after graceful owner close') + } + return Math.round(performance.now() - startedAt) + } finally { + sidecar.child.kill('SIGKILL') + await stopProcess(helper) + } +} + +async function runInternalTrial(expectation) { + let sidecar + let helper + let invalidPeer + let invalidPeerRejected = false + try { + const session = await startAuthenticatedSession() + sidecar = session.sidecar + helper = session.helper + const authenticatedAt = performance.now() + const initial = await sampleProcess(helper.pid) + const activeRequests = await exerciseActiveRequests(sidecar) + const remainingHoldMs = Math.max(0, OWNER_HOLD_MS - (performance.now() - authenticatedAt)) + await sleep(remainingHoldMs) + const connected = await sampleProcess(helper.pid) + const invalidSocketPath = socketPathFromCommand(helper.command) + invalidPeer = await connectInvalidPeer(invalidSocketPath) + invalidPeerRejected = true + const survivedClaimDeadline = + performance.now() - authenticatedAt >= OWNER_HOLD_MS && isProcessAlive(helper.pid) + + sidecar.child.kill('SIGKILL') + await waitForChildExit(sidecar.child, PROCESS_EXIT_TIMEOUT_MS) + const abruptExitMs = await waitForProcessExit( + helper, + expectation === 'reaped' ? PROCESS_EXIT_TIMEOUT_MS : RETAIN_PROOF_MS + ) + const helperExitedAfterAbruptLoss = abruptExitMs !== null + if (expectation === 'reaped' && !helperExitedAfterAbruptLoss) { + throw new Error('Expected helper to exit after abrupt authenticated owner loss') + } + if (expectation === 'retained' && helperExitedAfterAbruptLoss) { + throw new Error('Expected baseline helper to remain after abrupt owner loss') + } + const postLossRssBytes = helperExitedAfterAbruptLoss ? 0 : processSnapshot(helper.pid).rssBytes + await stopProcess(helper) + helper = null + invalidPeer.destroy() + invalidPeer = null + + const gracefulExitMs = await verifyGracefulClose() + return { + authenticated: session.authenticated, + survivedClaimDeadline, + invalidPeerRejectedAndDidNotRetain: invalidPeerRejected && helperExitedAfterAbruptLoss, + connectedRssBytes: connected.rssBytes, + connectedCpuMilliseconds: Math.max( + 0, + Math.round((connected.cpuTimeSeconds - initial.cpuTimeSeconds) * 1_000) + ), + activeRequests, + cpuSampleMs: Math.round(performance.now() - authenticatedAt), + helperExitedAfterAbruptLoss, + abruptExitMs: abruptExitMs === null ? null : Math.round(abruptExitMs), + postLossRssBytes, + gracefulExitMs + } + } finally { + invalidPeer?.destroy() + sidecar?.child.kill('SIGKILL') + await stopProcess(helper) + } +} + +function parseArgs(argv) { + const options = { expect: '', trials: DEFAULT_TRIALS, output: '' } + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + const value = argv[index + 1] + if (arg === '--expect' || arg === '--trials' || arg === '--output') { + if (!value) { + throw new Error(`Missing value for ${arg}`) + } + options[arg.slice(2)] = arg === '--trials' ? Number(value) : value + index += 1 + } else { + throw new Error(`Unknown argument: ${arg}`) + } + } + if (!['retained', 'reaped'].includes(options.expect)) { + throw new Error('--expect must be retained or reaped') + } + if (!Number.isInteger(options.trials) || options.trials < 1) { + throw new Error('--trials must be a positive integer') + } + return options +} + +function electronPath() { + const electronModulePath = fileURLToPath(import.meta.resolve('electron')) + return execFileSync( + process.execPath, + ['-e', `process.stdout.write(require(${JSON.stringify(electronModulePath)}))`], + { encoding: 'utf8' } + ) +} + +function buildArtifacts() { + execFileSync('pnpm', ['exec', 'electron-vite', 'build'], { + cwd: repoRoot, + stdio: 'inherit' + }) + execFileSync('pnpm', ['build:computer-macos'], { + cwd: repoRoot, + stdio: 'inherit' + }) +} + +function artifactSha256(artifactPath) { + return createHash('sha256').update(readFileSync(artifactPath)).digest('hex') +} + +function runTrial(executable, expectation) { + let launcherDir + let trialTempDir + let helperRecordPath + let resultPath + let stderrPath + let stdoutPath + let result + let serializedResult + let parsedResult + let parsedResultAvailable = false + let trialError + let cleanupError + let stderrDescriptor + let stdoutDescriptor + let trialOutput = '' + try { + launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-helper-owner-bench-launcher-')) + trialTempDir = mkdtempSync(path.join(path.sep, 'tmp', 'orca-owner-bench-')) + helperRecordPath = path.join(launcherDir, 'helper.json') + resultPath = path.join(launcherDir, 'result.json') + stderrPath = path.join(launcherDir, 'stderr.log') + stdoutPath = path.join(launcherDir, 'stdout.log') + writeFileSync( + path.join(launcherDir, 'package.json'), + JSON.stringify({ name: 'orca-helper-owner-benchmark', main: 'main.cjs' }) + ) + writeFileSync( + path.join(launcherDir, 'main.cjs'), + `import(${JSON.stringify(pathToFileURL(scriptPath).href)}).catch((error) => { + console.error(error) + process.exitCode = 1 +})\n` + ) + const env = { + ...process.env, + TMPDIR: trialTempDir, + [INTERNAL_ENV]: '1', + [EXPECTATION_ENV]: expectation, + [HELPER_RECORD_PATH_ENV]: helperRecordPath, + [RESULT_PATH_ENV]: resultPath + } + delete env.ELECTRON_RUN_AS_NODE + stderrDescriptor = openSync(stderrPath, 'w') + stdoutDescriptor = openSync(stdoutPath, 'w') + result = spawnBenchmarkProcess(executable, [launcherDir], { + cwd: repoRoot, + env, + stdio: ['ignore', stdoutDescriptor, stderrDescriptor], + timeout: TRIAL_TIMEOUT_MS + }) + if (result.status === 0 && existsSync(resultPath)) { + serializedResult = readFileSync(resultPath, 'utf8') + parsedResult = parseBenchmarkTrialResult(serializedResult) + parsedResultAvailable = true + } + } catch (error) { + trialError = error + } finally { + const failedTrial = benchmarkTrialNeedsCleanup(result, parsedResultAvailable) + const trialMarker = trialTempDir ? `TMPDIR=${trialTempDir}` : undefined + const cleanup = cleanupOwnerLossTrial({ + failed: failedTrial, + pid: result?.pid, + marker: trialMarker, + recordPath: helperRecordPath, + helperPath, + tempDir: trialTempDir, + stderrDescriptor, + stdoutDescriptor, + outputPaths: [stderrPath, stdoutPath], + launcherDir + }) + cleanupError = cleanup.error + trialOutput = cleanup.output + } + if (!trialError && result?.status !== 0) { + trialError = new Error( + `Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n${trialOutput}` + ) + } + if (!trialError && !serializedResult) { + trialError = new Error(`Electron trial did not write a result:\n${trialOutput}`) + } + throwBenchmarkTrialFailures(trialError, cleanupError) + return parsedResult +} + +function runBenchmark() { + if (process.platform !== 'darwin') { + throw new Error('The computer-use helper owner benchmark is macOS-only') + } + const options = parseArgs(process.argv.slice(2)) + const dirty = execFileSync('git', ['status', '--porcelain'], { + cwd: repoRoot, + encoding: 'utf8' + }).trim() + if (dirty) { + throw new Error('Commit or stash changes before running the provenance-bound benchmark') + } + buildArtifacts() + if (!existsSync(sidecarPath) || !existsSync(helperPath)) { + throw new Error('Fresh production sidecar/helper build did not produce the expected artifacts') + } + const executable = electronPath() + const results = Array.from({ length: options.trials }, () => runTrial(executable, options.expect)) + const rssBytes = results.map((result) => result.connectedRssBytes) + const cpuMilliseconds = results.map((result) => result.connectedCpuMilliseconds) + const activeRequestTotals = results.map((result) => result.activeRequests.totalMs) + const activeRequestRates = results.map((result) => result.activeRequests.requestsPerSecond) + const activeRequestMedians = results.map((result) => result.activeRequests.medianLatencyMs) + const activeRequestP95s = results.map((result) => result.activeRequests.p95LatencyMs) + const activeRequestMaxes = results.map((result) => result.activeRequests.maxLatencyMs) + const postLossRssBytes = results.map((result) => result.postLossRssBytes) + const report = { + benchmark: 'macos-computer-helper-authenticated-owner-loss', + revision: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repoRoot, + encoding: 'utf8' + }).trim(), + artifacts: { + sidecarSha256: artifactSha256(sidecarPath), + helperSha256: artifactSha256(helperPath) + }, + sources: { + benchmarkSha256: artifactSha256(scriptPath), + metricsSha256: artifactSha256(metricsPath), + processCleanupSha256: artifactSha256(processCleanupPath), + trialCleanupSha256: artifactSha256(trialCleanupPath) + }, + expectation: options.expect, + trials: options.trials, + ownerHoldMs: OWNER_HOLD_MS, + activeRequestCount: ACTIVE_REQUEST_COUNT, + authenticated: results.every((result) => result.authenticated), + survivedClaimDeadline: results.every((result) => result.survivedClaimDeadline), + invalidPeerRejectedAndDidNotRetain: results.every( + (result) => result.invalidPeerRejectedAndDidNotRetain + ), + connectedRssMiB: rssBytes.map((value) => Number((value / MIB).toFixed(2))), + medianConnectedRssMiB: Number((median(rssBytes) / MIB).toFixed(2)), + connectedCpuMilliseconds: cpuMilliseconds, + medianConnectedCpuMilliseconds: median(cpuMilliseconds), + activeRequestTotalMs: activeRequestTotals.map((value) => Number(value.toFixed(3))), + medianActiveRequestTotalMs: Number(median(activeRequestTotals).toFixed(3)), + activeRequestsPerSecond: activeRequestRates.map((value) => Number(value.toFixed(2))), + medianActiveRequestsPerSecond: Number(median(activeRequestRates).toFixed(2)), + activeRequestMedianLatencyMs: activeRequestMedians.map((value) => Number(value.toFixed(3))), + medianActiveRequestMedianLatencyMs: Number(median(activeRequestMedians).toFixed(3)), + activeRequestP95LatencyMs: activeRequestP95s.map((value) => Number(value.toFixed(3))), + medianActiveRequestP95LatencyMs: Number(median(activeRequestP95s).toFixed(3)), + activeRequestMaxLatencyMs: activeRequestMaxes.map((value) => Number(value.toFixed(3))), + helperExitedAfterAbruptLoss: results.map((result) => result.helperExitedAfterAbruptLoss), + abruptExitMs: results.map((result) => result.abruptExitMs), + postLossRssMiB: postLossRssBytes.map((value) => Number((value / MIB).toFixed(2))), + medianPostLossRssMiB: Number((median(postLossRssBytes) / MIB).toFixed(2)), + gracefulExitMs: results.map((result) => result.gracefulExitMs) + } + const serialized = `${JSON.stringify(report, null, 2)}\n` + process.stdout.write(serialized) + if (options.output) { + writeFileSync(path.resolve(options.output), serialized) + } +} + +if (process.env[INTERNAL_ENV] === '1') { + const { app } = await import('electron') + await app.whenReady() + try { + const result = await runInternalTrial(process.env[EXPECTATION_ENV]) + writeFileSync(process.env[RESULT_PATH_ENV], JSON.stringify(result)) + } finally { + app.quit() + } +} else { + runBenchmark() +} diff --git a/config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs b/config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs new file mode 100644 index 00000000000..512a663cddb --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest' +import { signalValidatedProcessGroup } from './macos-computer-helper-owner-loss-processes.mjs' + +describe('macOS helper owner-loss benchmark group recovery', () => { + it('retains uncertain stop state across cleanup stage failures', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }] + const groupState = { stopped: false } + let scanCount = 0 + let continueAttempts = 0 + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount >= 3) { + throw new Error( + scanCount === 3 ? 'post-stop inspection failed' : 'final inspection failed' + ) + } + return members + }, + signalProcess: (pid, signal) => { + if (pid === -41 && signal === 'SIGCONT') { + continueAttempts += 1 + if (continueAttempts === 1) { + throw new Error('first compensation failed') + } + } + } + } + + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations) + ).toThrow('Benchmark process group signal recovery failed') + expect(groupState.stopped).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('final inspection failed') + expect(continueAttempts).toBe(2) + expect(groupState.stopped).toBe(false) + }) + + it('retains an uncertain anchor stop across cleanup stage failures', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }] + const groupState = { stopped: false, anchorPid: null } + let scanCount = 0 + let continueAttempts = 0 + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount >= 2) { + throw new Error( + scanCount === 2 ? 'post-anchor inspection failed' : 'final inspection failed' + ) + } + return members + }, + signalProcess: (pid, signal) => { + if (pid === 41 && signal === 'SIGCONT') { + continueAttempts += 1 + if (continueAttempts === 1) { + throw new Error('first anchor compensation failed') + } + } + } + } + + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations) + ).toThrow('Benchmark process group signal recovery failed') + expect(groupState.anchorPid).toBe(41) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('final inspection failed') + expect(continueAttempts).toBe(2) + expect(groupState.anchorPid).toBeNull() + }) + + it('recovers a retained anchor before selecting a new one', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const firstAnchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` } + const finalAnchor = { pid: 42, pgid: 41, command: `/child ${marker}` } + const groupState = { stopped: false, anchorPid: null } + let scanCount = 0 + let firstAnchorContinues = 0 + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount === 2) { + throw new Error('post-anchor inspection failed') + } + return scanCount === 1 ? [firstAnchor] : [finalAnchor] + }, + signalProcess: (pid, signal) => { + if (pid === 41 && signal === 'SIGCONT') { + firstAnchorContinues += 1 + if (firstAnchorContinues === 1) { + throw new Error('first anchor compensation failed') + } + } + } + } + + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations) + ).toThrow('Benchmark process group signal recovery failed') + expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(true) + expect(firstAnchorContinues).toBe(2) + expect(groupState).toEqual({ stopped: false, anchorPid: null }) + }) + + it('preserves group authority errors when compensation fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const ownershipError = 'Benchmark process group no longer belongs to this trial' + let thrown + + try { + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: true }, + { + processIdentities: () => [{ pid: 41, pgid: 41, command: '/unrelated' }], + signalProcess: () => { + throw new Error('resume denied') + } + } + ) + } catch (error) { + thrown = error + } + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual([ownershipError, 'resume denied']) + }) +}) diff --git a/config/scripts/macos-computer-helper-owner-loss-metrics.mjs b/config/scripts/macos-computer-helper-owner-loss-metrics.mjs new file mode 100644 index 00000000000..b65a062ad34 --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-metrics.mjs @@ -0,0 +1,62 @@ +import { execFileSync } from 'node:child_process' + +const SAMPLE_COUNT = 5 +const SAMPLE_INTERVAL_MS = 200 + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +export function median(values) { + const sorted = [...values].sort((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] +} + +export function percentile(values, fraction) { + const sorted = [...values].sort((left, right) => left - right) + return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] +} + +function parseCpuTimeSeconds(value) { + const [dayOrTime, clock] = value.includes('-') ? value.split('-', 2) : [null, value] + const days = dayOrTime === null ? 0 : Number(dayOrTime) + const parts = clock.split(':').map(Number) + if (!Number.isFinite(days) || parts.some((part) => !Number.isFinite(part))) { + throw new Error(`Invalid process CPU time: ${value}`) + } + const seconds = parts.pop() ?? 0 + const minutes = parts.pop() ?? 0 + const hours = parts.pop() ?? 0 + return days * 86_400 + hours * 3_600 + minutes * 60 + seconds +} + +export function processSnapshot(pid) { + const raw = execFileSync( + 'ps', + ['-o', 'rss=', '-o', 'time=', '-o', 'command=', '-p', String(pid)], + { encoding: 'utf8' } + ).trim() + const match = raw.match(/^(\d+)\s+(\S+)\s+(.+)$/) + if (!match) { + throw new Error(`Could not inspect process ${pid}: ${raw}`) + } + return { + rssBytes: Number(match[1]) * 1024, + cpuTimeSeconds: parseCpuTimeSeconds(match[2]), + command: match[3] + } +} + +export async function sampleProcess(pid) { + const samples = [] + for (let index = 0; index < SAMPLE_COUNT; index += 1) { + samples.push(processSnapshot(pid)) + await sleep(SAMPLE_INTERVAL_MS) + } + return { + rssBytes: median(samples.map((sample) => sample.rssBytes)), + cpuTimeSeconds: samples.at(-1).cpuTimeSeconds, + command: samples.at(-1).command + } +} diff --git a/config/scripts/macos-computer-helper-owner-loss-processes.mjs b/config/scripts/macos-computer-helper-owner-loss-processes.mjs new file mode 100644 index 00000000000..952d726d28d --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-processes.mjs @@ -0,0 +1,418 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs' + +const PROCESS_EXIT_TIMEOUT_MS = 2_000 +const PROCESS_POLL_MS = 25 +const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)) + +const processIdentityOperations = { + executePs: execFileSync, + signalProcess: process.kill.bind(process) +} + +export function processIdentity(pid, operations = processIdentityOperations) { + if (!Number.isInteger(pid) || pid <= 0) { + return null + } + try { + const output = operations + .executePs('ps', ['-p', String(pid), '-o', 'pid=,pgid=,command='], { + encoding: 'utf8' + }) + .trim() + const match = output.match(/^(\d+)\s+(\d+)\s+(.+)$/) + if (!match) { + throw new Error(`Could not parse process identity for ${pid}`) + } + return { pid: Number(match[1]), pgid: Number(match[2]), command: match[3] } + } catch (error) { + try { + operations.signalProcess(pid, 0) + } catch (lookupError) { + if (lookupError?.code === 'ESRCH') { + return null + } + } + throw error + } +} + +function matchingDetachedProcesses(identities, expectedCommandFragments) { + return identities.filter( + (identity) => + identity.pgid === identity.pid && + expectedCommandFragments.every((fragment) => identity.command.includes(fragment)) + ) +} + +const matchingProcessOperations = { + processIdentities, + signalProcessIdentity, + waitForIdentityExit +} + +export function killProcessMatchingCommand( + expectedCommandFragments, + operations = matchingProcessOperations +) { + const matches = matchingDetachedProcesses( + operations.processIdentities(), + expectedCommandFragments + ) + if (matches.length === 0) { + return false + } + const errors = [] + for (const match of matches) { + try { + if (operations.signalProcessIdentity(match, expectedCommandFragments[0], 'SIGKILL')) { + operations.waitForIdentityExit(match) + } + } catch (error) { + errors.push(error) + } + } + try { + const remaining = matchingDetachedProcesses( + operations.processIdentities(), + expectedCommandFragments + ) + if (remaining.length > 0) { + errors.push( + new Error( + `Benchmark helper cleanup left matching processes: ${remaining + .map((identity) => identity.pid) + .join(', ')}` + ) + ) + } + } catch (error) { + errors.push(error) + } + if (errors.length === 1) { + throw errors[0] + } + if (errors.length > 1) { + throw new AggregateError(errors, 'Benchmark exact-command cleanup failed') + } + return true +} + +function sleepSync(milliseconds) { + Atomics.wait(sleepBuffer, 0, 0, milliseconds) +} + +function validateDetachedIdentity(identity, expectedCommandFragment) { + if ( + !Number.isInteger(identity?.pid) || + identity.pid <= 0 || + identity.pgid !== identity.pid || + typeof identity.command !== 'string' || + !identity.command.includes(expectedCommandFragment) + ) { + throw new Error('Recorded benchmark helper identity is invalid') + } +} + +function sameIdentity(left, right) { + return left?.pid === right?.pid && left?.pgid === right?.pgid && left?.command === right?.command +} + +function processIdentities(includeEnvironment = false) { + const args = includeEnvironment + ? ['eww', '-axo', 'pid=,pgid=,command='] + : ['-axo', 'pid=,pgid=,command='] + const output = execFileSync('ps', args, { + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }) + return output + .split('\n') + .map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/)) + .filter(Boolean) + .map((match) => ({ + pid: Number(match[1]), + pgid: Number(match[2]), + command: match[3] + })) +} + +function waitForIdentityExit(identity) { + const deadline = Date.now() + PROCESS_EXIT_TIMEOUT_MS + while (Date.now() < deadline) { + if (!processIdentityIsCurrent(identity)) { + return true + } + sleepSync(PROCESS_POLL_MS) + } + throw new Error(`Recorded benchmark helper ${identity.pid} did not exit`) +} + +export function spawnBenchmarkProcess(executable, args, options) { + return spawnSync(executable, args, { + ...options, + detached: true, + killSignal: 'SIGKILL' + }) +} + +export function runBenchmarkCleanupStages(stages) { + const errors = [] + for (const stage of stages) { + try { + stage() + } catch (error) { + errors.push(error) + } + } + if (errors.length === 1) { + throw errors[0] + } + if (errors.length > 1) { + throw new AggregateError(errors, 'Benchmark trial cleanup failed') + } +} + +export function throwBenchmarkTrialFailures(trialError, cleanupError) { + if (trialError && cleanupError) { + throw new AggregateError([trialError, cleanupError], 'Electron trial and cleanup failed') + } + if (trialError) { + throw trialError + } + if (cleanupError) { + throw cleanupError + } +} + +export function parseBenchmarkTrialResult(serializedResult) { + return JSON.parse(serializedResult) +} + +export function benchmarkTrialNeedsCleanup(spawnResult, parsedResultAvailable) { + return spawnResult?.status !== 0 || !parsedResultAvailable +} + +const processGroupSignalOperations = { + processIdentities, + signalProcess: process.kill.bind(process) +} + +function compensateStoppedGroup(pgid, groupState, operations) { + const errors = [] + const targets = [ + groupState.stopped ? [-pgid, 'stopped'] : null, + groupState.anchorPid ? [groupState.anchorPid, 'anchorPid'] : null + ].filter(Boolean) + for (const [pid, stateKey] of targets) { + try { + operations.signalProcess(pid, 'SIGCONT') + groupState[stateKey] = stateKey === 'stopped' ? false : null + } catch (error) { + if (error?.code === 'ESRCH') { + groupState[stateKey] = stateKey === 'stopped' ? false : null + } else { + errors.push(error) + } + } + } + return errors +} + +export function signalValidatedProcessGroup( + pgid, + environmentFragment, + signal, + groupState = { stopped: false, anchorPid: null }, + operations = processGroupSignalOperations +) { + if (!Number.isInteger(pgid) || pgid <= 0) { + return false + } + let members + try { + members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid) + } catch (error) { + const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations) + if (recoveryErrors.length > 0) { + throw new AggregateError( + [error, ...recoveryErrors], + 'Benchmark process group recovery failed before validation' + ) + } + throw error + } + if (members.length === 0) { + const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations) + if (recoveryErrors.length > 0) { + throw new AggregateError(recoveryErrors, 'Benchmark missing process group recovery failed') + } + return false + } + if (members.some((identity) => !identity.command.includes(environmentFragment))) { + const ownershipError = new Error('Benchmark process group no longer belongs to this trial') + const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations) + if (recoveryErrors.length > 0) { + throw new AggregateError( + [ownershipError, ...recoveryErrors], + 'Benchmark process group authority recovery failed' + ) + } + throw ownershipError + } + if (groupState.anchorPid) { + try { + operations.signalProcess(groupState.anchorPid, 'SIGCONT') + groupState.anchorPid = null + } catch (error) { + if (error?.code === 'ESRCH') { + groupState.anchorPid = null + } else { + throw new AggregateError([error], 'Benchmark pending anchor recovery failed') + } + } + } + const anchor = members[0] + try { + operations.signalProcess(anchor.pid, 'SIGSTOP') + groupState.anchorPid = anchor.pid + const stoppedAnchor = operations + .processIdentities(true) + .find((identity) => identity.pid === anchor.pid) + if (!sameIdentity(stoppedAnchor, anchor)) { + throw new Error('Benchmark process group anchor changed before signaling') + } + operations.signalProcess(-pgid, 'SIGSTOP') + groupState.stopped = true + groupState.anchorPid = null + const stoppedMembers = operations + .processIdentities(true) + .filter((identity) => identity.pgid === pgid) + if ( + stoppedMembers.length === 0 || + stoppedMembers.some((identity) => !identity.command.includes(environmentFragment)) + ) { + throw new Error('Benchmark process group changed before signaling') + } + if (signal !== 'SIGSTOP') { + operations.signalProcess(-pgid, signal) + if (signal !== 'SIGKILL') { + operations.signalProcess(-pgid, 'SIGCONT') + } + groupState.stopped = false + groupState.anchorPid = null + } + return true + } catch (error) { + const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations) + if (recoveryErrors.length > 0) { + throw new AggregateError( + [error, ...recoveryErrors], + 'Benchmark process group signal recovery failed' + ) + } + if (error.code === 'ESRCH') { + return false + } + throw error + } +} + +export function writeProcessRecord(recordPath, processIdentity) { + const temporaryPath = `${recordPath}.${process.pid}.tmp` + writeFileSync(temporaryPath, JSON.stringify(processIdentity)) + renameSync(temporaryPath, recordPath) +} + +export function processIdentityIsCurrent(identity) { + return sameIdentity(processIdentity(identity?.pid), identity) +} + +const processSignalOperations = { + processIdentity, + signalProcess: process.kill.bind(process) +} + +export function signalProcessIdentity( + identity, + expectedCommandFragment, + signal, + operations = processSignalOperations +) { + validateDetachedIdentity(identity, expectedCommandFragment) + const currentIdentity = operations.processIdentity(identity.pid) + if (!currentIdentity) { + return false + } + if (!sameIdentity(currentIdentity, identity)) { + throw new Error('Recorded benchmark helper PID now belongs to another process') + } + let stopped = false + try { + operations.signalProcess(identity.pid, 'SIGSTOP') + stopped = true + const stoppedIdentity = operations.processIdentity(identity.pid) + if (!sameIdentity(stoppedIdentity, identity)) { + throw new Error('Recorded benchmark helper PID changed before signaling') + } + operations.signalProcess(-identity.pgid, signal) + if (signal !== 'SIGKILL') { + operations.signalProcess(-identity.pgid, 'SIGCONT') + } + stopped = false + return true + } catch (error) { + let resumeError + if (stopped) { + try { + operations.signalProcess(identity.pid, 'SIGCONT') + } catch (caught) { + if (caught?.code !== 'ESRCH') { + resumeError = caught + } + } + } + if (resumeError) { + throw new AggregateError([error, resumeError], 'Benchmark helper signal recovery failed') + } + if (error.code === 'ESRCH') { + return false + } + throw error + } +} + +export function killRecordedProcess(recordPath, expectedCommandFragment) { + if (!existsSync(recordPath)) { + return false + } + const record = JSON.parse(readFileSync(recordPath, 'utf8')) + if (!signalProcessIdentity(record, expectedCommandFragment, 'SIGKILL')) { + return false + } + return waitForIdentityExit(record) +} + +export function killRecordedAndMatchingProcesses( + recordPath, + recordedCommandFragment, + matchingCommandFragments +) { + const errors = [] + try { + killRecordedProcess(recordPath, recordedCommandFragment) + } catch (error) { + errors.push(error) + } + try { + killProcessMatchingCommand(matchingCommandFragments) + } catch (error) { + errors.push(error) + } + if (errors.length === 1) { + throw errors[0] + } + if (errors.length > 1) { + throw new AggregateError(errors, 'Benchmark helper cleanup failed') + } +} diff --git a/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs b/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs new file mode 100644 index 00000000000..764eb73009f --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs @@ -0,0 +1,558 @@ +import { execFileSync, spawn } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + benchmarkTrialNeedsCleanup, + killProcessMatchingCommand, + killRecordedAndMatchingProcesses, + killRecordedProcess, + parseBenchmarkTrialResult, + processIdentity, + runBenchmarkCleanupStages, + signalProcessIdentity, + signalValidatedProcessGroup, + spawnBenchmarkProcess, + throwBenchmarkTrialFailures, + writeProcessRecord +} from './macos-computer-helper-owner-loss-processes.mjs' +import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs' + +const describeMacOS = process.platform === 'darwin' ? describe : describe.skip +const spawnedPids = new Set() +const temporaryDirectories = new Set() + +afterEach(() => { + for (const pid of spawnedPids) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } + spawnedPids.clear() + for (const temporaryDirectory of temporaryDirectories) { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + temporaryDirectories.clear() +}) + +describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { + it('enforces a hard timeout when the trial ignores SIGTERM', () => { + const startedAt = Date.now() + const result = spawnBenchmarkProcess( + process.execPath, + ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1_000)"], + { stdio: 'ignore', timeout: 100 } + ) + expect(result.error?.code).toBe('ETIMEDOUT') + expect(result.signal).toBe('SIGKILL') + expect(Date.now() - startedAt).toBeLessThan(2_000) + expect(() => process.kill(result.pid, 0)).toThrow() + }) + + it('runs every cleanup stage before aggregating errors', () => { + const completed = [] + let thrown + + try { + runBenchmarkCleanupStages([ + () => { + completed.push(1) + throw new Error('first failure') + }, + () => { + completed.push(2) + }, + () => { + completed.push(3) + throw new Error('last failure') + } + ]) + } catch (error) { + thrown = error + } + expect(completed).toEqual([1, 2, 3]) + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual(['first failure', 'last failure']) + }) + + it('preserves malformed-result and cleanup failures', () => { + let trialError + try { + parseBenchmarkTrialResult('{malformed') + } catch (error) { + trialError = error + } + const cleanupError = new Error('cleanup failed') + let thrown + + try { + throwBenchmarkTrialFailures(trialError, cleanupError) + } catch (error) { + thrown = error + } + expect(trialError).toBeInstanceOf(SyntaxError) + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors).toEqual([trialError, cleanupError]) + }) + + it('cleans up a status-zero trial whose result could not be parsed', () => { + expect(benchmarkTrialNeedsCleanup(undefined, false)).toBe(true) + expect(benchmarkTrialNeedsCleanup({ status: 0 }, false)).toBe(true) + expect(benchmarkTrialNeedsCleanup({ status: 0 }, true)).toBe(false) + expect(benchmarkTrialNeedsCleanup({ status: 1 }, true)).toBe(true) + }) + + it('removes a launcher directory after partial trial setup', () => { + const launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-owner-partial-setup-test-')) + temporaryDirectories.add(launcherDir) + + const cleanup = cleanupOwnerLossTrial({ + failed: true, + launcherDir, + outputPaths: [] + }) + + expect(cleanup.error).toBeUndefined() + expect(existsSync(launcherDir)).toBe(false) + temporaryDirectories.delete(launcherDir) + }) + + it('kills a timed-out trial group only after validating its environment', async () => { + const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'orca-owner-benchmark-group-test-')) + temporaryDirectories.add(temporaryDirectory) + const childPidPath = path.join(temporaryDirectory, 'child.pid') + const environmentName = `ORCA_OWNER_GROUP_${process.pid}` + const environmentValue = `${Date.now()}` + const fixture = ` + const { spawn } = require('node:child_process') + const { writeFileSync } = require('node:fs') + const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { + stdio: 'ignore' + }) + writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid)) + setInterval(() => {}, 1000) + ` + const result = spawnBenchmarkProcess(process.execPath, ['-e', fixture], { + env: { ...process.env, [environmentName]: environmentValue }, + stdio: 'ignore', + timeout: 100 + }) + const childPid = Number(readFileSync(childPidPath, 'utf8')) + spawnedPids.add(childPid) + const environmentFragment = `${environmentName}=${environmentValue}` + const groupState = { stopped: false } + + expect(() => + signalValidatedProcessGroup(result.pid, `${environmentName}=wrong`, 'SIGSTOP') + ).toThrow('Benchmark process group no longer belongs to this trial') + expect(() => process.kill(childPid, 0)).not.toThrow() + expect( + signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP', groupState) + ).toBe(true) + expect( + signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL', groupState) + ).toBe(true) + await expect + .poll(() => { + try { + process.kill(childPid, 0) + return true + } catch { + return false + } + }) + .toBe(false) + + spawnedPids.delete(childPid) + }) + + it('resumes the group after post-stop revalidation fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [ + { pid: 41, pgid: 41, command: `/launcher ${marker}` }, + { pid: 42, pgid: 41, command: `/child ${marker}` } + ] + const signals = [] + let scanCount = 0 + + expect(() => + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: false }, + { + processIdentities: () => { + scanCount += 1 + if (scanCount === 3) { + throw new Error('transient group inspection failure') + } + return members + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + ) + ).toThrow('transient group inspection failure') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [-41, 'SIGSTOP'], + [-41, 'SIGCONT'] + ]) + }) + + it('compensates a possible stop after group anchor replacement', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` } + const replacement = { pid: 41, pgid: 99, command: '/unrelated' } + const signals = [] + let scanCount = 0 + + expect(() => + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: false }, + { + processIdentities: () => { + scanCount += 1 + return scanCount === 1 ? [anchor] : [replacement] + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + ) + ).toThrow('Benchmark process group anchor changed before signaling') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [41, 'SIGCONT'] + ]) + }) + + it('resumes a previously frozen group when final inspection fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }] + const signals = [] + let scanCount = 0 + const groupState = { stopped: false } + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount === 4) { + throw new Error('transient final inspection failure') + } + return members + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('transient final inspection failure') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [-41, 'SIGSTOP'], + [-41, 'SIGCONT'] + ]) + }) + + it('resumes a previously frozen group when final anchor stop fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [ + { pid: 41, pgid: 41, command: `/launcher ${marker}` }, + { pid: 42, pgid: 41, command: `/child ${marker}` } + ] + const signals = [] + let finalCall = false + const groupState = { stopped: false } + const missingProcessError = Object.assign(new Error('anchor exited'), { code: 'ESRCH' }) + const operations = { + processIdentities: () => members, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + if (finalCall && pid === 41 && signal === 'SIGSTOP') { + throw missingProcessError + } + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + finalCall = true + expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(false) + expect(signals.at(-1)).toEqual([-41, 'SIGCONT']) + }) + + it('resumes a previously frozen group after final anchor replacement', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` } + const child = { pid: 42, pgid: 41, command: `/child ${marker}` } + const replacement = { pid: 41, pgid: 99, command: '/unrelated' } + const signals = [] + let scanCount = 0 + const groupState = { stopped: false } + const operations = { + processIdentities: () => { + scanCount += 1 + return scanCount === 5 ? [replacement, child] : [anchor, child] + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('Benchmark process group anchor changed before signaling') + expect(signals.slice(-2)).toEqual([ + [-41, 'SIGCONT'], + [41, 'SIGCONT'] + ]) + }) + + it('kills a recorded helper in a separate process group', async () => { + const temporaryDirectory = mkdtempSync( + path.join(tmpdir(), 'orca-owner-benchmark-cleanup-test-') + ) + temporaryDirectories.add(temporaryDirectory) + const recordPath = path.join(temporaryDirectory, 'helper.json') + const marker = `orca-owner-cleanup-${process.pid}-${Date.now()}` + const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], { + detached: true, + stdio: 'ignore' + }) + spawnedPids.add(helper.pid) + helper.unref() + const exited = new Promise((resolve) => helper.once('exit', resolve)) + const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], { + encoding: 'utf8' + }).trim() + const processGroup = Number( + execFileSync('ps', ['-p', String(helper.pid), '-o', 'pgid='], { + encoding: 'utf8' + }).trim() + ) + writeProcessRecord(recordPath, { pid: helper.pid, pgid: processGroup, command }) + + expect(processGroup).toBe(helper.pid) + expect(killRecordedProcess(recordPath, marker)).toBe(true) + await exited + expect(() => process.kill(helper.pid, 0)).toThrow() + + spawnedPids.delete(helper.pid) + }) + + it('kills every unrecorded helper using its unique trial command', async () => { + const marker = `orca-owner-unrecorded-${process.pid}-${Date.now()}` + const helpers = Array.from({ length: 2 }, () => + spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], { + detached: true, + stdio: 'ignore' + }) + ) + for (const helper of helpers) { + spawnedPids.add(helper.pid) + helper.unref() + } + const exited = Promise.all( + helpers.map((helper) => new Promise((resolve) => helper.once('exit', resolve))) + ) + + expect(killProcessMatchingCommand([process.execPath, marker])).toBe(true) + await exited + for (const helper of helpers) { + expect(() => process.kill(helper.pid, 0)).toThrow() + spawnedPids.delete(helper.pid) + } + }) + + it('continues exact-match cleanup after an earlier match fails', () => { + const marker = `orca-owner-multiple-${process.pid}-${Date.now()}` + const matches = [ + { pid: 41, pgid: 41, command: `/helper ${marker}` }, + { pid: 42, pgid: 42, command: `/helper ${marker}` } + ] + const attempted = [] + let scanCount = 0 + + expect(() => + killProcessMatchingCommand(['/helper', marker], { + processIdentities: () => { + scanCount += 1 + return scanCount === 1 ? matches : [matches[0]] + }, + signalProcessIdentity: (identity) => { + attempted.push(identity.pid) + if (identity.pid === matches[0].pid) { + throw new Error('identity changed') + } + return true + }, + waitForIdentityExit: () => {} + }) + ).toThrow('Benchmark exact-command cleanup failed') + expect(attempted).toEqual([41, 42]) + }) + + it('does not treat an identity query failure as process exit', () => { + const queryError = new Error('transient ps failure') + + expect(() => + processIdentity(41, { + executePs: () => { + throw queryError + }, + signalProcess: () => {} + }) + ).toThrow(queryError) + }) + + it('resumes a helper when post-stop identity inspection fails', () => { + const identity = { pid: 41, pgid: 41, command: '/helper marker' } + const signals = [] + let inspectionCount = 0 + + expect(() => + signalProcessIdentity(identity, 'marker', 'SIGKILL', { + processIdentity: () => { + inspectionCount += 1 + if (inspectionCount === 2) { + throw new Error('transient ps failure after stop') + } + return identity + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + }) + ).toThrow('transient ps failure after stop') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [41, 'SIGCONT'] + ]) + }) + + it('compensates a possible stop after helper PID replacement', () => { + const identity = { pid: 41, pgid: 41, command: '/helper marker' } + const replacement = { pid: 41, pgid: 41, command: '/unrelated' } + const signals = [] + let inspectionCount = 0 + + expect(() => + signalProcessIdentity(identity, 'marker', 'SIGKILL', { + processIdentity: () => { + inspectionCount += 1 + return inspectionCount === 1 ? identity : replacement + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + }) + ).toThrow('Recorded benchmark helper PID changed before signaling') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [41, 'SIGCONT'] + ]) + }) + + it('preserves helper identity errors when compensation fails', () => { + const identity = { pid: 41, pgid: 41, command: '/helper marker' } + const replacement = { pid: 41, pgid: 41, command: '/unrelated' } + let inspectionCount = 0 + let thrown + + try { + signalProcessIdentity(identity, 'marker', 'SIGKILL', { + processIdentity: () => { + inspectionCount += 1 + return inspectionCount === 1 ? identity : replacement + }, + signalProcess: (_pid, signal) => { + if (signal === 'SIGCONT') { + throw new Error('resume denied') + } + } + }) + } catch (error) { + thrown = error + } + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual([ + 'Recorded benchmark helper PID changed before signaling', + 'resume denied' + ]) + }) + + it('treats a missing PID as process exit after an identity query failure', () => { + const missingProcessError = Object.assign(new Error('missing process'), { code: 'ESRCH' }) + + expect( + processIdentity(41, { + executePs: () => { + throw new Error('ps found no process') + }, + signalProcess: () => { + throw missingProcessError + } + }) + ).toBeNull() + }) + + it('runs unique-command cleanup after an invalid process record', async () => { + const temporaryDirectory = mkdtempSync( + path.join(tmpdir(), 'orca-owner-benchmark-fallback-test-') + ) + temporaryDirectories.add(temporaryDirectory) + const recordPath = path.join(temporaryDirectory, 'helper.json') + const marker = `orca-owner-invalid-record-${process.pid}-${Date.now()}` + const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], { + detached: true, + stdio: 'ignore' + }) + spawnedPids.add(helper.pid) + helper.unref() + const exited = new Promise((resolve) => helper.once('exit', resolve)) + const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], { + encoding: 'utf8' + }).trim() + writeProcessRecord(recordPath, { pid: helper.pid, pgid: helper.pid - 1, command }) + + expect(() => + killRecordedAndMatchingProcesses(recordPath, marker, [process.execPath, marker]) + ).toThrow('Recorded benchmark helper identity is invalid') + await exited + expect(() => process.kill(helper.pid, 0)).toThrow() + + spawnedPids.delete(helper.pid) + }) + + it('rejects a record that is not a detached process-group identity', () => { + const temporaryDirectory = mkdtempSync( + path.join(tmpdir(), 'orca-owner-benchmark-identity-test-') + ) + temporaryDirectories.add(temporaryDirectory) + const recordPath = path.join(temporaryDirectory, 'helper.json') + const marker = `orca-owner-invalid-identity-${process.pid}-${Date.now()}` + const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], { + stdio: 'ignore' + }) + spawnedPids.add(helper.pid) + helper.unref() + const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], { + encoding: 'utf8' + }).trim() + writeProcessRecord(recordPath, { pid: helper.pid, pgid: helper.pid - 1, command }) + + expect(() => killRecordedProcess(recordPath, marker)).toThrow( + 'Recorded benchmark helper identity is invalid' + ) + expect(() => process.kill(helper.pid, 0)).not.toThrow() + }) +}) diff --git a/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs b/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs new file mode 100644 index 00000000000..c3fb4aaf384 --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs @@ -0,0 +1,63 @@ +import { closeSync, existsSync, readFileSync, rmSync } from 'node:fs' +import { + killRecordedAndMatchingProcesses, + runBenchmarkCleanupStages, + signalValidatedProcessGroup +} from './macos-computer-helper-owner-loss-processes.mjs' + +export function cleanupOwnerLossTrial(options) { + const groupState = { stopped: false, anchorPid: null } + let error + let output = '' + try { + runBenchmarkCleanupStages([ + () => { + if (options.failed && Number.isInteger(options.pid) && options.marker) { + signalValidatedProcessGroup(options.pid, options.marker, 'SIGSTOP', groupState) + } + }, + () => { + if (options.failed && options.recordPath && options.tempDir) { + killRecordedAndMatchingProcesses(options.recordPath, options.helperPath, [ + options.helperPath, + options.tempDir + ]) + } + }, + () => { + if (options.failed && Number.isInteger(options.pid) && options.marker) { + signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL', groupState) + } + }, + () => { + if (options.stderrDescriptor !== undefined) { + closeSync(options.stderrDescriptor) + } + }, + () => { + if (options.stdoutDescriptor !== undefined) { + closeSync(options.stdoutDescriptor) + } + }, + () => { + output = (options.outputPaths ?? []) + .filter((outputPath) => outputPath && existsSync(outputPath)) + .map((outputPath) => readFileSync(outputPath, 'utf8')) + .join('') + }, + () => { + if (options.launcherDir) { + rmSync(options.launcherDir, { recursive: true, force: true }) + } + }, + () => { + if (options.tempDir) { + rmSync(options.tempDir, { recursive: true, force: true }) + } + } + ]) + } catch (caught) { + error = caught + } + return { error, output } +} diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 8edeb06131b..50ecae7d024 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -2579,9 +2579,12 @@ private enum KeyMap { } private final class AgentRuntime: NSObject, NSApplicationDelegate { + private static let unclaimedSessionDeadline: TimeInterval = 30 + private let socketPath: String private let token: String? private var listener: SocketListener? + private var unclaimedSessionTimeout: DispatchWorkItem? init(socketPath: String, token: String?) { self.socketPath = socketPath @@ -2590,9 +2593,29 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate { func applicationDidFinishLaunching(_ notification: Notification) { do { - let listener = try SocketListener(socketPath: socketPath, token: token) + let timeout = DispatchWorkItem { + fputs("computer-use agent received no authenticated session before its deadline\n", stderr) + NSApp.terminate(nil) + } + unclaimedSessionTimeout = timeout + let listener = try SocketListener( + socketPath: socketPath, + token: token, + onSessionClaimed: { + timeout.cancel() + }, + onSessionClosed: { + DispatchQueue.main.async { + NSApp.terminate(nil) + } + } + ) self.listener = listener listener.start() + DispatchQueue.main.asyncAfter( + deadline: .now() + Self.unclaimedSessionDeadline, + execute: timeout + ) } catch { fputs("failed to start computer-use socket: \(error)\n", stderr) NSApp.terminate(nil) @@ -2600,6 +2623,8 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate { } func applicationWillTerminate(_ notification: Notification) { + unclaimedSessionTimeout?.cancel() + unclaimedSessionTimeout = nil listener?.stop() } } @@ -3505,14 +3530,26 @@ private final class ButtonTarget: NSObject { private final class SocketListener: @unchecked Sendable { private let socketPath: String private let token: String? + private let onSessionClaimed: () -> Void + private let onSessionClosed: () -> Void private let provider = Provider() private let providerLock = NSLock() + private let sessionLock = NSLock() + private var sessionOwnership = AgentSessionOwnership() + private var lastConnectionID: UInt64 = 0 private var socketFd: Int32 = -1 private var isStopped = false - init(socketPath: String, token: String?) throws { + init( + socketPath: String, + token: String?, + onSessionClaimed: @escaping () -> Void, + onSessionClosed: @escaping () -> Void + ) throws { self.socketPath = socketPath self.token = token + self.onSessionClaimed = onSessionClaimed + self.onSessionClosed = onSessionClosed try bindSocket() } @@ -3587,14 +3624,35 @@ private final class SocketListener: @unchecked Sendable { } continue } + guard let connectionID = allocateConnectionID() else { + fputs("computer-use socket exhausted connection identities\n", stderr) + close(fd) + continue + } Thread.detachNewThread { [weak self] in - self?.handleConnection(fd) + self?.handleConnection(fd, connectionID: connectionID) } } } - private func handleConnection(_ fd: Int32) { - defer { close(fd) } + private func allocateConnectionID() -> AgentSessionConnectionID? { + sessionLock.lock() + defer { sessionLock.unlock() } + guard lastConnectionID < UInt64.max else { return nil } + lastConnectionID += 1 + return AgentSessionConnectionID(rawValue: lastConnectionID) + } + + private func handleConnection(_ fd: Int32, connectionID: AgentSessionConnectionID) { + var registeredSession = false + var hangupMonitor: AuthenticatedConnectionHangupMonitor? + defer { + hangupMonitor?.cancel() + if registeredSession { + disconnectSession(connectionID) + } + close(fd) + } let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true let decoder = JSONDecoder() while let line = readLine(from: fd) { @@ -3603,6 +3661,40 @@ private final class SocketListener: @unchecked Sendable { else { continue } + if !registeredSession && isAuthenticatedAgentSession( + expectedToken: token, + requestToken: request.token, + authorizedPeer: authorizedPeer + ) { + let monitor: AuthenticatedConnectionHangupMonitor + do { + monitor = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: fd, + onHangup: { [weak self] in + self?.disconnectSession(connectionID) + } + ) + } catch { + fputs("computer-use owner monitor failed: \(error)\n", stderr) + return + } + sessionLock.lock() + let registration = sessionOwnership.registerConnection( + connectionID, + authenticated: true + ) + sessionLock.unlock() + guard registration != .rejected else { + monitor.cancel() + return + } + registeredSession = true + hangupMonitor = monitor + monitor.start() + if registration == .claimed { + onSessionClaimed() + } + } let response = handleRequest( provider: provider, lock: providerLock, @@ -3613,6 +3705,15 @@ private final class SocketListener: @unchecked Sendable { writeJSON(response, to: fd) } } + + private func disconnectSession(_ connectionID: AgentSessionConnectionID) { + sessionLock.lock() + let shouldTerminate = sessionOwnership.disconnect(connectionID) + sessionLock.unlock() + if shouldTerminate { + onSessionClosed() + } + } } private func existingPathMode(_ path: String) -> mode_t? { diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AgentSessionOwnership.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AgentSessionOwnership.swift new file mode 100644 index 00000000000..d39543e9895 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AgentSessionOwnership.swift @@ -0,0 +1,50 @@ +public struct AgentSessionConnectionID: Hashable, Sendable { + public let rawValue: UInt64 + + public init(rawValue: UInt64) { + self.rawValue = rawValue + } +} + +public enum AgentSessionRegistration: Sendable { + case rejected + case claimed + case retained +} + +public struct AgentSessionOwnership: Sendable { + private var authenticatedConnections: Set = [] + private var wasClaimed = false + private var sessionClosed = false + + public init() {} + + public mutating func registerConnection( + _ connection: AgentSessionConnectionID, + authenticated: Bool + ) -> AgentSessionRegistration { + guard authenticated, !sessionClosed else { return .rejected } + let inserted = authenticatedConnections.insert(connection).inserted + guard inserted else { return .rejected } + guard !wasClaimed else { return .retained } + wasClaimed = true + return .claimed + } + + public mutating func disconnect(_ connection: AgentSessionConnectionID) -> Bool { + guard authenticatedConnections.remove(connection) != nil else { return false } + guard wasClaimed, authenticatedConnections.isEmpty else { return false } + sessionClosed = true + return true + } +} + +public func isAuthenticatedAgentSession( + expectedToken: String?, + requestToken: String?, + authorizedPeer: Bool +) -> Bool { + guard authorizedPeer else { return false } + guard let expectedToken else { return true } + return requestToken == expectedToken +} diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AuthenticatedConnectionHangupMonitor.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AuthenticatedConnectionHangupMonitor.swift new file mode 100644 index 00000000000..55e4ec91721 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/AuthenticatedConnectionHangupMonitor.swift @@ -0,0 +1,182 @@ +import Darwin +import Dispatch +import Foundation + +public final class AuthenticatedConnectionHangupMonitor: @unchecked Sendable { + typealias RegisterEvents = @Sendable ( + Int32, + UnsafePointer?, + Int32 + ) -> POSIXErrorCode? + typealias CloseDescriptor = @Sendable (Int32) -> Void + + private static let cancelEventIdentifier: UInt = 1 + + private let closeDescriptor: CloseDescriptor + private let eventQueue: DispatchQueue + private let eventQueueDescriptor: Int32 + private let onEvent: @Sendable () -> Void + private let stateLock = NSLock() + private let onHangup: @Sendable () -> Void + private var isCancelled = false + private var didReportHangup = false + private var isFinished = false + private var isStarted = false + + public convenience init( + fileDescriptor: Int32, + queue: DispatchQueue = DispatchQueue( + label: "com.stablyai.orca.computer-use-owner-hangup" + ), + onEvent: @escaping @Sendable () -> Void = {}, + onHangup: @escaping @Sendable () -> Void + ) throws { + try self.init( + fileDescriptor: fileDescriptor, + queue: queue, + registerEvents: { descriptor, events, count in + guard kevent(descriptor, events, count, nil, 0, nil) == 0 else { + return POSIXErrorCode(rawValue: errno) ?? .EIO + } + return nil + }, + closeDescriptor: { descriptor in + close(descriptor) + }, + onEvent: onEvent, + onHangup: onHangup + ) + } + + init( + fileDescriptor: Int32, + queue: DispatchQueue = DispatchQueue( + label: "com.stablyai.orca.computer-use-owner-hangup" + ), + registerEvents: RegisterEvents, + closeDescriptor: @escaping CloseDescriptor, + onEvent: @escaping @Sendable () -> Void = {}, + onHangup: @escaping @Sendable () -> Void + ) throws { + self.closeDescriptor = closeDescriptor + eventQueue = queue + self.onEvent = onEvent + self.onHangup = onHangup + let descriptor = kqueue() + guard descriptor >= 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard fileDescriptor >= 0 else { + closeDescriptor(descriptor) + throw POSIXError(.EBADF) + } + eventQueueDescriptor = descriptor + var registrations = [ + kevent( + ident: UInt(fileDescriptor), + filter: Int16(EVFILT_READ), + flags: UInt16(EV_ADD | EV_CLEAR), + fflags: UInt32(NOTE_LOWAT), + data: Int.max, + udata: nil + ), + kevent( + ident: Self.cancelEventIdentifier, + filter: Int16(EVFILT_USER), + flags: UInt16(EV_ADD | EV_CLEAR), + fflags: 0, + data: 0, + udata: nil + ) + ] + let registrationError = registrations.withUnsafeMutableBufferPointer { buffer in + registerEvents(descriptor, buffer.baseAddress, Int32(buffer.count)) + } + if let registrationError { + isFinished = true + closeDescriptor(descriptor) + throw POSIXError(registrationError) + } + } + + deinit { + cancel() + } + + public func start() { + stateLock.lock() + guard !isStarted, !isCancelled, !isFinished else { + stateLock.unlock() + return + } + isStarted = true + stateLock.unlock() + eventQueue.async { [self] in + waitForHangup() + } + } + + public func cancel() { + stateLock.lock() + guard !isFinished else { + stateLock.unlock() + return + } + isCancelled = true + guard isStarted else { + isFinished = true + closeDescriptor(eventQueueDescriptor) + stateLock.unlock() + return + } + var event = kevent( + ident: Self.cancelEventIdentifier, + filter: Int16(EVFILT_USER), + flags: 0, + fflags: UInt32(NOTE_TRIGGER), + data: 0, + udata: nil + ) + _ = kevent(eventQueueDescriptor, &event, 1, nil, 0, nil) + stateLock.unlock() + } + + private func waitForHangup() { + defer { finish() } + while true { + var event = kevent() + let result = kevent(eventQueueDescriptor, nil, 0, &event, 1, nil) + if result < 0 && errno == EINTR { + continue + } + guard result > 0 else { return } + onEvent() + if event.filter == Int16(EVFILT_USER) { + return + } + if event.flags & UInt16(EV_EOF | EV_ERROR) != 0 { + reportHangup() + return + } + } + } + + private func finish() { + stateLock.lock() + isFinished = true + closeDescriptor(eventQueueDescriptor) + stateLock.unlock() + } + + private func reportHangup() { + stateLock.lock() + guard !isCancelled, !didReportHangup else { + stateLock.unlock() + return + } + didReportHangup = true + stateLock.unlock() + + onHangup() + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AgentSessionOwnershipTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AgentSessionOwnershipTests.swift new file mode 100644 index 00000000000..8094501c919 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AgentSessionOwnershipTests.swift @@ -0,0 +1,104 @@ +import OrcaComputerUseMacOSCore +import XCTest + +final class AgentSessionOwnershipTests: XCTestCase { + func testUnclaimedDisconnectDoesNotTerminateAgent() { + var ownership = AgentSessionOwnership() + + XCTAssertFalse(ownership.disconnect(connection(12))) + } + + func testUnauthenticatedConnectionCannotClaimOrRetainAgent() { + var ownership = AgentSessionOwnership() + + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: false), .rejected) + XCTAssertFalse(ownership.disconnect(connection(12))) + } + + func testLastAuthenticatedDisconnectTerminatesAgent() { + var ownership = AgentSessionOwnership() + + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed) + XCTAssertTrue(ownership.disconnect(connection(12))) + } + + func testAgentWaitsForEveryAuthenticatedConnectionToClose() { + var ownership = AgentSessionOwnership() + + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed) + XCTAssertEqual(ownership.registerConnection(connection(13), authenticated: true), .retained) + XCTAssertFalse(ownership.disconnect(connection(12))) + XCTAssertTrue(ownership.disconnect(connection(13))) + } + + func testDuplicateRegistrationDoesNotRetainAgent() { + var ownership = AgentSessionOwnership() + + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed) + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .rejected) + XCTAssertTrue(ownership.disconnect(connection(12))) + } + + func testClosedSessionRejectsNewConnections() { + var ownership = AgentSessionOwnership() + + XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed) + XCTAssertTrue(ownership.disconnect(connection(12))) + XCTAssertEqual(ownership.registerConnection(connection(13), authenticated: true), .rejected) + XCTAssertFalse(ownership.disconnect(connection(13))) + } + + func testStaleDisconnectCannotRemoveReusedFileDescriptorOwner() { + var ownership = AgentSessionOwnership() + let staleConnection = connection(12) + let otherConnection = connection(13) + let reusedDescriptorConnection = connection(14) + + XCTAssertEqual(ownership.registerConnection(staleConnection, authenticated: true), .claimed) + XCTAssertEqual(ownership.registerConnection(otherConnection, authenticated: true), .retained) + XCTAssertFalse(ownership.disconnect(staleConnection)) + XCTAssertEqual( + ownership.registerConnection(reusedDescriptorConnection, authenticated: true), + .retained + ) + + XCTAssertFalse(ownership.disconnect(staleConnection)) + XCTAssertFalse(ownership.disconnect(otherConnection)) + XCTAssertTrue(ownership.disconnect(reusedDescriptorConnection)) + } + + func testTokenlessSessionStillRequiresAuthorizedPeer() { + XCTAssertFalse(isAuthenticatedAgentSession( + expectedToken: nil, + requestToken: nil, + authorizedPeer: false + )) + XCTAssertTrue(isAuthenticatedAgentSession( + expectedToken: nil, + requestToken: nil, + authorizedPeer: true + )) + } + + func testTokenSessionRequiresAuthorizedPeerAndMatchingToken() { + XCTAssertFalse(isAuthenticatedAgentSession( + expectedToken: "expected", + requestToken: "expected", + authorizedPeer: false + )) + XCTAssertFalse(isAuthenticatedAgentSession( + expectedToken: "expected", + requestToken: "wrong", + authorizedPeer: true + )) + XCTAssertTrue(isAuthenticatedAgentSession( + expectedToken: "expected", + requestToken: "expected", + authorizedPeer: true + )) + } +} + +private func connection(_ rawValue: UInt64) -> AgentSessionConnectionID { + AgentSessionConnectionID(rawValue: rawValue) +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AuthenticatedConnectionHangupMonitorTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AuthenticatedConnectionHangupMonitorTests.swift new file mode 100644 index 00000000000..1d335e98bb6 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AuthenticatedConnectionHangupMonitorTests.swift @@ -0,0 +1,195 @@ +import Darwin +import Dispatch +@testable import OrcaComputerUseMacOSCore +import XCTest + +final class AuthenticatedConnectionHangupMonitorTests: XCTestCase { + func testRejectsInvalidDescriptorDuringSetup() { + XCTAssertThrowsError( + try AuthenticatedConnectionHangupMonitor( + fileDescriptor: -1, + onHangup: {} + ) + ) + } + + func testRegistrationFailureClosesEventQueueOnce() { + let closes = DescriptorCloseRecorder() + + XCTAssertThrowsError( + try AuthenticatedConnectionHangupMonitor( + fileDescriptor: 42, + registerEvents: { _, _, _ in .EBADF }, + closeDescriptor: { descriptor in + closes.recordAndClose(descriptor) + }, + onHangup: {} + ) + ) + XCTAssertEqual(closes.count, 1) + } + + func testReportsPeerCloseWhileRequestHandlingIsBlocked() throws { + let descriptors = try makeSocketPair() + let hangup = expectation(description: "peer hangup") + let processingStarted = expectation(description: "processing started") + let releaseProcessing = DispatchSemaphore(value: 0) + let processingFinished = DispatchSemaphore(value: 0) + let monitor = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: descriptors.local, + onHangup: { + hangup.fulfill() + } + ) + monitor.start() + + DispatchQueue.global().async { + processingStarted.fulfill() + releaseProcessing.wait() + processingFinished.signal() + } + wait(for: [processingStarted], timeout: 1) + close(descriptors.peer) + + wait(for: [hangup], timeout: 1) + XCTAssertEqual(processingFinished.wait(timeout: .now() + 0.05), .timedOut) + + releaseProcessing.signal() + XCTAssertEqual(processingFinished.wait(timeout: .now() + 1), .success) + monitor.cancel() + close(descriptors.local) + } + + func testReadableDataDoesNotLookLikeHangup() throws { + let descriptors = try makeSocketPair() + let hangup = expectation(description: "peer hangup") + let callbacks = CallbackRecorder() + let events = CallbackRecorder() + let monitor = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: descriptors.local, + onEvent: { + events.record() + }, + onHangup: { + callbacks.record() + hangup.fulfill() + } + ) + monitor.start() + var byte: UInt8 = 7 + + XCTAssertEqual(write(descriptors.peer, &byte, 1), 1) + usleep(50_000) + XCTAssertEqual(callbacks.count, 0) + XCTAssertEqual(events.count, 0) + close(descriptors.peer) + wait(for: [hangup], timeout: 1) + XCTAssertEqual(events.count, 1) + + monitor.cancel() + close(descriptors.local) + } + + func testCancelPreventsLaterHangupCallback() throws { + let descriptors = try makeSocketPair() + let hangup = expectation(description: "peer hangup") + hangup.isInverted = true + let monitor = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: descriptors.local, + onHangup: { + hangup.fulfill() + } + ) + monitor.start() + + monitor.cancel() + close(descriptors.peer) + + wait(for: [hangup], timeout: 0.1) + close(descriptors.local) + } + + func testCancelReleasesMonitorWhileSocketStaysOpen() throws { + let descriptors = try makeSocketPair() + var monitor: AuthenticatedConnectionHangupMonitor? = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: descriptors.local, + onHangup: {} + ) + weak var retainedMonitor = monitor + + monitor?.cancel() + monitor = nil + + let deadline = Date().addingTimeInterval(1) + while retainedMonitor != nil, Date() < deadline { + usleep(10_000) + } + XCTAssertNil(retainedMonitor) + close(descriptors.peer) + close(descriptors.local) + } + + func testStartReportsHangupQueuedAfterSuccessfulSetup() throws { + let descriptors = try makeSocketPair() + let hangup = expectation(description: "queued peer hangup") + let monitor = try AuthenticatedConnectionHangupMonitor( + fileDescriptor: descriptors.local, + onHangup: { + hangup.fulfill() + } + ) + + close(descriptors.peer) + monitor.start() + + wait(for: [hangup], timeout: 1) + monitor.cancel() + close(descriptors.local) + } +} + +private final class CallbackRecorder: @unchecked Sendable { + private let lock = NSLock() + private var recordedCount = 0 + + var count: Int { + lock.lock() + defer { lock.unlock() } + return recordedCount + } + + func record() { + lock.lock() + recordedCount += 1 + lock.unlock() + } +} + +private final class DescriptorCloseRecorder: @unchecked Sendable { + private let lock = NSLock() + private var closeCount = 0 + + var count: Int { + lock.lock() + defer { lock.unlock() } + return closeCount + } + + func recordAndClose(_ descriptor: Int32) { + lock.lock() + closeCount += 1 + let shouldClose = closeCount == 1 + lock.unlock() + if shouldClose { + close(descriptor) + } + } +} + +private func makeSocketPair() throws -> (local: Int32, peer: Int32) { + var descriptors: [Int32] = [0, 0] + guard socketpair(AF_UNIX, SOCK_STREAM, 0, &descriptors) == 0 else { + throw POSIXError(.EIO) + } + return (descriptors[0], descriptors[1]) +} diff --git a/package.json b/package.json index 4c1e8474517..cacfbe341e1 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "test:e2e:terminal-ime-native": "node config/scripts/run-terminal-ibus-hangul-e2e.mjs", "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts", "bench:idle-cpu": "pnpm run ensure:electron-runtime && node config/scripts/run-idle-cpu-benchmark.mjs", + "bench:macos-computer-helper-owner-loss": "node config/scripts/macos-computer-helper-owner-loss-benchmark.mjs", "bench:startup": "pnpm run ensure:electron-runtime && node tools/benchmarks/startup-time-bench.mjs", "bench:daemon-coldstart": "pnpm run ensure:electron-runtime && node tools/benchmarks/daemon-coldstart-bench.mjs", "bench:main-thread-jank": "pnpm run ensure:electron-runtime && node tools/benchmarks/main-thread-jank-bench.mjs",