mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
fix(computer): reap mac helper after client loss (#11493)
* perf(computer): add mac helper owner-loss benchmark Measure the release helper's resident memory before and after its owner-session deadline. Record exact revisions, per-trial RSS, retained state, and clean-exit latency so lifecycle reclamation is reproducible. * fix(computer): reap mac helper after client loss Bind the detached macOS helper lifetime to authenticated socket ownership. Reap the helper after its final authenticated client disconnects, and add a startup deadline for sessions that never authenticate. * test(computer): harden owner benchmark cleanup * test(computer): make owner benchmark cleanup failure-safe * test(computer): close remaining owner cleanup races
This commit is contained in:
@@ -9,6 +9,12 @@ on:
|
||||
- 'config/scripts/build-windows-cli-launcher.mjs'
|
||||
- 'config/scripts/build-windows-cli-launcher.test.mjs'
|
||||
- 'config/scripts/computer-e2e-workflow.test.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-processes.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
- 'config/scripts/computer-use-modifier-safety.test.mjs'
|
||||
- 'config/scripts/computer-use-skill-guidance.test.mjs'
|
||||
- 'config/scripts/computer-use-smoke.mjs'
|
||||
@@ -59,6 +65,8 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: package.json
|
||||
@@ -85,6 +93,8 @@ jobs:
|
||||
config/scripts/build-windows-cli-launcher.test.mjs
|
||||
src/main/ssh/ssh-remote-cli-launcher.test.ts
|
||||
config/scripts/computer-e2e-workflow.test.mjs
|
||||
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
|
||||
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
|
||||
config/scripts/computer-use-modifier-safety.test.mjs
|
||||
config/scripts/computer-use-skill-guidance.test.mjs
|
||||
config/scripts/computer-use-smoke.test.mjs
|
||||
@@ -140,6 +150,32 @@ jobs:
|
||||
ORCA_COMPUTER_E2E: '1'
|
||||
run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts
|
||||
|
||||
mac-native-owner-smoke:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: macos-15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: package.json
|
||||
- uses: pnpm/action-setup@v6
|
||||
with:
|
||||
run_install: false
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- name: Owner-loss benchmark process cleanup
|
||||
run: >-
|
||||
pnpm vitest run
|
||||
config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs
|
||||
config/scripts/macos-computer-helper-owner-loss-processes.test.mjs
|
||||
- name: Authenticated helper owner-loss smoke
|
||||
run: pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1
|
||||
- name: Swift tests and signed universal helper verification
|
||||
run: pnpm verify:computer-native
|
||||
|
||||
mac:
|
||||
# macOS Accessibility and Screen Recording require user-granted TCC entries.
|
||||
# Keep this on manual/scheduled permission-bearing runners instead of PR CI.
|
||||
|
||||
@@ -47,6 +47,7 @@ describe('computer-use e2e workflow', () => {
|
||||
expect(triggerPaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
'config/scripts/computer-e2e-workflow.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs',
|
||||
'config/scripts/computer-use-modifier-safety.test.mjs',
|
||||
'config/scripts/computer-use-skill-guidance.test.mjs',
|
||||
'config/scripts/computer-use-smoke.mjs',
|
||||
@@ -70,9 +71,14 @@ describe('computer-use e2e workflow', () => {
|
||||
const nativeSmokeRuns = workflow.jobs['native-smoke'].steps
|
||||
.map((step) => step.run)
|
||||
.filter((run) => typeof run === 'string')
|
||||
const checkout = workflow.jobs['native-smoke'].steps.find(
|
||||
(step) => step.uses === 'actions/checkout@v6'
|
||||
)
|
||||
const regressionRun = nativeSmokeRuns.find((run) => run.includes('pnpm vitest run'))
|
||||
const expectedRegressionFiles = [
|
||||
'config/scripts/computer-e2e-workflow.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs',
|
||||
'config/scripts/computer-use-modifier-safety.test.mjs',
|
||||
'config/scripts/computer-use-skill-guidance.test.mjs',
|
||||
'config/scripts/computer-use-smoke.test.mjs',
|
||||
@@ -106,12 +112,68 @@ describe('computer-use e2e workflow', () => {
|
||||
'src/shared/remote-runtime-client.test.ts'
|
||||
]
|
||||
|
||||
expect(checkout.with['persist-credentials']).toBe(false)
|
||||
expect(regressionRun).toBeTruthy()
|
||||
for (const file of expectedRegressionFiles) {
|
||||
expect(regressionRun).toContain(file)
|
||||
}
|
||||
})
|
||||
|
||||
it('builds and tests the macOS helper on pull requests without TCC e2e', () => {
|
||||
const workflow = parse(
|
||||
readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8')
|
||||
)
|
||||
const job = workflow.jobs['mac-native-owner-smoke']
|
||||
const runs = job.steps.map((step) => step.run).filter((run) => typeof run === 'string')
|
||||
const checkout = job.steps.find((step) => step.uses === 'actions/checkout@v6')
|
||||
|
||||
expect(job.if).toBe("github.event_name == 'pull_request'")
|
||||
expect(job['runs-on']).toBe('macos-15')
|
||||
expect(checkout.with['persist-credentials']).toBe(false)
|
||||
expect(runs).toContain('pnpm bench:macos-computer-helper-owner-loss --expect reaped --trials 1')
|
||||
const cleanupRun = runs.find((run) =>
|
||||
run.includes('config/scripts/macos-computer-helper-owner-loss-processes.test.mjs')
|
||||
)
|
||||
expect(cleanupRun).toContain(
|
||||
'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs'
|
||||
)
|
||||
expect(runs).toContain('pnpm verify:computer-native')
|
||||
expect(runs.join('\n')).not.toContain('test:e2e:computer')
|
||||
expect(workflow.on.pull_request.paths).toEqual(
|
||||
expect.arrayContaining([
|
||||
'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-group-recovery.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-metrics.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('runs deterministic macOS owner-loss benchmark cleanup coverage', () => {
|
||||
const benchmark = readFileSync(
|
||||
join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'),
|
||||
'utf8'
|
||||
)
|
||||
const cleanup = readFileSync(
|
||||
join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(benchmark).toContain('spawnBenchmarkProcess(executable, [launcherDir]')
|
||||
expect(benchmark).toContain("stdio: ['ignore', stdoutDescriptor, stderrDescriptor]")
|
||||
expect(benchmark).toContain('cleanupOwnerLossTrial({')
|
||||
const parseIndex = benchmark.indexOf('parseBenchmarkTrialResult(serializedResult)')
|
||||
const cleanupIndex = benchmark.indexOf('cleanupOwnerLossTrial({')
|
||||
expect(parseIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(cleanupIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(parseIndex).toBeLessThan(cleanupIndex)
|
||||
expect(benchmark).toContain('trialCleanupSha256: artifactSha256(trialCleanupPath)')
|
||||
expect(cleanup).toContain('killRecordedAndMatchingProcesses(options.recordPath')
|
||||
expect(cleanup).toContain("signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL'")
|
||||
})
|
||||
|
||||
it('boots the built daemon under plain Node in the PR native-smoke job after the main build', () => {
|
||||
const workflow = parse(
|
||||
readFileSync(join(projectDir, '.github/workflows/computer-e2e.yml'), 'utf8')
|
||||
|
||||
@@ -0,0 +1,614 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from 'node:crypto'
|
||||
import { execFileSync, fork } from 'node:child_process'
|
||||
import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import net from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import {
|
||||
median,
|
||||
percentile,
|
||||
processSnapshot,
|
||||
sampleProcess
|
||||
} from './macos-computer-helper-owner-loss-metrics.mjs'
|
||||
import {
|
||||
benchmarkTrialNeedsCleanup,
|
||||
parseBenchmarkTrialResult,
|
||||
processIdentityIsCurrent,
|
||||
signalProcessIdentity,
|
||||
spawnBenchmarkProcess,
|
||||
throwBenchmarkTrialFailures,
|
||||
writeProcessRecord
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
|
||||
const INTERNAL_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_INTERNAL'
|
||||
const EXPECTATION_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_EXPECTATION'
|
||||
const HELPER_RECORD_PATH_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_HELPER_RECORD_PATH'
|
||||
const RESULT_PATH_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_RESULT_PATH'
|
||||
const ACTIVE_REQUEST_COUNT = 100_000
|
||||
const DEFAULT_TRIALS = 3
|
||||
const OWNER_HOLD_MS = 31_000
|
||||
const PROCESS_EXIT_TIMEOUT_MS = 5_000
|
||||
const RETAIN_PROOF_MS = 3_000
|
||||
const TRIAL_TIMEOUT_MS = OWNER_HOLD_MS + 4 * PROCESS_EXIT_TIMEOUT_MS + 120_000
|
||||
const MIB = 1024 * 1024
|
||||
const scriptPath = import.meta.filename
|
||||
const repoRoot = path.resolve(import.meta.dirname, '..', '..')
|
||||
const metricsPath = path.join(import.meta.dirname, 'macos-computer-helper-owner-loss-metrics.mjs')
|
||||
const processCleanupPath = path.join(
|
||||
import.meta.dirname,
|
||||
'macos-computer-helper-owner-loss-processes.mjs'
|
||||
)
|
||||
const trialCleanupPath = path.join(
|
||||
import.meta.dirname,
|
||||
'macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
)
|
||||
const sidecarPath = path.join(repoRoot, 'out', 'main', 'computer-sidecar.js')
|
||||
const helperAppPath = path.join(
|
||||
repoRoot,
|
||||
'native',
|
||||
'computer-use-macos',
|
||||
'.build',
|
||||
'release',
|
||||
'Orca Computer Use.app'
|
||||
)
|
||||
const helperPath = path.join(helperAppPath, 'Contents', 'MacOS', 'orca-computer-use-macos')
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForProcessExit(identity, timeoutMs) {
|
||||
const startedAt = performance.now()
|
||||
while (performance.now() - startedAt < timeoutMs) {
|
||||
if (!processIdentityIsCurrent(identity)) {
|
||||
return performance.now() - startedAt
|
||||
}
|
||||
await sleep(50)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
async function stopProcess(identity) {
|
||||
if (!identity || !processIdentityIsCurrent(identity)) {
|
||||
return
|
||||
}
|
||||
signalProcessIdentity(identity, helperPath, 'SIGTERM')
|
||||
if ((await waitForProcessExit(identity, 2_000)) !== null) {
|
||||
return
|
||||
}
|
||||
signalProcessIdentity(identity, helperPath, 'SIGKILL')
|
||||
await waitForProcessExit(identity, 2_000)
|
||||
}
|
||||
|
||||
function startSidecar() {
|
||||
const errors = []
|
||||
const child = fork(sidecarPath, [], {
|
||||
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_COMPUTER_SIDECAR: '1',
|
||||
ORCA_COMPUTER_MACOS_HELPER_APP_PATH: helperAppPath
|
||||
}
|
||||
})
|
||||
child.on('error', (error) => errors.push(error.stack ?? error.message))
|
||||
child.stderr?.on('data', (chunk) => errors.push(String(chunk)))
|
||||
return { child, errors }
|
||||
}
|
||||
|
||||
function requestSidecar(sidecar, id, method) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup()
|
||||
reject(new Error(`Sidecar ${method} request timed out: ${sidecar.errors.join('')}`))
|
||||
}, 10_000)
|
||||
const onMessage = (message) => {
|
||||
if (message?.id !== id) {
|
||||
return
|
||||
}
|
||||
cleanup()
|
||||
if (message.ok) {
|
||||
resolve(message.result)
|
||||
} else {
|
||||
reject(new Error(`Sidecar ${method} failed: ${JSON.stringify(message.error)}`))
|
||||
}
|
||||
}
|
||||
const onExit = (code, signal) => {
|
||||
cleanup()
|
||||
reject(
|
||||
new Error(
|
||||
`Sidecar exited during ${method}: ${JSON.stringify({ code, signal, stderr: sidecar.errors.join('') })}`
|
||||
)
|
||||
)
|
||||
}
|
||||
const onError = (error) => {
|
||||
cleanup()
|
||||
reject(new Error(`Sidecar ${method} process error: ${error.message}`))
|
||||
}
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout)
|
||||
sidecar.child.off('message', onMessage)
|
||||
sidecar.child.off('exit', onExit)
|
||||
sidecar.child.off('error', onError)
|
||||
}
|
||||
sidecar.child.on('message', onMessage)
|
||||
sidecar.child.once('exit', onExit)
|
||||
sidecar.child.once('error', onError)
|
||||
try {
|
||||
sidecar.child.send({ id, method, params: {} })
|
||||
} catch (error) {
|
||||
onError(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForHelper(sidecarPid) {
|
||||
const deadline = Date.now() + 10_000
|
||||
while (Date.now() < deadline) {
|
||||
const output = execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,command='], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 20 * 1024 * 1024
|
||||
})
|
||||
for (const line of output.split('\n')) {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/)
|
||||
if (
|
||||
match &&
|
||||
Number(match[2]) === sidecarPid &&
|
||||
Number(match[3]) === Number(match[1]) &&
|
||||
match[4].includes(helperPath) &&
|
||||
match[4].includes(' --agent ')
|
||||
) {
|
||||
return { pid: Number(match[1]), pgid: Number(match[3]), command: match[4] }
|
||||
}
|
||||
}
|
||||
await sleep(50)
|
||||
}
|
||||
throw new Error(`Could not find helper owned by sidecar ${sidecarPid}`)
|
||||
}
|
||||
|
||||
function socketPathFromCommand(command) {
|
||||
const match = command.match(/ --agent (.+?) --token-file /)
|
||||
if (!match) {
|
||||
throw new Error(`Could not read helper socket path from command: ${command}`)
|
||||
}
|
||||
return match[1]
|
||||
}
|
||||
|
||||
function connectInvalidPeer(socketPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = net.createConnection(socketPath)
|
||||
let accepted = false
|
||||
const timeout = setTimeout(() => {
|
||||
socket.destroy()
|
||||
reject(new Error('Invalid-peer connection timed out'))
|
||||
}, 5_000)
|
||||
let buffer = ''
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout)
|
||||
socket.off('error', onError)
|
||||
socket.off('data', onData)
|
||||
}
|
||||
const onError = (error) => {
|
||||
if (accepted) {
|
||||
return
|
||||
}
|
||||
cleanup()
|
||||
reject(error)
|
||||
}
|
||||
const onData = (chunk) => {
|
||||
buffer += chunk
|
||||
const newline = buffer.indexOf('\n')
|
||||
if (newline < 0) {
|
||||
return
|
||||
}
|
||||
let response
|
||||
try {
|
||||
response = JSON.parse(buffer.slice(0, newline))
|
||||
} catch (error) {
|
||||
cleanup()
|
||||
socket.destroy()
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
if (response.ok !== false || response.error?.code !== 'permission_denied') {
|
||||
cleanup()
|
||||
socket.destroy()
|
||||
reject(new Error(`Invalid peer was not rejected: ${JSON.stringify(response)}`))
|
||||
return
|
||||
}
|
||||
clearTimeout(timeout)
|
||||
socket.off('data', onData)
|
||||
accepted = true
|
||||
resolve(socket)
|
||||
}
|
||||
socket.setEncoding('utf8')
|
||||
socket.on('error', onError)
|
||||
socket.on('data', onData)
|
||||
socket.once('connect', () => {
|
||||
socket.write(
|
||||
`${JSON.stringify({ id: 991, method: 'handshake', params: {}, token: 'invalid' })}\n`
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForChildExit(child, timeoutMs) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return true
|
||||
}
|
||||
return await Promise.race([
|
||||
new Promise((resolve) => child.once('exit', () => resolve(true))),
|
||||
sleep(timeoutMs).then(() => false)
|
||||
])
|
||||
}
|
||||
|
||||
async function startAuthenticatedSession() {
|
||||
const sidecar = startSidecar()
|
||||
let helper
|
||||
try {
|
||||
const capabilitiesResult = requestSidecar(sidecar, 1, 'capabilities').then(
|
||||
(capabilities) => ({ capabilities }),
|
||||
(error) => ({ error })
|
||||
)
|
||||
helper = await waitForHelper(sidecar.child.pid)
|
||||
const helperRecordPath = process.env[HELPER_RECORD_PATH_ENV]
|
||||
if (!helperRecordPath) {
|
||||
throw new Error('Missing helper process record path')
|
||||
}
|
||||
writeProcessRecord(helperRecordPath, helper)
|
||||
const { capabilities, error } = await capabilitiesResult
|
||||
if (error) {
|
||||
throw error
|
||||
}
|
||||
if (capabilities?.protocolVersion !== 1) {
|
||||
throw new Error(`Unexpected helper handshake: ${JSON.stringify(capabilities)}`)
|
||||
}
|
||||
return { authenticated: capabilities.protocolVersion === 1, sidecar, helper }
|
||||
} catch (error) {
|
||||
sidecar.child.kill('SIGKILL')
|
||||
await stopProcess(helper)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function exerciseActiveRequests(sidecar) {
|
||||
const latencies = []
|
||||
const startedAt = performance.now()
|
||||
for (let index = 0; index < ACTIVE_REQUEST_COUNT; index += 1) {
|
||||
const requestStartedAt = performance.now()
|
||||
const result = await requestSidecar(sidecar, 10_000 + index, 'listApps')
|
||||
if (!Array.isArray(result?.apps)) {
|
||||
throw new Error(`Unexpected listApps response: ${JSON.stringify(result)}`)
|
||||
}
|
||||
latencies.push(performance.now() - requestStartedAt)
|
||||
}
|
||||
const totalMs = performance.now() - startedAt
|
||||
return {
|
||||
totalMs,
|
||||
requestsPerSecond: (ACTIVE_REQUEST_COUNT * 1_000) / totalMs,
|
||||
medianLatencyMs: median(latencies),
|
||||
p95LatencyMs: percentile(latencies, 0.95),
|
||||
maxLatencyMs: Math.max(...latencies)
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyGracefulClose() {
|
||||
const { sidecar, helper } = await startAuthenticatedSession()
|
||||
try {
|
||||
const startedAt = performance.now()
|
||||
sidecar.child.disconnect()
|
||||
if (!(await waitForChildExit(sidecar.child, PROCESS_EXIT_TIMEOUT_MS))) {
|
||||
throw new Error('Sidecar did not exit after graceful IPC close')
|
||||
}
|
||||
const helperExitMs = await waitForProcessExit(helper, PROCESS_EXIT_TIMEOUT_MS)
|
||||
if (helperExitMs === null) {
|
||||
throw new Error('Helper did not exit after graceful owner close')
|
||||
}
|
||||
return Math.round(performance.now() - startedAt)
|
||||
} finally {
|
||||
sidecar.child.kill('SIGKILL')
|
||||
await stopProcess(helper)
|
||||
}
|
||||
}
|
||||
|
||||
async function runInternalTrial(expectation) {
|
||||
let sidecar
|
||||
let helper
|
||||
let invalidPeer
|
||||
let invalidPeerRejected = false
|
||||
try {
|
||||
const session = await startAuthenticatedSession()
|
||||
sidecar = session.sidecar
|
||||
helper = session.helper
|
||||
const authenticatedAt = performance.now()
|
||||
const initial = await sampleProcess(helper.pid)
|
||||
const activeRequests = await exerciseActiveRequests(sidecar)
|
||||
const remainingHoldMs = Math.max(0, OWNER_HOLD_MS - (performance.now() - authenticatedAt))
|
||||
await sleep(remainingHoldMs)
|
||||
const connected = await sampleProcess(helper.pid)
|
||||
const invalidSocketPath = socketPathFromCommand(helper.command)
|
||||
invalidPeer = await connectInvalidPeer(invalidSocketPath)
|
||||
invalidPeerRejected = true
|
||||
const survivedClaimDeadline =
|
||||
performance.now() - authenticatedAt >= OWNER_HOLD_MS && isProcessAlive(helper.pid)
|
||||
|
||||
sidecar.child.kill('SIGKILL')
|
||||
await waitForChildExit(sidecar.child, PROCESS_EXIT_TIMEOUT_MS)
|
||||
const abruptExitMs = await waitForProcessExit(
|
||||
helper,
|
||||
expectation === 'reaped' ? PROCESS_EXIT_TIMEOUT_MS : RETAIN_PROOF_MS
|
||||
)
|
||||
const helperExitedAfterAbruptLoss = abruptExitMs !== null
|
||||
if (expectation === 'reaped' && !helperExitedAfterAbruptLoss) {
|
||||
throw new Error('Expected helper to exit after abrupt authenticated owner loss')
|
||||
}
|
||||
if (expectation === 'retained' && helperExitedAfterAbruptLoss) {
|
||||
throw new Error('Expected baseline helper to remain after abrupt owner loss')
|
||||
}
|
||||
const postLossRssBytes = helperExitedAfterAbruptLoss ? 0 : processSnapshot(helper.pid).rssBytes
|
||||
await stopProcess(helper)
|
||||
helper = null
|
||||
invalidPeer.destroy()
|
||||
invalidPeer = null
|
||||
|
||||
const gracefulExitMs = await verifyGracefulClose()
|
||||
return {
|
||||
authenticated: session.authenticated,
|
||||
survivedClaimDeadline,
|
||||
invalidPeerRejectedAndDidNotRetain: invalidPeerRejected && helperExitedAfterAbruptLoss,
|
||||
connectedRssBytes: connected.rssBytes,
|
||||
connectedCpuMilliseconds: Math.max(
|
||||
0,
|
||||
Math.round((connected.cpuTimeSeconds - initial.cpuTimeSeconds) * 1_000)
|
||||
),
|
||||
activeRequests,
|
||||
cpuSampleMs: Math.round(performance.now() - authenticatedAt),
|
||||
helperExitedAfterAbruptLoss,
|
||||
abruptExitMs: abruptExitMs === null ? null : Math.round(abruptExitMs),
|
||||
postLossRssBytes,
|
||||
gracefulExitMs
|
||||
}
|
||||
} finally {
|
||||
invalidPeer?.destroy()
|
||||
sidecar?.child.kill('SIGKILL')
|
||||
await stopProcess(helper)
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { expect: '', trials: DEFAULT_TRIALS, output: '' }
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (arg === '--expect' || arg === '--trials' || arg === '--output') {
|
||||
if (!value) {
|
||||
throw new Error(`Missing value for ${arg}`)
|
||||
}
|
||||
options[arg.slice(2)] = arg === '--trials' ? Number(value) : value
|
||||
index += 1
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`)
|
||||
}
|
||||
}
|
||||
if (!['retained', 'reaped'].includes(options.expect)) {
|
||||
throw new Error('--expect must be retained or reaped')
|
||||
}
|
||||
if (!Number.isInteger(options.trials) || options.trials < 1) {
|
||||
throw new Error('--trials must be a positive integer')
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
function electronPath() {
|
||||
const electronModulePath = fileURLToPath(import.meta.resolve('electron'))
|
||||
return execFileSync(
|
||||
process.execPath,
|
||||
['-e', `process.stdout.write(require(${JSON.stringify(electronModulePath)}))`],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
}
|
||||
|
||||
function buildArtifacts() {
|
||||
execFileSync('pnpm', ['exec', 'electron-vite', 'build'], {
|
||||
cwd: repoRoot,
|
||||
stdio: 'inherit'
|
||||
})
|
||||
execFileSync('pnpm', ['build:computer-macos'], {
|
||||
cwd: repoRoot,
|
||||
stdio: 'inherit'
|
||||
})
|
||||
}
|
||||
|
||||
function artifactSha256(artifactPath) {
|
||||
return createHash('sha256').update(readFileSync(artifactPath)).digest('hex')
|
||||
}
|
||||
|
||||
function runTrial(executable, expectation) {
|
||||
let launcherDir
|
||||
let trialTempDir
|
||||
let helperRecordPath
|
||||
let resultPath
|
||||
let stderrPath
|
||||
let stdoutPath
|
||||
let result
|
||||
let serializedResult
|
||||
let parsedResult
|
||||
let parsedResultAvailable = false
|
||||
let trialError
|
||||
let cleanupError
|
||||
let stderrDescriptor
|
||||
let stdoutDescriptor
|
||||
let trialOutput = ''
|
||||
try {
|
||||
launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-helper-owner-bench-launcher-'))
|
||||
trialTempDir = mkdtempSync(path.join(path.sep, 'tmp', 'orca-owner-bench-'))
|
||||
helperRecordPath = path.join(launcherDir, 'helper.json')
|
||||
resultPath = path.join(launcherDir, 'result.json')
|
||||
stderrPath = path.join(launcherDir, 'stderr.log')
|
||||
stdoutPath = path.join(launcherDir, 'stdout.log')
|
||||
writeFileSync(
|
||||
path.join(launcherDir, 'package.json'),
|
||||
JSON.stringify({ name: 'orca-helper-owner-benchmark', main: 'main.cjs' })
|
||||
)
|
||||
writeFileSync(
|
||||
path.join(launcherDir, 'main.cjs'),
|
||||
`import(${JSON.stringify(pathToFileURL(scriptPath).href)}).catch((error) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})\n`
|
||||
)
|
||||
const env = {
|
||||
...process.env,
|
||||
TMPDIR: trialTempDir,
|
||||
[INTERNAL_ENV]: '1',
|
||||
[EXPECTATION_ENV]: expectation,
|
||||
[HELPER_RECORD_PATH_ENV]: helperRecordPath,
|
||||
[RESULT_PATH_ENV]: resultPath
|
||||
}
|
||||
delete env.ELECTRON_RUN_AS_NODE
|
||||
stderrDescriptor = openSync(stderrPath, 'w')
|
||||
stdoutDescriptor = openSync(stdoutPath, 'w')
|
||||
result = spawnBenchmarkProcess(executable, [launcherDir], {
|
||||
cwd: repoRoot,
|
||||
env,
|
||||
stdio: ['ignore', stdoutDescriptor, stderrDescriptor],
|
||||
timeout: TRIAL_TIMEOUT_MS
|
||||
})
|
||||
if (result.status === 0 && existsSync(resultPath)) {
|
||||
serializedResult = readFileSync(resultPath, 'utf8')
|
||||
parsedResult = parseBenchmarkTrialResult(serializedResult)
|
||||
parsedResultAvailable = true
|
||||
}
|
||||
} catch (error) {
|
||||
trialError = error
|
||||
} finally {
|
||||
const failedTrial = benchmarkTrialNeedsCleanup(result, parsedResultAvailable)
|
||||
const trialMarker = trialTempDir ? `TMPDIR=${trialTempDir}` : undefined
|
||||
const cleanup = cleanupOwnerLossTrial({
|
||||
failed: failedTrial,
|
||||
pid: result?.pid,
|
||||
marker: trialMarker,
|
||||
recordPath: helperRecordPath,
|
||||
helperPath,
|
||||
tempDir: trialTempDir,
|
||||
stderrDescriptor,
|
||||
stdoutDescriptor,
|
||||
outputPaths: [stderrPath, stdoutPath],
|
||||
launcherDir
|
||||
})
|
||||
cleanupError = cleanup.error
|
||||
trialOutput = cleanup.output
|
||||
}
|
||||
if (!trialError && result?.status !== 0) {
|
||||
trialError = new Error(
|
||||
`Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n${trialOutput}`
|
||||
)
|
||||
}
|
||||
if (!trialError && !serializedResult) {
|
||||
trialError = new Error(`Electron trial did not write a result:\n${trialOutput}`)
|
||||
}
|
||||
throwBenchmarkTrialFailures(trialError, cleanupError)
|
||||
return parsedResult
|
||||
}
|
||||
|
||||
function runBenchmark() {
|
||||
if (process.platform !== 'darwin') {
|
||||
throw new Error('The computer-use helper owner benchmark is macOS-only')
|
||||
}
|
||||
const options = parseArgs(process.argv.slice(2))
|
||||
const dirty = execFileSync('git', ['status', '--porcelain'], {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
if (dirty) {
|
||||
throw new Error('Commit or stash changes before running the provenance-bound benchmark')
|
||||
}
|
||||
buildArtifacts()
|
||||
if (!existsSync(sidecarPath) || !existsSync(helperPath)) {
|
||||
throw new Error('Fresh production sidecar/helper build did not produce the expected artifacts')
|
||||
}
|
||||
const executable = electronPath()
|
||||
const results = Array.from({ length: options.trials }, () => runTrial(executable, options.expect))
|
||||
const rssBytes = results.map((result) => result.connectedRssBytes)
|
||||
const cpuMilliseconds = results.map((result) => result.connectedCpuMilliseconds)
|
||||
const activeRequestTotals = results.map((result) => result.activeRequests.totalMs)
|
||||
const activeRequestRates = results.map((result) => result.activeRequests.requestsPerSecond)
|
||||
const activeRequestMedians = results.map((result) => result.activeRequests.medianLatencyMs)
|
||||
const activeRequestP95s = results.map((result) => result.activeRequests.p95LatencyMs)
|
||||
const activeRequestMaxes = results.map((result) => result.activeRequests.maxLatencyMs)
|
||||
const postLossRssBytes = results.map((result) => result.postLossRssBytes)
|
||||
const report = {
|
||||
benchmark: 'macos-computer-helper-authenticated-owner-loss',
|
||||
revision: execFileSync('git', ['rev-parse', 'HEAD'], {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8'
|
||||
}).trim(),
|
||||
artifacts: {
|
||||
sidecarSha256: artifactSha256(sidecarPath),
|
||||
helperSha256: artifactSha256(helperPath)
|
||||
},
|
||||
sources: {
|
||||
benchmarkSha256: artifactSha256(scriptPath),
|
||||
metricsSha256: artifactSha256(metricsPath),
|
||||
processCleanupSha256: artifactSha256(processCleanupPath),
|
||||
trialCleanupSha256: artifactSha256(trialCleanupPath)
|
||||
},
|
||||
expectation: options.expect,
|
||||
trials: options.trials,
|
||||
ownerHoldMs: OWNER_HOLD_MS,
|
||||
activeRequestCount: ACTIVE_REQUEST_COUNT,
|
||||
authenticated: results.every((result) => result.authenticated),
|
||||
survivedClaimDeadline: results.every((result) => result.survivedClaimDeadline),
|
||||
invalidPeerRejectedAndDidNotRetain: results.every(
|
||||
(result) => result.invalidPeerRejectedAndDidNotRetain
|
||||
),
|
||||
connectedRssMiB: rssBytes.map((value) => Number((value / MIB).toFixed(2))),
|
||||
medianConnectedRssMiB: Number((median(rssBytes) / MIB).toFixed(2)),
|
||||
connectedCpuMilliseconds: cpuMilliseconds,
|
||||
medianConnectedCpuMilliseconds: median(cpuMilliseconds),
|
||||
activeRequestTotalMs: activeRequestTotals.map((value) => Number(value.toFixed(3))),
|
||||
medianActiveRequestTotalMs: Number(median(activeRequestTotals).toFixed(3)),
|
||||
activeRequestsPerSecond: activeRequestRates.map((value) => Number(value.toFixed(2))),
|
||||
medianActiveRequestsPerSecond: Number(median(activeRequestRates).toFixed(2)),
|
||||
activeRequestMedianLatencyMs: activeRequestMedians.map((value) => Number(value.toFixed(3))),
|
||||
medianActiveRequestMedianLatencyMs: Number(median(activeRequestMedians).toFixed(3)),
|
||||
activeRequestP95LatencyMs: activeRequestP95s.map((value) => Number(value.toFixed(3))),
|
||||
medianActiveRequestP95LatencyMs: Number(median(activeRequestP95s).toFixed(3)),
|
||||
activeRequestMaxLatencyMs: activeRequestMaxes.map((value) => Number(value.toFixed(3))),
|
||||
helperExitedAfterAbruptLoss: results.map((result) => result.helperExitedAfterAbruptLoss),
|
||||
abruptExitMs: results.map((result) => result.abruptExitMs),
|
||||
postLossRssMiB: postLossRssBytes.map((value) => Number((value / MIB).toFixed(2))),
|
||||
medianPostLossRssMiB: Number((median(postLossRssBytes) / MIB).toFixed(2)),
|
||||
gracefulExitMs: results.map((result) => result.gracefulExitMs)
|
||||
}
|
||||
const serialized = `${JSON.stringify(report, null, 2)}\n`
|
||||
process.stdout.write(serialized)
|
||||
if (options.output) {
|
||||
writeFileSync(path.resolve(options.output), serialized)
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env[INTERNAL_ENV] === '1') {
|
||||
const { app } = await import('electron')
|
||||
await app.whenReady()
|
||||
try {
|
||||
const result = await runInternalTrial(process.env[EXPECTATION_ENV])
|
||||
writeFileSync(process.env[RESULT_PATH_ENV], JSON.stringify(result))
|
||||
} finally {
|
||||
app.quit()
|
||||
}
|
||||
} else {
|
||||
runBenchmark()
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { signalValidatedProcessGroup } from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
|
||||
describe('macOS helper owner-loss benchmark group recovery', () => {
|
||||
it('retains uncertain stop state across cleanup stage failures', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }]
|
||||
const groupState = { stopped: false }
|
||||
let scanCount = 0
|
||||
let continueAttempts = 0
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount >= 3) {
|
||||
throw new Error(
|
||||
scanCount === 3 ? 'post-stop inspection failed' : 'final inspection failed'
|
||||
)
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
if (pid === -41 && signal === 'SIGCONT') {
|
||||
continueAttempts += 1
|
||||
if (continueAttempts === 1) {
|
||||
throw new Error('first compensation failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)
|
||||
).toThrow('Benchmark process group signal recovery failed')
|
||||
expect(groupState.stopped).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('final inspection failed')
|
||||
expect(continueAttempts).toBe(2)
|
||||
expect(groupState.stopped).toBe(false)
|
||||
})
|
||||
|
||||
it('retains an uncertain anchor stop across cleanup stage failures', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }]
|
||||
const groupState = { stopped: false, anchorPid: null }
|
||||
let scanCount = 0
|
||||
let continueAttempts = 0
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount >= 2) {
|
||||
throw new Error(
|
||||
scanCount === 2 ? 'post-anchor inspection failed' : 'final inspection failed'
|
||||
)
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
if (pid === 41 && signal === 'SIGCONT') {
|
||||
continueAttempts += 1
|
||||
if (continueAttempts === 1) {
|
||||
throw new Error('first anchor compensation failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)
|
||||
).toThrow('Benchmark process group signal recovery failed')
|
||||
expect(groupState.anchorPid).toBe(41)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('final inspection failed')
|
||||
expect(continueAttempts).toBe(2)
|
||||
expect(groupState.anchorPid).toBeNull()
|
||||
})
|
||||
|
||||
it('recovers a retained anchor before selecting a new one', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const firstAnchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` }
|
||||
const finalAnchor = { pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
const groupState = { stopped: false, anchorPid: null }
|
||||
let scanCount = 0
|
||||
let firstAnchorContinues = 0
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 2) {
|
||||
throw new Error('post-anchor inspection failed')
|
||||
}
|
||||
return scanCount === 1 ? [firstAnchor] : [finalAnchor]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
if (pid === 41 && signal === 'SIGCONT') {
|
||||
firstAnchorContinues += 1
|
||||
if (firstAnchorContinues === 1) {
|
||||
throw new Error('first anchor compensation failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)
|
||||
).toThrow('Benchmark process group signal recovery failed')
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(true)
|
||||
expect(firstAnchorContinues).toBe(2)
|
||||
expect(groupState).toEqual({ stopped: false, anchorPid: null })
|
||||
})
|
||||
|
||||
it('preserves group authority errors when compensation fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const ownershipError = 'Benchmark process group no longer belongs to this trial'
|
||||
let thrown
|
||||
|
||||
try {
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: true },
|
||||
{
|
||||
processIdentities: () => [{ pid: 41, pgid: 41, command: '/unrelated' }],
|
||||
signalProcess: () => {
|
||||
throw new Error('resume denied')
|
||||
}
|
||||
}
|
||||
)
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual([ownershipError, 'resume denied'])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
|
||||
const SAMPLE_COUNT = 5
|
||||
const SAMPLE_INTERVAL_MS = 200
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
export function median(values) {
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
const middle = Math.floor(sorted.length / 2)
|
||||
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]
|
||||
}
|
||||
|
||||
export function percentile(values, fraction) {
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)]
|
||||
}
|
||||
|
||||
function parseCpuTimeSeconds(value) {
|
||||
const [dayOrTime, clock] = value.includes('-') ? value.split('-', 2) : [null, value]
|
||||
const days = dayOrTime === null ? 0 : Number(dayOrTime)
|
||||
const parts = clock.split(':').map(Number)
|
||||
if (!Number.isFinite(days) || parts.some((part) => !Number.isFinite(part))) {
|
||||
throw new Error(`Invalid process CPU time: ${value}`)
|
||||
}
|
||||
const seconds = parts.pop() ?? 0
|
||||
const minutes = parts.pop() ?? 0
|
||||
const hours = parts.pop() ?? 0
|
||||
return days * 86_400 + hours * 3_600 + minutes * 60 + seconds
|
||||
}
|
||||
|
||||
export function processSnapshot(pid) {
|
||||
const raw = execFileSync(
|
||||
'ps',
|
||||
['-o', 'rss=', '-o', 'time=', '-o', 'command=', '-p', String(pid)],
|
||||
{ encoding: 'utf8' }
|
||||
).trim()
|
||||
const match = raw.match(/^(\d+)\s+(\S+)\s+(.+)$/)
|
||||
if (!match) {
|
||||
throw new Error(`Could not inspect process ${pid}: ${raw}`)
|
||||
}
|
||||
return {
|
||||
rssBytes: Number(match[1]) * 1024,
|
||||
cpuTimeSeconds: parseCpuTimeSeconds(match[2]),
|
||||
command: match[3]
|
||||
}
|
||||
}
|
||||
|
||||
export async function sampleProcess(pid) {
|
||||
const samples = []
|
||||
for (let index = 0; index < SAMPLE_COUNT; index += 1) {
|
||||
samples.push(processSnapshot(pid))
|
||||
await sleep(SAMPLE_INTERVAL_MS)
|
||||
}
|
||||
return {
|
||||
rssBytes: median(samples.map((sample) => sample.rssBytes)),
|
||||
cpuTimeSeconds: samples.at(-1).cpuTimeSeconds,
|
||||
command: samples.at(-1).command
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
import { execFileSync, spawnSync } from 'node:child_process'
|
||||
import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
|
||||
|
||||
const PROCESS_EXIT_TIMEOUT_MS = 2_000
|
||||
const PROCESS_POLL_MS = 25
|
||||
const sleepBuffer = new Int32Array(new SharedArrayBuffer(4))
|
||||
|
||||
const processIdentityOperations = {
|
||||
executePs: execFileSync,
|
||||
signalProcess: process.kill.bind(process)
|
||||
}
|
||||
|
||||
export function processIdentity(pid, operations = processIdentityOperations) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const output = operations
|
||||
.executePs('ps', ['-p', String(pid), '-o', 'pid=,pgid=,command='], {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
.trim()
|
||||
const match = output.match(/^(\d+)\s+(\d+)\s+(.+)$/)
|
||||
if (!match) {
|
||||
throw new Error(`Could not parse process identity for ${pid}`)
|
||||
}
|
||||
return { pid: Number(match[1]), pgid: Number(match[2]), command: match[3] }
|
||||
} catch (error) {
|
||||
try {
|
||||
operations.signalProcess(pid, 0)
|
||||
} catch (lookupError) {
|
||||
if (lookupError?.code === 'ESRCH') {
|
||||
return null
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function matchingDetachedProcesses(identities, expectedCommandFragments) {
|
||||
return identities.filter(
|
||||
(identity) =>
|
||||
identity.pgid === identity.pid &&
|
||||
expectedCommandFragments.every((fragment) => identity.command.includes(fragment))
|
||||
)
|
||||
}
|
||||
|
||||
const matchingProcessOperations = {
|
||||
processIdentities,
|
||||
signalProcessIdentity,
|
||||
waitForIdentityExit
|
||||
}
|
||||
|
||||
export function killProcessMatchingCommand(
|
||||
expectedCommandFragments,
|
||||
operations = matchingProcessOperations
|
||||
) {
|
||||
const matches = matchingDetachedProcesses(
|
||||
operations.processIdentities(),
|
||||
expectedCommandFragments
|
||||
)
|
||||
if (matches.length === 0) {
|
||||
return false
|
||||
}
|
||||
const errors = []
|
||||
for (const match of matches) {
|
||||
try {
|
||||
if (operations.signalProcessIdentity(match, expectedCommandFragments[0], 'SIGKILL')) {
|
||||
operations.waitForIdentityExit(match)
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
}
|
||||
try {
|
||||
const remaining = matchingDetachedProcesses(
|
||||
operations.processIdentities(),
|
||||
expectedCommandFragments
|
||||
)
|
||||
if (remaining.length > 0) {
|
||||
errors.push(
|
||||
new Error(
|
||||
`Benchmark helper cleanup left matching processes: ${remaining
|
||||
.map((identity) => identity.pid)
|
||||
.join(', ')}`
|
||||
)
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
if (errors.length === 1) {
|
||||
throw errors[0]
|
||||
}
|
||||
if (errors.length > 1) {
|
||||
throw new AggregateError(errors, 'Benchmark exact-command cleanup failed')
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function sleepSync(milliseconds) {
|
||||
Atomics.wait(sleepBuffer, 0, 0, milliseconds)
|
||||
}
|
||||
|
||||
function validateDetachedIdentity(identity, expectedCommandFragment) {
|
||||
if (
|
||||
!Number.isInteger(identity?.pid) ||
|
||||
identity.pid <= 0 ||
|
||||
identity.pgid !== identity.pid ||
|
||||
typeof identity.command !== 'string' ||
|
||||
!identity.command.includes(expectedCommandFragment)
|
||||
) {
|
||||
throw new Error('Recorded benchmark helper identity is invalid')
|
||||
}
|
||||
}
|
||||
|
||||
function sameIdentity(left, right) {
|
||||
return left?.pid === right?.pid && left?.pgid === right?.pgid && left?.command === right?.command
|
||||
}
|
||||
|
||||
function processIdentities(includeEnvironment = false) {
|
||||
const args = includeEnvironment
|
||||
? ['eww', '-axo', 'pid=,pgid=,command=']
|
||||
: ['-axo', 'pid=,pgid=,command=']
|
||||
const output = execFileSync('ps', args, {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 20 * 1024 * 1024
|
||||
})
|
||||
return output
|
||||
.split('\n')
|
||||
.map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/))
|
||||
.filter(Boolean)
|
||||
.map((match) => ({
|
||||
pid: Number(match[1]),
|
||||
pgid: Number(match[2]),
|
||||
command: match[3]
|
||||
}))
|
||||
}
|
||||
|
||||
function waitForIdentityExit(identity) {
|
||||
const deadline = Date.now() + PROCESS_EXIT_TIMEOUT_MS
|
||||
while (Date.now() < deadline) {
|
||||
if (!processIdentityIsCurrent(identity)) {
|
||||
return true
|
||||
}
|
||||
sleepSync(PROCESS_POLL_MS)
|
||||
}
|
||||
throw new Error(`Recorded benchmark helper ${identity.pid} did not exit`)
|
||||
}
|
||||
|
||||
export function spawnBenchmarkProcess(executable, args, options) {
|
||||
return spawnSync(executable, args, {
|
||||
...options,
|
||||
detached: true,
|
||||
killSignal: 'SIGKILL'
|
||||
})
|
||||
}
|
||||
|
||||
export function runBenchmarkCleanupStages(stages) {
|
||||
const errors = []
|
||||
for (const stage of stages) {
|
||||
try {
|
||||
stage()
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
}
|
||||
if (errors.length === 1) {
|
||||
throw errors[0]
|
||||
}
|
||||
if (errors.length > 1) {
|
||||
throw new AggregateError(errors, 'Benchmark trial cleanup failed')
|
||||
}
|
||||
}
|
||||
|
||||
export function throwBenchmarkTrialFailures(trialError, cleanupError) {
|
||||
if (trialError && cleanupError) {
|
||||
throw new AggregateError([trialError, cleanupError], 'Electron trial and cleanup failed')
|
||||
}
|
||||
if (trialError) {
|
||||
throw trialError
|
||||
}
|
||||
if (cleanupError) {
|
||||
throw cleanupError
|
||||
}
|
||||
}
|
||||
|
||||
export function parseBenchmarkTrialResult(serializedResult) {
|
||||
return JSON.parse(serializedResult)
|
||||
}
|
||||
|
||||
export function benchmarkTrialNeedsCleanup(spawnResult, parsedResultAvailable) {
|
||||
return spawnResult?.status !== 0 || !parsedResultAvailable
|
||||
}
|
||||
|
||||
const processGroupSignalOperations = {
|
||||
processIdentities,
|
||||
signalProcess: process.kill.bind(process)
|
||||
}
|
||||
|
||||
function compensateStoppedGroup(pgid, groupState, operations) {
|
||||
const errors = []
|
||||
const targets = [
|
||||
groupState.stopped ? [-pgid, 'stopped'] : null,
|
||||
groupState.anchorPid ? [groupState.anchorPid, 'anchorPid'] : null
|
||||
].filter(Boolean)
|
||||
for (const [pid, stateKey] of targets) {
|
||||
try {
|
||||
operations.signalProcess(pid, 'SIGCONT')
|
||||
groupState[stateKey] = stateKey === 'stopped' ? false : null
|
||||
} catch (error) {
|
||||
if (error?.code === 'ESRCH') {
|
||||
groupState[stateKey] = stateKey === 'stopped' ? false : null
|
||||
} else {
|
||||
errors.push(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors
|
||||
}
|
||||
|
||||
export function signalValidatedProcessGroup(
|
||||
pgid,
|
||||
environmentFragment,
|
||||
signal,
|
||||
groupState = { stopped: false, anchorPid: null },
|
||||
operations = processGroupSignalOperations
|
||||
) {
|
||||
if (!Number.isInteger(pgid) || pgid <= 0) {
|
||||
return false
|
||||
}
|
||||
let members
|
||||
try {
|
||||
members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid)
|
||||
} catch (error) {
|
||||
const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations)
|
||||
if (recoveryErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[error, ...recoveryErrors],
|
||||
'Benchmark process group recovery failed before validation'
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (members.length === 0) {
|
||||
const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations)
|
||||
if (recoveryErrors.length > 0) {
|
||||
throw new AggregateError(recoveryErrors, 'Benchmark missing process group recovery failed')
|
||||
}
|
||||
return false
|
||||
}
|
||||
if (members.some((identity) => !identity.command.includes(environmentFragment))) {
|
||||
const ownershipError = new Error('Benchmark process group no longer belongs to this trial')
|
||||
const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations)
|
||||
if (recoveryErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[ownershipError, ...recoveryErrors],
|
||||
'Benchmark process group authority recovery failed'
|
||||
)
|
||||
}
|
||||
throw ownershipError
|
||||
}
|
||||
if (groupState.anchorPid) {
|
||||
try {
|
||||
operations.signalProcess(groupState.anchorPid, 'SIGCONT')
|
||||
groupState.anchorPid = null
|
||||
} catch (error) {
|
||||
if (error?.code === 'ESRCH') {
|
||||
groupState.anchorPid = null
|
||||
} else {
|
||||
throw new AggregateError([error], 'Benchmark pending anchor recovery failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
const anchor = members[0]
|
||||
try {
|
||||
operations.signalProcess(anchor.pid, 'SIGSTOP')
|
||||
groupState.anchorPid = anchor.pid
|
||||
const stoppedAnchor = operations
|
||||
.processIdentities(true)
|
||||
.find((identity) => identity.pid === anchor.pid)
|
||||
if (!sameIdentity(stoppedAnchor, anchor)) {
|
||||
throw new Error('Benchmark process group anchor changed before signaling')
|
||||
}
|
||||
operations.signalProcess(-pgid, 'SIGSTOP')
|
||||
groupState.stopped = true
|
||||
groupState.anchorPid = null
|
||||
const stoppedMembers = operations
|
||||
.processIdentities(true)
|
||||
.filter((identity) => identity.pgid === pgid)
|
||||
if (
|
||||
stoppedMembers.length === 0 ||
|
||||
stoppedMembers.some((identity) => !identity.command.includes(environmentFragment))
|
||||
) {
|
||||
throw new Error('Benchmark process group changed before signaling')
|
||||
}
|
||||
if (signal !== 'SIGSTOP') {
|
||||
operations.signalProcess(-pgid, signal)
|
||||
if (signal !== 'SIGKILL') {
|
||||
operations.signalProcess(-pgid, 'SIGCONT')
|
||||
}
|
||||
groupState.stopped = false
|
||||
groupState.anchorPid = null
|
||||
}
|
||||
return true
|
||||
} catch (error) {
|
||||
const recoveryErrors = compensateStoppedGroup(pgid, groupState, operations)
|
||||
if (recoveryErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[error, ...recoveryErrors],
|
||||
'Benchmark process group signal recovery failed'
|
||||
)
|
||||
}
|
||||
if (error.code === 'ESRCH') {
|
||||
return false
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export function writeProcessRecord(recordPath, processIdentity) {
|
||||
const temporaryPath = `${recordPath}.${process.pid}.tmp`
|
||||
writeFileSync(temporaryPath, JSON.stringify(processIdentity))
|
||||
renameSync(temporaryPath, recordPath)
|
||||
}
|
||||
|
||||
export function processIdentityIsCurrent(identity) {
|
||||
return sameIdentity(processIdentity(identity?.pid), identity)
|
||||
}
|
||||
|
||||
const processSignalOperations = {
|
||||
processIdentity,
|
||||
signalProcess: process.kill.bind(process)
|
||||
}
|
||||
|
||||
export function signalProcessIdentity(
|
||||
identity,
|
||||
expectedCommandFragment,
|
||||
signal,
|
||||
operations = processSignalOperations
|
||||
) {
|
||||
validateDetachedIdentity(identity, expectedCommandFragment)
|
||||
const currentIdentity = operations.processIdentity(identity.pid)
|
||||
if (!currentIdentity) {
|
||||
return false
|
||||
}
|
||||
if (!sameIdentity(currentIdentity, identity)) {
|
||||
throw new Error('Recorded benchmark helper PID now belongs to another process')
|
||||
}
|
||||
let stopped = false
|
||||
try {
|
||||
operations.signalProcess(identity.pid, 'SIGSTOP')
|
||||
stopped = true
|
||||
const stoppedIdentity = operations.processIdentity(identity.pid)
|
||||
if (!sameIdentity(stoppedIdentity, identity)) {
|
||||
throw new Error('Recorded benchmark helper PID changed before signaling')
|
||||
}
|
||||
operations.signalProcess(-identity.pgid, signal)
|
||||
if (signal !== 'SIGKILL') {
|
||||
operations.signalProcess(-identity.pgid, 'SIGCONT')
|
||||
}
|
||||
stopped = false
|
||||
return true
|
||||
} catch (error) {
|
||||
let resumeError
|
||||
if (stopped) {
|
||||
try {
|
||||
operations.signalProcess(identity.pid, 'SIGCONT')
|
||||
} catch (caught) {
|
||||
if (caught?.code !== 'ESRCH') {
|
||||
resumeError = caught
|
||||
}
|
||||
}
|
||||
}
|
||||
if (resumeError) {
|
||||
throw new AggregateError([error, resumeError], 'Benchmark helper signal recovery failed')
|
||||
}
|
||||
if (error.code === 'ESRCH') {
|
||||
return false
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export function killRecordedProcess(recordPath, expectedCommandFragment) {
|
||||
if (!existsSync(recordPath)) {
|
||||
return false
|
||||
}
|
||||
const record = JSON.parse(readFileSync(recordPath, 'utf8'))
|
||||
if (!signalProcessIdentity(record, expectedCommandFragment, 'SIGKILL')) {
|
||||
return false
|
||||
}
|
||||
return waitForIdentityExit(record)
|
||||
}
|
||||
|
||||
export function killRecordedAndMatchingProcesses(
|
||||
recordPath,
|
||||
recordedCommandFragment,
|
||||
matchingCommandFragments
|
||||
) {
|
||||
const errors = []
|
||||
try {
|
||||
killRecordedProcess(recordPath, recordedCommandFragment)
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
try {
|
||||
killProcessMatchingCommand(matchingCommandFragments)
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
if (errors.length === 1) {
|
||||
throw errors[0]
|
||||
}
|
||||
if (errors.length > 1) {
|
||||
throw new AggregateError(errors, 'Benchmark helper cleanup failed')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,558 @@
|
||||
import { execFileSync, spawn } from 'node:child_process'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
benchmarkTrialNeedsCleanup,
|
||||
killProcessMatchingCommand,
|
||||
killRecordedAndMatchingProcesses,
|
||||
killRecordedProcess,
|
||||
parseBenchmarkTrialResult,
|
||||
processIdentity,
|
||||
runBenchmarkCleanupStages,
|
||||
signalProcessIdentity,
|
||||
signalValidatedProcessGroup,
|
||||
spawnBenchmarkProcess,
|
||||
throwBenchmarkTrialFailures,
|
||||
writeProcessRecord
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
|
||||
const describeMacOS = process.platform === 'darwin' ? describe : describe.skip
|
||||
const spawnedPids = new Set()
|
||||
const temporaryDirectories = new Set()
|
||||
|
||||
afterEach(() => {
|
||||
for (const pid of spawnedPids) {
|
||||
try {
|
||||
process.kill(pid, 'SIGKILL')
|
||||
} catch {}
|
||||
}
|
||||
spawnedPids.clear()
|
||||
for (const temporaryDirectory of temporaryDirectories) {
|
||||
rmSync(temporaryDirectory, { recursive: true, force: true })
|
||||
}
|
||||
temporaryDirectories.clear()
|
||||
})
|
||||
|
||||
describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
it('enforces a hard timeout when the trial ignores SIGTERM', () => {
|
||||
const startedAt = Date.now()
|
||||
const result = spawnBenchmarkProcess(
|
||||
process.execPath,
|
||||
['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1_000)"],
|
||||
{ stdio: 'ignore', timeout: 100 }
|
||||
)
|
||||
expect(result.error?.code).toBe('ETIMEDOUT')
|
||||
expect(result.signal).toBe('SIGKILL')
|
||||
expect(Date.now() - startedAt).toBeLessThan(2_000)
|
||||
expect(() => process.kill(result.pid, 0)).toThrow()
|
||||
})
|
||||
|
||||
it('runs every cleanup stage before aggregating errors', () => {
|
||||
const completed = []
|
||||
let thrown
|
||||
|
||||
try {
|
||||
runBenchmarkCleanupStages([
|
||||
() => {
|
||||
completed.push(1)
|
||||
throw new Error('first failure')
|
||||
},
|
||||
() => {
|
||||
completed.push(2)
|
||||
},
|
||||
() => {
|
||||
completed.push(3)
|
||||
throw new Error('last failure')
|
||||
}
|
||||
])
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(completed).toEqual([1, 2, 3])
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual(['first failure', 'last failure'])
|
||||
})
|
||||
|
||||
it('preserves malformed-result and cleanup failures', () => {
|
||||
let trialError
|
||||
try {
|
||||
parseBenchmarkTrialResult('{malformed')
|
||||
} catch (error) {
|
||||
trialError = error
|
||||
}
|
||||
const cleanupError = new Error('cleanup failed')
|
||||
let thrown
|
||||
|
||||
try {
|
||||
throwBenchmarkTrialFailures(trialError, cleanupError)
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(trialError).toBeInstanceOf(SyntaxError)
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors).toEqual([trialError, cleanupError])
|
||||
})
|
||||
|
||||
it('cleans up a status-zero trial whose result could not be parsed', () => {
|
||||
expect(benchmarkTrialNeedsCleanup(undefined, false)).toBe(true)
|
||||
expect(benchmarkTrialNeedsCleanup({ status: 0 }, false)).toBe(true)
|
||||
expect(benchmarkTrialNeedsCleanup({ status: 0 }, true)).toBe(false)
|
||||
expect(benchmarkTrialNeedsCleanup({ status: 1 }, true)).toBe(true)
|
||||
})
|
||||
|
||||
it('removes a launcher directory after partial trial setup', () => {
|
||||
const launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-owner-partial-setup-test-'))
|
||||
temporaryDirectories.add(launcherDir)
|
||||
|
||||
const cleanup = cleanupOwnerLossTrial({
|
||||
failed: true,
|
||||
launcherDir,
|
||||
outputPaths: []
|
||||
})
|
||||
|
||||
expect(cleanup.error).toBeUndefined()
|
||||
expect(existsSync(launcherDir)).toBe(false)
|
||||
temporaryDirectories.delete(launcherDir)
|
||||
})
|
||||
|
||||
it('kills a timed-out trial group only after validating its environment', async () => {
|
||||
const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'orca-owner-benchmark-group-test-'))
|
||||
temporaryDirectories.add(temporaryDirectory)
|
||||
const childPidPath = path.join(temporaryDirectory, 'child.pid')
|
||||
const environmentName = `ORCA_OWNER_GROUP_${process.pid}`
|
||||
const environmentValue = `${Date.now()}`
|
||||
const fixture = `
|
||||
const { spawn } = require('node:child_process')
|
||||
const { writeFileSync } = require('node:fs')
|
||||
const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid))
|
||||
setInterval(() => {}, 1000)
|
||||
`
|
||||
const result = spawnBenchmarkProcess(process.execPath, ['-e', fixture], {
|
||||
env: { ...process.env, [environmentName]: environmentValue },
|
||||
stdio: 'ignore',
|
||||
timeout: 100
|
||||
})
|
||||
const childPid = Number(readFileSync(childPidPath, 'utf8'))
|
||||
spawnedPids.add(childPid)
|
||||
const environmentFragment = `${environmentName}=${environmentValue}`
|
||||
const groupState = { stopped: false }
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(result.pid, `${environmentName}=wrong`, 'SIGSTOP')
|
||||
).toThrow('Benchmark process group no longer belongs to this trial')
|
||||
expect(() => process.kill(childPid, 0)).not.toThrow()
|
||||
expect(
|
||||
signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP', groupState)
|
||||
).toBe(true)
|
||||
expect(
|
||||
signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL', groupState)
|
||||
).toBe(true)
|
||||
await expect
|
||||
.poll(() => {
|
||||
try {
|
||||
process.kill(childPid, 0)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
})
|
||||
.toBe(false)
|
||||
|
||||
spawnedPids.delete(childPid)
|
||||
})
|
||||
|
||||
it('resumes the group after post-stop revalidation fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [
|
||||
{ pid: 41, pgid: 41, command: `/launcher ${marker}` },
|
||||
{ pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
]
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: false },
|
||||
{
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 3) {
|
||||
throw new Error('transient group inspection failure')
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
)
|
||||
).toThrow('transient group inspection failure')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[-41, 'SIGSTOP'],
|
||||
[-41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('compensates a possible stop after group anchor replacement', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` }
|
||||
const replacement = { pid: 41, pgid: 99, command: '/unrelated' }
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: false },
|
||||
{
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 1 ? [anchor] : [replacement]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
)
|
||||
).toThrow('Benchmark process group anchor changed before signaling')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group when final inspection fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }]
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
const groupState = { stopped: false }
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 4) {
|
||||
throw new Error('transient final inspection failure')
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('transient final inspection failure')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[-41, 'SIGSTOP'],
|
||||
[-41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group when final anchor stop fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [
|
||||
{ pid: 41, pgid: 41, command: `/launcher ${marker}` },
|
||||
{ pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
]
|
||||
const signals = []
|
||||
let finalCall = false
|
||||
const groupState = { stopped: false }
|
||||
const missingProcessError = Object.assign(new Error('anchor exited'), { code: 'ESRCH' })
|
||||
const operations = {
|
||||
processIdentities: () => members,
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
if (finalCall && pid === 41 && signal === 'SIGSTOP') {
|
||||
throw missingProcessError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
finalCall = true
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(false)
|
||||
expect(signals.at(-1)).toEqual([-41, 'SIGCONT'])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group after final anchor replacement', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` }
|
||||
const child = { pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
const replacement = { pid: 41, pgid: 99, command: '/unrelated' }
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
const groupState = { stopped: false }
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 5 ? [replacement, child] : [anchor, child]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('Benchmark process group anchor changed before signaling')
|
||||
expect(signals.slice(-2)).toEqual([
|
||||
[-41, 'SIGCONT'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('kills a recorded helper in a separate process group', async () => {
|
||||
const temporaryDirectory = mkdtempSync(
|
||||
path.join(tmpdir(), 'orca-owner-benchmark-cleanup-test-')
|
||||
)
|
||||
temporaryDirectories.add(temporaryDirectory)
|
||||
const recordPath = path.join(temporaryDirectory, 'helper.json')
|
||||
const marker = `orca-owner-cleanup-${process.pid}-${Date.now()}`
|
||||
const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
spawnedPids.add(helper.pid)
|
||||
helper.unref()
|
||||
const exited = new Promise((resolve) => helper.once('exit', resolve))
|
||||
const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
const processGroup = Number(
|
||||
execFileSync('ps', ['-p', String(helper.pid), '-o', 'pgid='], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
)
|
||||
writeProcessRecord(recordPath, { pid: helper.pid, pgid: processGroup, command })
|
||||
|
||||
expect(processGroup).toBe(helper.pid)
|
||||
expect(killRecordedProcess(recordPath, marker)).toBe(true)
|
||||
await exited
|
||||
expect(() => process.kill(helper.pid, 0)).toThrow()
|
||||
|
||||
spawnedPids.delete(helper.pid)
|
||||
})
|
||||
|
||||
it('kills every unrecorded helper using its unique trial command', async () => {
|
||||
const marker = `orca-owner-unrecorded-${process.pid}-${Date.now()}`
|
||||
const helpers = Array.from({ length: 2 }, () =>
|
||||
spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
)
|
||||
for (const helper of helpers) {
|
||||
spawnedPids.add(helper.pid)
|
||||
helper.unref()
|
||||
}
|
||||
const exited = Promise.all(
|
||||
helpers.map((helper) => new Promise((resolve) => helper.once('exit', resolve)))
|
||||
)
|
||||
|
||||
expect(killProcessMatchingCommand([process.execPath, marker])).toBe(true)
|
||||
await exited
|
||||
for (const helper of helpers) {
|
||||
expect(() => process.kill(helper.pid, 0)).toThrow()
|
||||
spawnedPids.delete(helper.pid)
|
||||
}
|
||||
})
|
||||
|
||||
it('continues exact-match cleanup after an earlier match fails', () => {
|
||||
const marker = `orca-owner-multiple-${process.pid}-${Date.now()}`
|
||||
const matches = [
|
||||
{ pid: 41, pgid: 41, command: `/helper ${marker}` },
|
||||
{ pid: 42, pgid: 42, command: `/helper ${marker}` }
|
||||
]
|
||||
const attempted = []
|
||||
let scanCount = 0
|
||||
|
||||
expect(() =>
|
||||
killProcessMatchingCommand(['/helper', marker], {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 1 ? matches : [matches[0]]
|
||||
},
|
||||
signalProcessIdentity: (identity) => {
|
||||
attempted.push(identity.pid)
|
||||
if (identity.pid === matches[0].pid) {
|
||||
throw new Error('identity changed')
|
||||
}
|
||||
return true
|
||||
},
|
||||
waitForIdentityExit: () => {}
|
||||
})
|
||||
).toThrow('Benchmark exact-command cleanup failed')
|
||||
expect(attempted).toEqual([41, 42])
|
||||
})
|
||||
|
||||
it('does not treat an identity query failure as process exit', () => {
|
||||
const queryError = new Error('transient ps failure')
|
||||
|
||||
expect(() =>
|
||||
processIdentity(41, {
|
||||
executePs: () => {
|
||||
throw queryError
|
||||
},
|
||||
signalProcess: () => {}
|
||||
})
|
||||
).toThrow(queryError)
|
||||
})
|
||||
|
||||
it('resumes a helper when post-stop identity inspection fails', () => {
|
||||
const identity = { pid: 41, pgid: 41, command: '/helper marker' }
|
||||
const signals = []
|
||||
let inspectionCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalProcessIdentity(identity, 'marker', 'SIGKILL', {
|
||||
processIdentity: () => {
|
||||
inspectionCount += 1
|
||||
if (inspectionCount === 2) {
|
||||
throw new Error('transient ps failure after stop')
|
||||
}
|
||||
return identity
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
})
|
||||
).toThrow('transient ps failure after stop')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('compensates a possible stop after helper PID replacement', () => {
|
||||
const identity = { pid: 41, pgid: 41, command: '/helper marker' }
|
||||
const replacement = { pid: 41, pgid: 41, command: '/unrelated' }
|
||||
const signals = []
|
||||
let inspectionCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalProcessIdentity(identity, 'marker', 'SIGKILL', {
|
||||
processIdentity: () => {
|
||||
inspectionCount += 1
|
||||
return inspectionCount === 1 ? identity : replacement
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
})
|
||||
).toThrow('Recorded benchmark helper PID changed before signaling')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves helper identity errors when compensation fails', () => {
|
||||
const identity = { pid: 41, pgid: 41, command: '/helper marker' }
|
||||
const replacement = { pid: 41, pgid: 41, command: '/unrelated' }
|
||||
let inspectionCount = 0
|
||||
let thrown
|
||||
|
||||
try {
|
||||
signalProcessIdentity(identity, 'marker', 'SIGKILL', {
|
||||
processIdentity: () => {
|
||||
inspectionCount += 1
|
||||
return inspectionCount === 1 ? identity : replacement
|
||||
},
|
||||
signalProcess: (_pid, signal) => {
|
||||
if (signal === 'SIGCONT') {
|
||||
throw new Error('resume denied')
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual([
|
||||
'Recorded benchmark helper PID changed before signaling',
|
||||
'resume denied'
|
||||
])
|
||||
})
|
||||
|
||||
it('treats a missing PID as process exit after an identity query failure', () => {
|
||||
const missingProcessError = Object.assign(new Error('missing process'), { code: 'ESRCH' })
|
||||
|
||||
expect(
|
||||
processIdentity(41, {
|
||||
executePs: () => {
|
||||
throw new Error('ps found no process')
|
||||
},
|
||||
signalProcess: () => {
|
||||
throw missingProcessError
|
||||
}
|
||||
})
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('runs unique-command cleanup after an invalid process record', async () => {
|
||||
const temporaryDirectory = mkdtempSync(
|
||||
path.join(tmpdir(), 'orca-owner-benchmark-fallback-test-')
|
||||
)
|
||||
temporaryDirectories.add(temporaryDirectory)
|
||||
const recordPath = path.join(temporaryDirectory, 'helper.json')
|
||||
const marker = `orca-owner-invalid-record-${process.pid}-${Date.now()}`
|
||||
const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
spawnedPids.add(helper.pid)
|
||||
helper.unref()
|
||||
const exited = new Promise((resolve) => helper.once('exit', resolve))
|
||||
const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
writeProcessRecord(recordPath, { pid: helper.pid, pgid: helper.pid - 1, command })
|
||||
|
||||
expect(() =>
|
||||
killRecordedAndMatchingProcesses(recordPath, marker, [process.execPath, marker])
|
||||
).toThrow('Recorded benchmark helper identity is invalid')
|
||||
await exited
|
||||
expect(() => process.kill(helper.pid, 0)).toThrow()
|
||||
|
||||
spawnedPids.delete(helper.pid)
|
||||
})
|
||||
|
||||
it('rejects a record that is not a detached process-group identity', () => {
|
||||
const temporaryDirectory = mkdtempSync(
|
||||
path.join(tmpdir(), 'orca-owner-benchmark-identity-test-')
|
||||
)
|
||||
temporaryDirectories.add(temporaryDirectory)
|
||||
const recordPath = path.join(temporaryDirectory, 'helper.json')
|
||||
const marker = `orca-owner-invalid-identity-${process.pid}-${Date.now()}`
|
||||
const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)', marker], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
spawnedPids.add(helper.pid)
|
||||
helper.unref()
|
||||
const command = execFileSync('ps', ['-p', String(helper.pid), '-o', 'command='], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
writeProcessRecord(recordPath, { pid: helper.pid, pgid: helper.pid - 1, command })
|
||||
|
||||
expect(() => killRecordedProcess(recordPath, marker)).toThrow(
|
||||
'Recorded benchmark helper identity is invalid'
|
||||
)
|
||||
expect(() => process.kill(helper.pid, 0)).not.toThrow()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,63 @@
|
||||
import { closeSync, existsSync, readFileSync, rmSync } from 'node:fs'
|
||||
import {
|
||||
killRecordedAndMatchingProcesses,
|
||||
runBenchmarkCleanupStages,
|
||||
signalValidatedProcessGroup
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
|
||||
export function cleanupOwnerLossTrial(options) {
|
||||
const groupState = { stopped: false, anchorPid: null }
|
||||
let error
|
||||
let output = ''
|
||||
try {
|
||||
runBenchmarkCleanupStages([
|
||||
() => {
|
||||
if (options.failed && Number.isInteger(options.pid) && options.marker) {
|
||||
signalValidatedProcessGroup(options.pid, options.marker, 'SIGSTOP', groupState)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.failed && options.recordPath && options.tempDir) {
|
||||
killRecordedAndMatchingProcesses(options.recordPath, options.helperPath, [
|
||||
options.helperPath,
|
||||
options.tempDir
|
||||
])
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.failed && Number.isInteger(options.pid) && options.marker) {
|
||||
signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL', groupState)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.stderrDescriptor !== undefined) {
|
||||
closeSync(options.stderrDescriptor)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.stdoutDescriptor !== undefined) {
|
||||
closeSync(options.stdoutDescriptor)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
output = (options.outputPaths ?? [])
|
||||
.filter((outputPath) => outputPath && existsSync(outputPath))
|
||||
.map((outputPath) => readFileSync(outputPath, 'utf8'))
|
||||
.join('')
|
||||
},
|
||||
() => {
|
||||
if (options.launcherDir) {
|
||||
rmSync(options.launcherDir, { recursive: true, force: true })
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.tempDir) {
|
||||
rmSync(options.tempDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
])
|
||||
} catch (caught) {
|
||||
error = caught
|
||||
}
|
||||
return { error, output }
|
||||
}
|
||||
@@ -2579,9 +2579,12 @@ private enum KeyMap {
|
||||
}
|
||||
|
||||
private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
private static let unclaimedSessionDeadline: TimeInterval = 30
|
||||
|
||||
private let socketPath: String
|
||||
private let token: String?
|
||||
private var listener: SocketListener?
|
||||
private var unclaimedSessionTimeout: DispatchWorkItem?
|
||||
|
||||
init(socketPath: String, token: String?) {
|
||||
self.socketPath = socketPath
|
||||
@@ -2590,9 +2593,29 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
do {
|
||||
let listener = try SocketListener(socketPath: socketPath, token: token)
|
||||
let timeout = DispatchWorkItem {
|
||||
fputs("computer-use agent received no authenticated session before its deadline\n", stderr)
|
||||
NSApp.terminate(nil)
|
||||
}
|
||||
unclaimedSessionTimeout = timeout
|
||||
let listener = try SocketListener(
|
||||
socketPath: socketPath,
|
||||
token: token,
|
||||
onSessionClaimed: {
|
||||
timeout.cancel()
|
||||
},
|
||||
onSessionClosed: {
|
||||
DispatchQueue.main.async {
|
||||
NSApp.terminate(nil)
|
||||
}
|
||||
}
|
||||
)
|
||||
self.listener = listener
|
||||
listener.start()
|
||||
DispatchQueue.main.asyncAfter(
|
||||
deadline: .now() + Self.unclaimedSessionDeadline,
|
||||
execute: timeout
|
||||
)
|
||||
} catch {
|
||||
fputs("failed to start computer-use socket: \(error)\n", stderr)
|
||||
NSApp.terminate(nil)
|
||||
@@ -2600,6 +2623,8 @@ private final class AgentRuntime: NSObject, NSApplicationDelegate {
|
||||
}
|
||||
|
||||
func applicationWillTerminate(_ notification: Notification) {
|
||||
unclaimedSessionTimeout?.cancel()
|
||||
unclaimedSessionTimeout = nil
|
||||
listener?.stop()
|
||||
}
|
||||
}
|
||||
@@ -3505,14 +3530,26 @@ private final class ButtonTarget: NSObject {
|
||||
private final class SocketListener: @unchecked Sendable {
|
||||
private let socketPath: String
|
||||
private let token: String?
|
||||
private let onSessionClaimed: () -> Void
|
||||
private let onSessionClosed: () -> Void
|
||||
private let provider = Provider()
|
||||
private let providerLock = NSLock()
|
||||
private let sessionLock = NSLock()
|
||||
private var sessionOwnership = AgentSessionOwnership()
|
||||
private var lastConnectionID: UInt64 = 0
|
||||
private var socketFd: Int32 = -1
|
||||
private var isStopped = false
|
||||
|
||||
init(socketPath: String, token: String?) throws {
|
||||
init(
|
||||
socketPath: String,
|
||||
token: String?,
|
||||
onSessionClaimed: @escaping () -> Void,
|
||||
onSessionClosed: @escaping () -> Void
|
||||
) throws {
|
||||
self.socketPath = socketPath
|
||||
self.token = token
|
||||
self.onSessionClaimed = onSessionClaimed
|
||||
self.onSessionClosed = onSessionClosed
|
||||
try bindSocket()
|
||||
}
|
||||
|
||||
@@ -3587,14 +3624,35 @@ private final class SocketListener: @unchecked Sendable {
|
||||
}
|
||||
continue
|
||||
}
|
||||
guard let connectionID = allocateConnectionID() else {
|
||||
fputs("computer-use socket exhausted connection identities\n", stderr)
|
||||
close(fd)
|
||||
continue
|
||||
}
|
||||
Thread.detachNewThread { [weak self] in
|
||||
self?.handleConnection(fd)
|
||||
self?.handleConnection(fd, connectionID: connectionID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleConnection(_ fd: Int32) {
|
||||
defer { close(fd) }
|
||||
private func allocateConnectionID() -> AgentSessionConnectionID? {
|
||||
sessionLock.lock()
|
||||
defer { sessionLock.unlock() }
|
||||
guard lastConnectionID < UInt64.max else { return nil }
|
||||
lastConnectionID += 1
|
||||
return AgentSessionConnectionID(rawValue: lastConnectionID)
|
||||
}
|
||||
|
||||
private func handleConnection(_ fd: Int32, connectionID: AgentSessionConnectionID) {
|
||||
var registeredSession = false
|
||||
var hangupMonitor: AuthenticatedConnectionHangupMonitor?
|
||||
defer {
|
||||
hangupMonitor?.cancel()
|
||||
if registeredSession {
|
||||
disconnectSession(connectionID)
|
||||
}
|
||||
close(fd)
|
||||
}
|
||||
let authorizedPeer = peerProcessId(fd).map(isAuthorizedAgentPeer) == true
|
||||
let decoder = JSONDecoder()
|
||||
while let line = readLine(from: fd) {
|
||||
@@ -3603,6 +3661,40 @@ private final class SocketListener: @unchecked Sendable {
|
||||
else {
|
||||
continue
|
||||
}
|
||||
if !registeredSession && isAuthenticatedAgentSession(
|
||||
expectedToken: token,
|
||||
requestToken: request.token,
|
||||
authorizedPeer: authorizedPeer
|
||||
) {
|
||||
let monitor: AuthenticatedConnectionHangupMonitor
|
||||
do {
|
||||
monitor = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: fd,
|
||||
onHangup: { [weak self] in
|
||||
self?.disconnectSession(connectionID)
|
||||
}
|
||||
)
|
||||
} catch {
|
||||
fputs("computer-use owner monitor failed: \(error)\n", stderr)
|
||||
return
|
||||
}
|
||||
sessionLock.lock()
|
||||
let registration = sessionOwnership.registerConnection(
|
||||
connectionID,
|
||||
authenticated: true
|
||||
)
|
||||
sessionLock.unlock()
|
||||
guard registration != .rejected else {
|
||||
monitor.cancel()
|
||||
return
|
||||
}
|
||||
registeredSession = true
|
||||
hangupMonitor = monitor
|
||||
monitor.start()
|
||||
if registration == .claimed {
|
||||
onSessionClaimed()
|
||||
}
|
||||
}
|
||||
let response = handleRequest(
|
||||
provider: provider,
|
||||
lock: providerLock,
|
||||
@@ -3613,6 +3705,15 @@ private final class SocketListener: @unchecked Sendable {
|
||||
writeJSON(response, to: fd)
|
||||
}
|
||||
}
|
||||
|
||||
private func disconnectSession(_ connectionID: AgentSessionConnectionID) {
|
||||
sessionLock.lock()
|
||||
let shouldTerminate = sessionOwnership.disconnect(connectionID)
|
||||
sessionLock.unlock()
|
||||
if shouldTerminate {
|
||||
onSessionClosed()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func existingPathMode(_ path: String) -> mode_t? {
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
public struct AgentSessionConnectionID: Hashable, Sendable {
|
||||
public let rawValue: UInt64
|
||||
|
||||
public init(rawValue: UInt64) {
|
||||
self.rawValue = rawValue
|
||||
}
|
||||
}
|
||||
|
||||
public enum AgentSessionRegistration: Sendable {
|
||||
case rejected
|
||||
case claimed
|
||||
case retained
|
||||
}
|
||||
|
||||
public struct AgentSessionOwnership: Sendable {
|
||||
private var authenticatedConnections: Set<AgentSessionConnectionID> = []
|
||||
private var wasClaimed = false
|
||||
private var sessionClosed = false
|
||||
|
||||
public init() {}
|
||||
|
||||
public mutating func registerConnection(
|
||||
_ connection: AgentSessionConnectionID,
|
||||
authenticated: Bool
|
||||
) -> AgentSessionRegistration {
|
||||
guard authenticated, !sessionClosed else { return .rejected }
|
||||
let inserted = authenticatedConnections.insert(connection).inserted
|
||||
guard inserted else { return .rejected }
|
||||
guard !wasClaimed else { return .retained }
|
||||
wasClaimed = true
|
||||
return .claimed
|
||||
}
|
||||
|
||||
public mutating func disconnect(_ connection: AgentSessionConnectionID) -> Bool {
|
||||
guard authenticatedConnections.remove(connection) != nil else { return false }
|
||||
guard wasClaimed, authenticatedConnections.isEmpty else { return false }
|
||||
sessionClosed = true
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
public func isAuthenticatedAgentSession(
|
||||
expectedToken: String?,
|
||||
requestToken: String?,
|
||||
authorizedPeer: Bool
|
||||
) -> Bool {
|
||||
guard authorizedPeer else { return false }
|
||||
guard let expectedToken else { return true }
|
||||
return requestToken == expectedToken
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
import Darwin
|
||||
import Dispatch
|
||||
import Foundation
|
||||
|
||||
public final class AuthenticatedConnectionHangupMonitor: @unchecked Sendable {
|
||||
typealias RegisterEvents = @Sendable (
|
||||
Int32,
|
||||
UnsafePointer<kevent>?,
|
||||
Int32
|
||||
) -> POSIXErrorCode?
|
||||
typealias CloseDescriptor = @Sendable (Int32) -> Void
|
||||
|
||||
private static let cancelEventIdentifier: UInt = 1
|
||||
|
||||
private let closeDescriptor: CloseDescriptor
|
||||
private let eventQueue: DispatchQueue
|
||||
private let eventQueueDescriptor: Int32
|
||||
private let onEvent: @Sendable () -> Void
|
||||
private let stateLock = NSLock()
|
||||
private let onHangup: @Sendable () -> Void
|
||||
private var isCancelled = false
|
||||
private var didReportHangup = false
|
||||
private var isFinished = false
|
||||
private var isStarted = false
|
||||
|
||||
public convenience init(
|
||||
fileDescriptor: Int32,
|
||||
queue: DispatchQueue = DispatchQueue(
|
||||
label: "com.stablyai.orca.computer-use-owner-hangup"
|
||||
),
|
||||
onEvent: @escaping @Sendable () -> Void = {},
|
||||
onHangup: @escaping @Sendable () -> Void
|
||||
) throws {
|
||||
try self.init(
|
||||
fileDescriptor: fileDescriptor,
|
||||
queue: queue,
|
||||
registerEvents: { descriptor, events, count in
|
||||
guard kevent(descriptor, events, count, nil, 0, nil) == 0 else {
|
||||
return POSIXErrorCode(rawValue: errno) ?? .EIO
|
||||
}
|
||||
return nil
|
||||
},
|
||||
closeDescriptor: { descriptor in
|
||||
close(descriptor)
|
||||
},
|
||||
onEvent: onEvent,
|
||||
onHangup: onHangup
|
||||
)
|
||||
}
|
||||
|
||||
init(
|
||||
fileDescriptor: Int32,
|
||||
queue: DispatchQueue = DispatchQueue(
|
||||
label: "com.stablyai.orca.computer-use-owner-hangup"
|
||||
),
|
||||
registerEvents: RegisterEvents,
|
||||
closeDescriptor: @escaping CloseDescriptor,
|
||||
onEvent: @escaping @Sendable () -> Void = {},
|
||||
onHangup: @escaping @Sendable () -> Void
|
||||
) throws {
|
||||
self.closeDescriptor = closeDescriptor
|
||||
eventQueue = queue
|
||||
self.onEvent = onEvent
|
||||
self.onHangup = onHangup
|
||||
let descriptor = kqueue()
|
||||
guard descriptor >= 0 else {
|
||||
throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)
|
||||
}
|
||||
guard fileDescriptor >= 0 else {
|
||||
closeDescriptor(descriptor)
|
||||
throw POSIXError(.EBADF)
|
||||
}
|
||||
eventQueueDescriptor = descriptor
|
||||
var registrations = [
|
||||
kevent(
|
||||
ident: UInt(fileDescriptor),
|
||||
filter: Int16(EVFILT_READ),
|
||||
flags: UInt16(EV_ADD | EV_CLEAR),
|
||||
fflags: UInt32(NOTE_LOWAT),
|
||||
data: Int.max,
|
||||
udata: nil
|
||||
),
|
||||
kevent(
|
||||
ident: Self.cancelEventIdentifier,
|
||||
filter: Int16(EVFILT_USER),
|
||||
flags: UInt16(EV_ADD | EV_CLEAR),
|
||||
fflags: 0,
|
||||
data: 0,
|
||||
udata: nil
|
||||
)
|
||||
]
|
||||
let registrationError = registrations.withUnsafeMutableBufferPointer { buffer in
|
||||
registerEvents(descriptor, buffer.baseAddress, Int32(buffer.count))
|
||||
}
|
||||
if let registrationError {
|
||||
isFinished = true
|
||||
closeDescriptor(descriptor)
|
||||
throw POSIXError(registrationError)
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
cancel()
|
||||
}
|
||||
|
||||
public func start() {
|
||||
stateLock.lock()
|
||||
guard !isStarted, !isCancelled, !isFinished else {
|
||||
stateLock.unlock()
|
||||
return
|
||||
}
|
||||
isStarted = true
|
||||
stateLock.unlock()
|
||||
eventQueue.async { [self] in
|
||||
waitForHangup()
|
||||
}
|
||||
}
|
||||
|
||||
public func cancel() {
|
||||
stateLock.lock()
|
||||
guard !isFinished else {
|
||||
stateLock.unlock()
|
||||
return
|
||||
}
|
||||
isCancelled = true
|
||||
guard isStarted else {
|
||||
isFinished = true
|
||||
closeDescriptor(eventQueueDescriptor)
|
||||
stateLock.unlock()
|
||||
return
|
||||
}
|
||||
var event = kevent(
|
||||
ident: Self.cancelEventIdentifier,
|
||||
filter: Int16(EVFILT_USER),
|
||||
flags: 0,
|
||||
fflags: UInt32(NOTE_TRIGGER),
|
||||
data: 0,
|
||||
udata: nil
|
||||
)
|
||||
_ = kevent(eventQueueDescriptor, &event, 1, nil, 0, nil)
|
||||
stateLock.unlock()
|
||||
}
|
||||
|
||||
private func waitForHangup() {
|
||||
defer { finish() }
|
||||
while true {
|
||||
var event = kevent()
|
||||
let result = kevent(eventQueueDescriptor, nil, 0, &event, 1, nil)
|
||||
if result < 0 && errno == EINTR {
|
||||
continue
|
||||
}
|
||||
guard result > 0 else { return }
|
||||
onEvent()
|
||||
if event.filter == Int16(EVFILT_USER) {
|
||||
return
|
||||
}
|
||||
if event.flags & UInt16(EV_EOF | EV_ERROR) != 0 {
|
||||
reportHangup()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func finish() {
|
||||
stateLock.lock()
|
||||
isFinished = true
|
||||
closeDescriptor(eventQueueDescriptor)
|
||||
stateLock.unlock()
|
||||
}
|
||||
|
||||
private func reportHangup() {
|
||||
stateLock.lock()
|
||||
guard !isCancelled, !didReportHangup else {
|
||||
stateLock.unlock()
|
||||
return
|
||||
}
|
||||
didReportHangup = true
|
||||
stateLock.unlock()
|
||||
|
||||
onHangup()
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class AgentSessionOwnershipTests: XCTestCase {
|
||||
func testUnclaimedDisconnectDoesNotTerminateAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertFalse(ownership.disconnect(connection(12)))
|
||||
}
|
||||
|
||||
func testUnauthenticatedConnectionCannotClaimOrRetainAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: false), .rejected)
|
||||
XCTAssertFalse(ownership.disconnect(connection(12)))
|
||||
}
|
||||
|
||||
func testLastAuthenticatedDisconnectTerminatesAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed)
|
||||
XCTAssertTrue(ownership.disconnect(connection(12)))
|
||||
}
|
||||
|
||||
func testAgentWaitsForEveryAuthenticatedConnectionToClose() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed)
|
||||
XCTAssertEqual(ownership.registerConnection(connection(13), authenticated: true), .retained)
|
||||
XCTAssertFalse(ownership.disconnect(connection(12)))
|
||||
XCTAssertTrue(ownership.disconnect(connection(13)))
|
||||
}
|
||||
|
||||
func testDuplicateRegistrationDoesNotRetainAgent() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed)
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .rejected)
|
||||
XCTAssertTrue(ownership.disconnect(connection(12)))
|
||||
}
|
||||
|
||||
func testClosedSessionRejectsNewConnections() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(connection(12), authenticated: true), .claimed)
|
||||
XCTAssertTrue(ownership.disconnect(connection(12)))
|
||||
XCTAssertEqual(ownership.registerConnection(connection(13), authenticated: true), .rejected)
|
||||
XCTAssertFalse(ownership.disconnect(connection(13)))
|
||||
}
|
||||
|
||||
func testStaleDisconnectCannotRemoveReusedFileDescriptorOwner() {
|
||||
var ownership = AgentSessionOwnership()
|
||||
let staleConnection = connection(12)
|
||||
let otherConnection = connection(13)
|
||||
let reusedDescriptorConnection = connection(14)
|
||||
|
||||
XCTAssertEqual(ownership.registerConnection(staleConnection, authenticated: true), .claimed)
|
||||
XCTAssertEqual(ownership.registerConnection(otherConnection, authenticated: true), .retained)
|
||||
XCTAssertFalse(ownership.disconnect(staleConnection))
|
||||
XCTAssertEqual(
|
||||
ownership.registerConnection(reusedDescriptorConnection, authenticated: true),
|
||||
.retained
|
||||
)
|
||||
|
||||
XCTAssertFalse(ownership.disconnect(staleConnection))
|
||||
XCTAssertFalse(ownership.disconnect(otherConnection))
|
||||
XCTAssertTrue(ownership.disconnect(reusedDescriptorConnection))
|
||||
}
|
||||
|
||||
func testTokenlessSessionStillRequiresAuthorizedPeer() {
|
||||
XCTAssertFalse(isAuthenticatedAgentSession(
|
||||
expectedToken: nil,
|
||||
requestToken: nil,
|
||||
authorizedPeer: false
|
||||
))
|
||||
XCTAssertTrue(isAuthenticatedAgentSession(
|
||||
expectedToken: nil,
|
||||
requestToken: nil,
|
||||
authorizedPeer: true
|
||||
))
|
||||
}
|
||||
|
||||
func testTokenSessionRequiresAuthorizedPeerAndMatchingToken() {
|
||||
XCTAssertFalse(isAuthenticatedAgentSession(
|
||||
expectedToken: "expected",
|
||||
requestToken: "expected",
|
||||
authorizedPeer: false
|
||||
))
|
||||
XCTAssertFalse(isAuthenticatedAgentSession(
|
||||
expectedToken: "expected",
|
||||
requestToken: "wrong",
|
||||
authorizedPeer: true
|
||||
))
|
||||
XCTAssertTrue(isAuthenticatedAgentSession(
|
||||
expectedToken: "expected",
|
||||
requestToken: "expected",
|
||||
authorizedPeer: true
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
private func connection(_ rawValue: UInt64) -> AgentSessionConnectionID {
|
||||
AgentSessionConnectionID(rawValue: rawValue)
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
import Darwin
|
||||
import Dispatch
|
||||
@testable import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class AuthenticatedConnectionHangupMonitorTests: XCTestCase {
|
||||
func testRejectsInvalidDescriptorDuringSetup() {
|
||||
XCTAssertThrowsError(
|
||||
try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: -1,
|
||||
onHangup: {}
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
func testRegistrationFailureClosesEventQueueOnce() {
|
||||
let closes = DescriptorCloseRecorder()
|
||||
|
||||
XCTAssertThrowsError(
|
||||
try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: 42,
|
||||
registerEvents: { _, _, _ in .EBADF },
|
||||
closeDescriptor: { descriptor in
|
||||
closes.recordAndClose(descriptor)
|
||||
},
|
||||
onHangup: {}
|
||||
)
|
||||
)
|
||||
XCTAssertEqual(closes.count, 1)
|
||||
}
|
||||
|
||||
func testReportsPeerCloseWhileRequestHandlingIsBlocked() throws {
|
||||
let descriptors = try makeSocketPair()
|
||||
let hangup = expectation(description: "peer hangup")
|
||||
let processingStarted = expectation(description: "processing started")
|
||||
let releaseProcessing = DispatchSemaphore(value: 0)
|
||||
let processingFinished = DispatchSemaphore(value: 0)
|
||||
let monitor = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: descriptors.local,
|
||||
onHangup: {
|
||||
hangup.fulfill()
|
||||
}
|
||||
)
|
||||
monitor.start()
|
||||
|
||||
DispatchQueue.global().async {
|
||||
processingStarted.fulfill()
|
||||
releaseProcessing.wait()
|
||||
processingFinished.signal()
|
||||
}
|
||||
wait(for: [processingStarted], timeout: 1)
|
||||
close(descriptors.peer)
|
||||
|
||||
wait(for: [hangup], timeout: 1)
|
||||
XCTAssertEqual(processingFinished.wait(timeout: .now() + 0.05), .timedOut)
|
||||
|
||||
releaseProcessing.signal()
|
||||
XCTAssertEqual(processingFinished.wait(timeout: .now() + 1), .success)
|
||||
monitor.cancel()
|
||||
close(descriptors.local)
|
||||
}
|
||||
|
||||
func testReadableDataDoesNotLookLikeHangup() throws {
|
||||
let descriptors = try makeSocketPair()
|
||||
let hangup = expectation(description: "peer hangup")
|
||||
let callbacks = CallbackRecorder()
|
||||
let events = CallbackRecorder()
|
||||
let monitor = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: descriptors.local,
|
||||
onEvent: {
|
||||
events.record()
|
||||
},
|
||||
onHangup: {
|
||||
callbacks.record()
|
||||
hangup.fulfill()
|
||||
}
|
||||
)
|
||||
monitor.start()
|
||||
var byte: UInt8 = 7
|
||||
|
||||
XCTAssertEqual(write(descriptors.peer, &byte, 1), 1)
|
||||
usleep(50_000)
|
||||
XCTAssertEqual(callbacks.count, 0)
|
||||
XCTAssertEqual(events.count, 0)
|
||||
close(descriptors.peer)
|
||||
wait(for: [hangup], timeout: 1)
|
||||
XCTAssertEqual(events.count, 1)
|
||||
|
||||
monitor.cancel()
|
||||
close(descriptors.local)
|
||||
}
|
||||
|
||||
func testCancelPreventsLaterHangupCallback() throws {
|
||||
let descriptors = try makeSocketPair()
|
||||
let hangup = expectation(description: "peer hangup")
|
||||
hangup.isInverted = true
|
||||
let monitor = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: descriptors.local,
|
||||
onHangup: {
|
||||
hangup.fulfill()
|
||||
}
|
||||
)
|
||||
monitor.start()
|
||||
|
||||
monitor.cancel()
|
||||
close(descriptors.peer)
|
||||
|
||||
wait(for: [hangup], timeout: 0.1)
|
||||
close(descriptors.local)
|
||||
}
|
||||
|
||||
func testCancelReleasesMonitorWhileSocketStaysOpen() throws {
|
||||
let descriptors = try makeSocketPair()
|
||||
var monitor: AuthenticatedConnectionHangupMonitor? = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: descriptors.local,
|
||||
onHangup: {}
|
||||
)
|
||||
weak var retainedMonitor = monitor
|
||||
|
||||
monitor?.cancel()
|
||||
monitor = nil
|
||||
|
||||
let deadline = Date().addingTimeInterval(1)
|
||||
while retainedMonitor != nil, Date() < deadline {
|
||||
usleep(10_000)
|
||||
}
|
||||
XCTAssertNil(retainedMonitor)
|
||||
close(descriptors.peer)
|
||||
close(descriptors.local)
|
||||
}
|
||||
|
||||
func testStartReportsHangupQueuedAfterSuccessfulSetup() throws {
|
||||
let descriptors = try makeSocketPair()
|
||||
let hangup = expectation(description: "queued peer hangup")
|
||||
let monitor = try AuthenticatedConnectionHangupMonitor(
|
||||
fileDescriptor: descriptors.local,
|
||||
onHangup: {
|
||||
hangup.fulfill()
|
||||
}
|
||||
)
|
||||
|
||||
close(descriptors.peer)
|
||||
monitor.start()
|
||||
|
||||
wait(for: [hangup], timeout: 1)
|
||||
monitor.cancel()
|
||||
close(descriptors.local)
|
||||
}
|
||||
}
|
||||
|
||||
private final class CallbackRecorder: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var recordedCount = 0
|
||||
|
||||
var count: Int {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return recordedCount
|
||||
}
|
||||
|
||||
func record() {
|
||||
lock.lock()
|
||||
recordedCount += 1
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
private final class DescriptorCloseRecorder: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var closeCount = 0
|
||||
|
||||
var count: Int {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return closeCount
|
||||
}
|
||||
|
||||
func recordAndClose(_ descriptor: Int32) {
|
||||
lock.lock()
|
||||
closeCount += 1
|
||||
let shouldClose = closeCount == 1
|
||||
lock.unlock()
|
||||
if shouldClose {
|
||||
close(descriptor)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func makeSocketPair() throws -> (local: Int32, peer: Int32) {
|
||||
var descriptors: [Int32] = [0, 0]
|
||||
guard socketpair(AF_UNIX, SOCK_STREAM, 0, &descriptors) == 0 else {
|
||||
throw POSIXError(.EIO)
|
||||
}
|
||||
return (descriptors[0], descriptors[1])
|
||||
}
|
||||
@@ -105,6 +105,7 @@
|
||||
"test:e2e:terminal-ime-native": "node config/scripts/run-terminal-ibus-hangul-e2e.mjs",
|
||||
"test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts",
|
||||
"bench:idle-cpu": "pnpm run ensure:electron-runtime && node config/scripts/run-idle-cpu-benchmark.mjs",
|
||||
"bench:macos-computer-helper-owner-loss": "node config/scripts/macos-computer-helper-owner-loss-benchmark.mjs",
|
||||
"bench:startup": "pnpm run ensure:electron-runtime && node tools/benchmarks/startup-time-bench.mjs",
|
||||
"bench:daemon-coldstart": "pnpm run ensure:electron-runtime && node tools/benchmarks/daemon-coldstart-bench.mjs",
|
||||
"bench:main-thread-jank": "pnpm run ensure:electron-runtime && node tools/benchmarks/main-thread-jank-bench.mjs",
|
||||
|
||||
Reference in New Issue
Block a user