mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 16:03:26 +00:00
test(computer): make owner benchmark cleanup failure-safe
This commit is contained in:
@@ -13,6 +13,7 @@ on:
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-processes.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs'
|
||||
- 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
- 'config/scripts/computer-use-modifier-safety.test.mjs'
|
||||
- 'config/scripts/computer-use-skill-guidance.test.mjs'
|
||||
- 'config/scripts/computer-use-smoke.mjs'
|
||||
|
||||
@@ -139,7 +139,8 @@ describe('computer-use e2e workflow', () => {
|
||||
'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-metrics.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs'
|
||||
'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs',
|
||||
'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
])
|
||||
)
|
||||
})
|
||||
@@ -149,10 +150,17 @@ describe('computer-use e2e workflow', () => {
|
||||
join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'),
|
||||
'utf8'
|
||||
)
|
||||
const cleanup = readFileSync(
|
||||
join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(benchmark).toContain('spawnBenchmarkProcess(executable, [launcherDir]')
|
||||
expect(benchmark).toContain("stdio: ['ignore', stdoutDescriptor, stderrDescriptor]")
|
||||
expect(benchmark).toContain('killRecordedAndMatchingProcesses(helperRecordPath, helperPath')
|
||||
expect(benchmark).toContain('cleanupOwnerLossTrial({')
|
||||
expect(benchmark).toContain('trialCleanupSha256: artifactSha256(trialCleanupPath)')
|
||||
expect(cleanup).toContain('killRecordedAndMatchingProcesses(options.recordPath')
|
||||
expect(cleanup).toContain("signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL'")
|
||||
})
|
||||
|
||||
it('boots the built daemon under plain Node in the PR native-smoke job after the main build', () => {
|
||||
|
||||
@@ -1,15 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from 'node:crypto'
|
||||
import { execFileSync, fork } from 'node:child_process'
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import net from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
@@ -21,13 +13,13 @@ import {
|
||||
sampleProcess
|
||||
} from './macos-computer-helper-owner-loss-metrics.mjs'
|
||||
import {
|
||||
killRecordedAndMatchingProcesses,
|
||||
processIdentityIsCurrent,
|
||||
signalProcessIdentity,
|
||||
signalValidatedProcessGroup,
|
||||
spawnBenchmarkProcess,
|
||||
throwBenchmarkTrialFailures,
|
||||
writeProcessRecord
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
|
||||
const INTERNAL_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_INTERNAL'
|
||||
const EXPECTATION_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_EXPECTATION'
|
||||
@@ -47,6 +39,10 @@ const processCleanupPath = path.join(
|
||||
import.meta.dirname,
|
||||
'macos-computer-helper-owner-loss-processes.mjs'
|
||||
)
|
||||
const trialCleanupPath = path.join(
|
||||
import.meta.dirname,
|
||||
'macos-computer-helper-owner-loss-trial-cleanup.mjs'
|
||||
)
|
||||
const sidecarPath = path.join(repoRoot, 'out', 'main', 'computer-sidecar.js')
|
||||
const helperAppPath = path.join(
|
||||
repoRoot,
|
||||
@@ -457,6 +453,8 @@ function runTrial(executable, expectation) {
|
||||
)
|
||||
let result
|
||||
let serializedResult
|
||||
let trialError
|
||||
let cleanupError
|
||||
let stderrDescriptor
|
||||
let stdoutDescriptor
|
||||
let trialOutput = ''
|
||||
@@ -481,48 +479,35 @@ function runTrial(executable, expectation) {
|
||||
if (result.status === 0 && existsSync(resultPath)) {
|
||||
serializedResult = readFileSync(resultPath, 'utf8')
|
||||
}
|
||||
} catch (error) {
|
||||
trialError = error
|
||||
} finally {
|
||||
const failedTrial = result?.status !== 0 || !serializedResult
|
||||
const trialEnvironmentFragment = `TMPDIR=${trialTempDir}`
|
||||
try {
|
||||
if (failedTrial) {
|
||||
signalValidatedProcessGroup(result?.pid, trialEnvironmentFragment, 'SIGSTOP')
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
if (failedTrial) {
|
||||
killRecordedAndMatchingProcesses(helperRecordPath, helperPath, [helperPath, trialTempDir])
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
if (failedTrial) {
|
||||
signalValidatedProcessGroup(result?.pid, trialEnvironmentFragment, 'SIGKILL')
|
||||
}
|
||||
} finally {
|
||||
if (stderrDescriptor !== undefined) {
|
||||
closeSync(stderrDescriptor)
|
||||
}
|
||||
if (stdoutDescriptor !== undefined) {
|
||||
closeSync(stdoutDescriptor)
|
||||
}
|
||||
trialOutput = [stderrPath, stdoutPath]
|
||||
.filter(existsSync)
|
||||
.map((outputPath) => readFileSync(outputPath, 'utf8'))
|
||||
.join('')
|
||||
rmSync(launcherDir, { recursive: true, force: true })
|
||||
rmSync(trialTempDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
const trialMarker = `TMPDIR=${trialTempDir}`
|
||||
const cleanup = cleanupOwnerLossTrial({
|
||||
failed: failedTrial,
|
||||
pid: result?.pid,
|
||||
marker: trialMarker,
|
||||
recordPath: helperRecordPath,
|
||||
helperPath,
|
||||
tempDir: trialTempDir,
|
||||
stderrDescriptor,
|
||||
stdoutDescriptor,
|
||||
outputPaths: [stderrPath, stdoutPath],
|
||||
launcherDir
|
||||
})
|
||||
cleanupError = cleanup.error
|
||||
trialOutput = cleanup.output
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
if (!trialError && result?.status !== 0) {
|
||||
trialError = new Error(
|
||||
`Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n${trialOutput}`
|
||||
)
|
||||
}
|
||||
if (!serializedResult) {
|
||||
throw new Error(`Electron trial did not write a result:\n${trialOutput}`)
|
||||
if (!trialError && !serializedResult) {
|
||||
trialError = new Error(`Electron trial did not write a result:\n${trialOutput}`)
|
||||
}
|
||||
throwBenchmarkTrialFailures(trialError, cleanupError)
|
||||
return JSON.parse(serializedResult)
|
||||
}
|
||||
|
||||
@@ -565,7 +550,8 @@ function runBenchmark() {
|
||||
sources: {
|
||||
benchmarkSha256: artifactSha256(scriptPath),
|
||||
metricsSha256: artifactSha256(metricsPath),
|
||||
processCleanupSha256: artifactSha256(processCleanupPath)
|
||||
processCleanupSha256: artifactSha256(processCleanupPath),
|
||||
trialCleanupSha256: artifactSha256(trialCleanupPath)
|
||||
},
|
||||
expectation: options.expect,
|
||||
trials: options.trials,
|
||||
|
||||
@@ -156,6 +156,35 @@ export function spawnBenchmarkProcess(executable, args, options) {
|
||||
})
|
||||
}
|
||||
|
||||
export function runBenchmarkCleanupStages(stages) {
|
||||
const errors = []
|
||||
for (const stage of stages) {
|
||||
try {
|
||||
stage()
|
||||
} catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
}
|
||||
if (errors.length === 1) {
|
||||
throw errors[0]
|
||||
}
|
||||
if (errors.length > 1) {
|
||||
throw new AggregateError(errors, 'Benchmark trial cleanup failed')
|
||||
}
|
||||
}
|
||||
|
||||
export function throwBenchmarkTrialFailures(trialError, cleanupError) {
|
||||
if (trialError && cleanupError) {
|
||||
throw new AggregateError([trialError, cleanupError], 'Electron trial and cleanup failed')
|
||||
}
|
||||
if (trialError) {
|
||||
throw trialError
|
||||
}
|
||||
if (cleanupError) {
|
||||
throw cleanupError
|
||||
}
|
||||
}
|
||||
|
||||
const processGroupSignalOperations = {
|
||||
processIdentities,
|
||||
signalProcess: process.kill.bind(process)
|
||||
@@ -165,31 +194,72 @@ export function signalValidatedProcessGroup(
|
||||
pgid,
|
||||
environmentFragment,
|
||||
signal,
|
||||
groupState = { stopped: false },
|
||||
operations = processGroupSignalOperations
|
||||
) {
|
||||
if (!Number.isInteger(pgid) || pgid <= 0) {
|
||||
return false
|
||||
}
|
||||
const members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid)
|
||||
let members
|
||||
try {
|
||||
members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid)
|
||||
} catch (error) {
|
||||
if (!groupState.stopped) {
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
operations.signalProcess(-pgid, 'SIGCONT')
|
||||
groupState.stopped = false
|
||||
} catch (resumeError) {
|
||||
if (resumeError?.code === 'ESRCH') {
|
||||
groupState.stopped = false
|
||||
} else {
|
||||
throw new AggregateError(
|
||||
[error, resumeError],
|
||||
'Benchmark process group recovery failed before validation'
|
||||
)
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (members.length === 0) {
|
||||
return false
|
||||
}
|
||||
if (members.some((identity) => !identity.command.includes(environmentFragment))) {
|
||||
throw new Error('Benchmark process group no longer belongs to this trial')
|
||||
const ownershipError = new Error('Benchmark process group no longer belongs to this trial')
|
||||
if (groupState.stopped) {
|
||||
try {
|
||||
operations.signalProcess(-pgid, 'SIGCONT')
|
||||
groupState.stopped = false
|
||||
} catch (resumeError) {
|
||||
if (resumeError?.code === 'ESRCH') {
|
||||
groupState.stopped = false
|
||||
} else {
|
||||
throw new AggregateError(
|
||||
[ownershipError, resumeError],
|
||||
'Benchmark process group authority recovery failed'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
throw ownershipError
|
||||
}
|
||||
const anchor = members[0]
|
||||
let stopped = false
|
||||
let anchorNeedsResume = false
|
||||
let groupNeedsResume = groupState.stopped
|
||||
try {
|
||||
operations.signalProcess(anchor.pid, 'SIGSTOP')
|
||||
stopped = true
|
||||
anchorNeedsResume = true
|
||||
const stoppedAnchor = operations
|
||||
.processIdentities(true)
|
||||
.find((identity) => identity.pid === anchor.pid)
|
||||
if (!sameIdentity(stoppedAnchor, anchor)) {
|
||||
stopped = false
|
||||
throw new Error('Benchmark process group anchor changed before signaling')
|
||||
}
|
||||
operations.signalProcess(-pgid, 'SIGSTOP')
|
||||
groupNeedsResume = true
|
||||
groupState.stopped = true
|
||||
anchorNeedsResume = false
|
||||
const stoppedMembers = operations
|
||||
.processIdentities(true)
|
||||
.filter((identity) => identity.pgid === pgid)
|
||||
@@ -204,29 +274,31 @@ export function signalValidatedProcessGroup(
|
||||
if (signal !== 'SIGKILL') {
|
||||
operations.signalProcess(-pgid, 'SIGCONT')
|
||||
}
|
||||
stopped = false
|
||||
groupNeedsResume = false
|
||||
groupState.stopped = false
|
||||
}
|
||||
return true
|
||||
} catch (error) {
|
||||
const recoveryErrors = []
|
||||
if (stopped) {
|
||||
if (groupNeedsResume) {
|
||||
try {
|
||||
const recoveryMembers = operations
|
||||
.processIdentities(true)
|
||||
.filter(
|
||||
(identity) => identity.pgid === pgid && identity.command.includes(environmentFragment)
|
||||
)
|
||||
for (const member of recoveryMembers) {
|
||||
try {
|
||||
operations.signalProcess(member.pid, 'SIGCONT')
|
||||
} catch (caught) {
|
||||
if (caught?.code !== 'ESRCH') {
|
||||
recoveryErrors.push(caught)
|
||||
}
|
||||
}
|
||||
}
|
||||
operations.signalProcess(-pgid, 'SIGCONT')
|
||||
groupState.stopped = false
|
||||
} catch (caught) {
|
||||
recoveryErrors.push(caught)
|
||||
if (caught?.code === 'ESRCH') {
|
||||
groupState.stopped = false
|
||||
} else {
|
||||
recoveryErrors.push(caught)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (anchorNeedsResume) {
|
||||
try {
|
||||
operations.signalProcess(anchor.pid, 'SIGCONT')
|
||||
} catch (caught) {
|
||||
if (caught?.code !== 'ESRCH') {
|
||||
recoveryErrors.push(caught)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (recoveryErrors.length > 0) {
|
||||
@@ -277,7 +349,6 @@ export function signalProcessIdentity(
|
||||
stopped = true
|
||||
const stoppedIdentity = operations.processIdentity(identity.pid)
|
||||
if (!sameIdentity(stoppedIdentity, identity)) {
|
||||
stopped = false
|
||||
throw new Error('Recorded benchmark helper PID changed before signaling')
|
||||
}
|
||||
operations.signalProcess(-identity.pgid, signal)
|
||||
@@ -290,10 +361,7 @@ export function signalProcessIdentity(
|
||||
let resumeError
|
||||
if (stopped) {
|
||||
try {
|
||||
const recoveryIdentity = operations.processIdentity(identity.pid)
|
||||
if (sameIdentity(recoveryIdentity, identity)) {
|
||||
operations.signalProcess(identity.pid, 'SIGCONT')
|
||||
}
|
||||
operations.signalProcess(identity.pid, 'SIGCONT')
|
||||
} catch (caught) {
|
||||
if (caught?.code !== 'ESRCH') {
|
||||
resumeError = caught
|
||||
|
||||
@@ -8,9 +8,11 @@ import {
|
||||
killRecordedAndMatchingProcesses,
|
||||
killRecordedProcess,
|
||||
processIdentity,
|
||||
runBenchmarkCleanupStages,
|
||||
signalProcessIdentity,
|
||||
signalValidatedProcessGroup,
|
||||
spawnBenchmarkProcess,
|
||||
throwBenchmarkTrialFailures,
|
||||
writeProcessRecord
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
|
||||
@@ -45,6 +47,46 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
expect(() => process.kill(result.pid, 0)).toThrow()
|
||||
})
|
||||
|
||||
it('runs every cleanup stage before aggregating errors', () => {
|
||||
const completed = []
|
||||
let thrown
|
||||
|
||||
try {
|
||||
runBenchmarkCleanupStages([
|
||||
() => {
|
||||
completed.push(1)
|
||||
throw new Error('first failure')
|
||||
},
|
||||
() => {
|
||||
completed.push(2)
|
||||
},
|
||||
() => {
|
||||
completed.push(3)
|
||||
throw new Error('last failure')
|
||||
}
|
||||
])
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(completed).toEqual([1, 2, 3])
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual(['first failure', 'last failure'])
|
||||
})
|
||||
|
||||
it('preserves both trial and cleanup failures', () => {
|
||||
const trialError = new Error('trial failed with captured output')
|
||||
const cleanupError = new Error('cleanup failed')
|
||||
let thrown
|
||||
|
||||
try {
|
||||
throwBenchmarkTrialFailures(trialError, cleanupError)
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors).toEqual([trialError, cleanupError])
|
||||
})
|
||||
|
||||
it('kills a timed-out trial group only after validating its environment', async () => {
|
||||
const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'orca-owner-benchmark-group-test-'))
|
||||
temporaryDirectories.add(temporaryDirectory)
|
||||
@@ -68,13 +110,18 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
const childPid = Number(readFileSync(childPidPath, 'utf8'))
|
||||
spawnedPids.add(childPid)
|
||||
const environmentFragment = `${environmentName}=${environmentValue}`
|
||||
const groupState = { stopped: false }
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(result.pid, `${environmentName}=wrong`, 'SIGSTOP')
|
||||
).toThrow('Benchmark process group no longer belongs to this trial')
|
||||
expect(() => process.kill(childPid, 0)).not.toThrow()
|
||||
expect(signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP')).toBe(true)
|
||||
expect(signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL')).toBe(true)
|
||||
expect(
|
||||
signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP', groupState)
|
||||
).toBe(true)
|
||||
expect(
|
||||
signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL', groupState)
|
||||
).toBe(true)
|
||||
await expect
|
||||
.poll(() => {
|
||||
try {
|
||||
@@ -89,7 +136,7 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
spawnedPids.delete(childPid)
|
||||
})
|
||||
|
||||
it('resumes marked group members after revalidation fails', () => {
|
||||
it('resumes the group after post-stop revalidation fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [
|
||||
{ pid: 41, pgid: 41, command: `/launcher ${marker}` },
|
||||
@@ -99,46 +146,206 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
let scanCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 3) {
|
||||
throw new Error('transient group inspection failure')
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: false },
|
||||
{
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 3) {
|
||||
throw new Error('transient group inspection failure')
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
})
|
||||
)
|
||||
).toThrow('transient group inspection failure')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[-41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT'],
|
||||
[42, 'SIGCONT']
|
||||
[-41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('does not signal a replacement group anchor', () => {
|
||||
it('compensates a possible stop after group anchor replacement', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` }
|
||||
const replacement = { pid: 41, pgid: 41, command: '/unrelated' }
|
||||
const replacement = { pid: 41, pgid: 99, command: '/unrelated' }
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 1 ? [anchor] : [replacement]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: false },
|
||||
{
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 1 ? [anchor] : [replacement]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
).toThrow('Benchmark process group anchor changed before signaling')
|
||||
expect(signals).toEqual([[41, 'SIGSTOP']])
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group when final inspection fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }]
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
const groupState = { stopped: false }
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount === 4) {
|
||||
throw new Error('transient final inspection failure')
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('transient final inspection failure')
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[-41, 'SIGSTOP'],
|
||||
[-41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group when final anchor stop fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [
|
||||
{ pid: 41, pgid: 41, command: `/launcher ${marker}` },
|
||||
{ pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
]
|
||||
const signals = []
|
||||
let finalCall = false
|
||||
const groupState = { stopped: false }
|
||||
const missingProcessError = Object.assign(new Error('anchor exited'), { code: 'ESRCH' })
|
||||
const operations = {
|
||||
processIdentities: () => members,
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
if (finalCall && pid === 41 && signal === 'SIGSTOP') {
|
||||
throw missingProcessError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
finalCall = true
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(false)
|
||||
expect(signals.at(-1)).toEqual([-41, 'SIGCONT'])
|
||||
})
|
||||
|
||||
it('resumes a previously frozen group after final anchor replacement', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` }
|
||||
const child = { pid: 42, pgid: 41, command: `/child ${marker}` }
|
||||
const replacement = { pid: 41, pgid: 99, command: '/unrelated' }
|
||||
const signals = []
|
||||
let scanCount = 0
|
||||
const groupState = { stopped: false }
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
return scanCount === 5 ? [replacement, child] : [anchor, child]
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
signals.push([pid, signal])
|
||||
}
|
||||
}
|
||||
|
||||
expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('Benchmark process group anchor changed before signaling')
|
||||
expect(signals.slice(-2)).toEqual([
|
||||
[-41, 'SIGCONT'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('retains uncertain stop state across cleanup stage failures', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }]
|
||||
const groupState = { stopped: false }
|
||||
let scanCount = 0
|
||||
let continueAttempts = 0
|
||||
const operations = {
|
||||
processIdentities: () => {
|
||||
scanCount += 1
|
||||
if (scanCount >= 3) {
|
||||
throw new Error(
|
||||
scanCount === 3 ? 'post-stop inspection failed' : 'final inspection failed'
|
||||
)
|
||||
}
|
||||
return members
|
||||
},
|
||||
signalProcess: (pid, signal) => {
|
||||
if (pid === -41 && signal === 'SIGCONT') {
|
||||
continueAttempts += 1
|
||||
if (continueAttempts === 1) {
|
||||
throw new Error('first compensation failed')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)
|
||||
).toThrow('Benchmark process group signal recovery failed')
|
||||
expect(groupState.stopped).toBe(true)
|
||||
expect(() =>
|
||||
signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)
|
||||
).toThrow('final inspection failed')
|
||||
expect(continueAttempts).toBe(2)
|
||||
expect(groupState.stopped).toBe(false)
|
||||
})
|
||||
|
||||
it('preserves group authority errors when compensation fails', () => {
|
||||
const marker = 'ORCA_OWNER_GROUP=trial'
|
||||
const ownershipError = 'Benchmark process group no longer belongs to this trial'
|
||||
let thrown
|
||||
|
||||
try {
|
||||
signalValidatedProcessGroup(
|
||||
41,
|
||||
marker,
|
||||
'SIGKILL',
|
||||
{ stopped: true },
|
||||
{
|
||||
processIdentities: () => [{ pid: 41, pgid: 41, command: '/unrelated' }],
|
||||
signalProcess: () => {
|
||||
throw new Error('resume denied')
|
||||
}
|
||||
}
|
||||
)
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual([ownershipError, 'resume denied'])
|
||||
})
|
||||
|
||||
it('kills a recorded helper in a separate process group', async () => {
|
||||
@@ -263,7 +470,7 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('does not signal a replacement PID after post-stop revalidation', () => {
|
||||
it('compensates a possible stop after helper PID replacement', () => {
|
||||
const identity = { pid: 41, pgid: 41, command: '/helper marker' }
|
||||
const replacement = { pid: 41, pgid: 41, command: '/unrelated' }
|
||||
const signals = []
|
||||
@@ -280,7 +487,38 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => {
|
||||
}
|
||||
})
|
||||
).toThrow('Recorded benchmark helper PID changed before signaling')
|
||||
expect(signals).toEqual([[41, 'SIGSTOP']])
|
||||
expect(signals).toEqual([
|
||||
[41, 'SIGSTOP'],
|
||||
[41, 'SIGCONT']
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves helper identity errors when compensation fails', () => {
|
||||
const identity = { pid: 41, pgid: 41, command: '/helper marker' }
|
||||
const replacement = { pid: 41, pgid: 41, command: '/unrelated' }
|
||||
let inspectionCount = 0
|
||||
let thrown
|
||||
|
||||
try {
|
||||
signalProcessIdentity(identity, 'marker', 'SIGKILL', {
|
||||
processIdentity: () => {
|
||||
inspectionCount += 1
|
||||
return inspectionCount === 1 ? identity : replacement
|
||||
},
|
||||
signalProcess: (_pid, signal) => {
|
||||
if (signal === 'SIGCONT') {
|
||||
throw new Error('resume denied')
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(AggregateError)
|
||||
expect(thrown.errors.map((error) => error.message)).toEqual([
|
||||
'Recorded benchmark helper PID changed before signaling',
|
||||
'resume denied'
|
||||
])
|
||||
})
|
||||
|
||||
it('treats a missing PID as process exit after an identity query failure', () => {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { closeSync, existsSync, readFileSync, rmSync } from 'node:fs'
|
||||
import {
|
||||
killRecordedAndMatchingProcesses,
|
||||
runBenchmarkCleanupStages,
|
||||
signalValidatedProcessGroup
|
||||
} from './macos-computer-helper-owner-loss-processes.mjs'
|
||||
|
||||
export function cleanupOwnerLossTrial(options) {
|
||||
const groupState = { stopped: false }
|
||||
let error
|
||||
let output = ''
|
||||
try {
|
||||
runBenchmarkCleanupStages([
|
||||
() => {
|
||||
if (options.failed) {
|
||||
signalValidatedProcessGroup(options.pid, options.marker, 'SIGSTOP', groupState)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.failed) {
|
||||
killRecordedAndMatchingProcesses(options.recordPath, options.helperPath, [
|
||||
options.helperPath,
|
||||
options.tempDir
|
||||
])
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.failed) {
|
||||
signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL', groupState)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.stderrDescriptor !== undefined) {
|
||||
closeSync(options.stderrDescriptor)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (options.stdoutDescriptor !== undefined) {
|
||||
closeSync(options.stdoutDescriptor)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
output = options.outputPaths
|
||||
.filter(existsSync)
|
||||
.map((outputPath) => readFileSync(outputPath, 'utf8'))
|
||||
.join('')
|
||||
},
|
||||
() => rmSync(options.launcherDir, { recursive: true, force: true }),
|
||||
() => rmSync(options.tempDir, { recursive: true, force: true })
|
||||
])
|
||||
} catch (caught) {
|
||||
error = caught
|
||||
}
|
||||
return { error, output }
|
||||
}
|
||||
Reference in New Issue
Block a user