diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 2478dbef301..dbd62e77200 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -13,6 +13,7 @@ on: - 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs' - 'config/scripts/macos-computer-helper-owner-loss-processes.mjs' - 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs' + - 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs' - 'config/scripts/computer-use-modifier-safety.test.mjs' - 'config/scripts/computer-use-skill-guidance.test.mjs' - 'config/scripts/computer-use-smoke.mjs' diff --git a/config/scripts/computer-e2e-workflow.test.mjs b/config/scripts/computer-e2e-workflow.test.mjs index 9725faa149d..17972e32031 100644 --- a/config/scripts/computer-e2e-workflow.test.mjs +++ b/config/scripts/computer-e2e-workflow.test.mjs @@ -139,7 +139,8 @@ describe('computer-use e2e workflow', () => { 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs', 'config/scripts/macos-computer-helper-owner-loss-metrics.mjs', 'config/scripts/macos-computer-helper-owner-loss-processes.mjs', - 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs' + 'config/scripts/macos-computer-helper-owner-loss-processes.test.mjs', + 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs' ]) ) }) @@ -149,10 +150,17 @@ describe('computer-use e2e workflow', () => { join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-benchmark.mjs'), 'utf8' ) + const cleanup = readFileSync( + join(projectDir, 'config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs'), + 'utf8' + ) expect(benchmark).toContain('spawnBenchmarkProcess(executable, [launcherDir]') expect(benchmark).toContain("stdio: ['ignore', stdoutDescriptor, stderrDescriptor]") - expect(benchmark).toContain('killRecordedAndMatchingProcesses(helperRecordPath, helperPath') + expect(benchmark).toContain('cleanupOwnerLossTrial({') + expect(benchmark).toContain('trialCleanupSha256: artifactSha256(trialCleanupPath)') + expect(cleanup).toContain('killRecordedAndMatchingProcesses(options.recordPath') + expect(cleanup).toContain("signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL'") }) it('boots the built daemon under plain Node in the PR native-smoke job after the main build', () => { diff --git a/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs b/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs index 12a2f23c213..5cb99835fd1 100644 --- a/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs +++ b/config/scripts/macos-computer-helper-owner-loss-benchmark.mjs @@ -1,15 +1,7 @@ #!/usr/bin/env node import { createHash } from 'node:crypto' import { execFileSync, fork } from 'node:child_process' -import { - closeSync, - existsSync, - mkdtempSync, - openSync, - readFileSync, - rmSync, - writeFileSync -} from 'node:fs' +import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs' import net from 'node:net' import { tmpdir } from 'node:os' import path from 'node:path' @@ -21,13 +13,13 @@ import { sampleProcess } from './macos-computer-helper-owner-loss-metrics.mjs' import { - killRecordedAndMatchingProcesses, processIdentityIsCurrent, signalProcessIdentity, - signalValidatedProcessGroup, spawnBenchmarkProcess, + throwBenchmarkTrialFailures, writeProcessRecord } from './macos-computer-helper-owner-loss-processes.mjs' +import { cleanupOwnerLossTrial } from './macos-computer-helper-owner-loss-trial-cleanup.mjs' const INTERNAL_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_INTERNAL' const EXPECTATION_ENV = 'ORCA_COMPUTER_HELPER_OWNER_BENCH_EXPECTATION' @@ -47,6 +39,10 @@ const processCleanupPath = path.join( import.meta.dirname, 'macos-computer-helper-owner-loss-processes.mjs' ) +const trialCleanupPath = path.join( + import.meta.dirname, + 'macos-computer-helper-owner-loss-trial-cleanup.mjs' +) const sidecarPath = path.join(repoRoot, 'out', 'main', 'computer-sidecar.js') const helperAppPath = path.join( repoRoot, @@ -457,6 +453,8 @@ function runTrial(executable, expectation) { ) let result let serializedResult + let trialError + let cleanupError let stderrDescriptor let stdoutDescriptor let trialOutput = '' @@ -481,48 +479,35 @@ function runTrial(executable, expectation) { if (result.status === 0 && existsSync(resultPath)) { serializedResult = readFileSync(resultPath, 'utf8') } + } catch (error) { + trialError = error } finally { const failedTrial = result?.status !== 0 || !serializedResult - const trialEnvironmentFragment = `TMPDIR=${trialTempDir}` - try { - if (failedTrial) { - signalValidatedProcessGroup(result?.pid, trialEnvironmentFragment, 'SIGSTOP') - } - } finally { - try { - if (failedTrial) { - killRecordedAndMatchingProcesses(helperRecordPath, helperPath, [helperPath, trialTempDir]) - } - } finally { - try { - if (failedTrial) { - signalValidatedProcessGroup(result?.pid, trialEnvironmentFragment, 'SIGKILL') - } - } finally { - if (stderrDescriptor !== undefined) { - closeSync(stderrDescriptor) - } - if (stdoutDescriptor !== undefined) { - closeSync(stdoutDescriptor) - } - trialOutput = [stderrPath, stdoutPath] - .filter(existsSync) - .map((outputPath) => readFileSync(outputPath, 'utf8')) - .join('') - rmSync(launcherDir, { recursive: true, force: true }) - rmSync(trialTempDir, { recursive: true, force: true }) - } - } - } + const trialMarker = `TMPDIR=${trialTempDir}` + const cleanup = cleanupOwnerLossTrial({ + failed: failedTrial, + pid: result?.pid, + marker: trialMarker, + recordPath: helperRecordPath, + helperPath, + tempDir: trialTempDir, + stderrDescriptor, + stdoutDescriptor, + outputPaths: [stderrPath, stdoutPath], + launcherDir + }) + cleanupError = cleanup.error + trialOutput = cleanup.output } - if (result.status !== 0) { - throw new Error( + if (!trialError && result?.status !== 0) { + trialError = new Error( `Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n${trialOutput}` ) } - if (!serializedResult) { - throw new Error(`Electron trial did not write a result:\n${trialOutput}`) + if (!trialError && !serializedResult) { + trialError = new Error(`Electron trial did not write a result:\n${trialOutput}`) } + throwBenchmarkTrialFailures(trialError, cleanupError) return JSON.parse(serializedResult) } @@ -565,7 +550,8 @@ function runBenchmark() { sources: { benchmarkSha256: artifactSha256(scriptPath), metricsSha256: artifactSha256(metricsPath), - processCleanupSha256: artifactSha256(processCleanupPath) + processCleanupSha256: artifactSha256(processCleanupPath), + trialCleanupSha256: artifactSha256(trialCleanupPath) }, expectation: options.expect, trials: options.trials, diff --git a/config/scripts/macos-computer-helper-owner-loss-processes.mjs b/config/scripts/macos-computer-helper-owner-loss-processes.mjs index e30b5215095..522bc4e92f8 100644 --- a/config/scripts/macos-computer-helper-owner-loss-processes.mjs +++ b/config/scripts/macos-computer-helper-owner-loss-processes.mjs @@ -156,6 +156,35 @@ export function spawnBenchmarkProcess(executable, args, options) { }) } +export function runBenchmarkCleanupStages(stages) { + const errors = [] + for (const stage of stages) { + try { + stage() + } catch (error) { + errors.push(error) + } + } + if (errors.length === 1) { + throw errors[0] + } + if (errors.length > 1) { + throw new AggregateError(errors, 'Benchmark trial cleanup failed') + } +} + +export function throwBenchmarkTrialFailures(trialError, cleanupError) { + if (trialError && cleanupError) { + throw new AggregateError([trialError, cleanupError], 'Electron trial and cleanup failed') + } + if (trialError) { + throw trialError + } + if (cleanupError) { + throw cleanupError + } +} + const processGroupSignalOperations = { processIdentities, signalProcess: process.kill.bind(process) @@ -165,31 +194,72 @@ export function signalValidatedProcessGroup( pgid, environmentFragment, signal, + groupState = { stopped: false }, operations = processGroupSignalOperations ) { if (!Number.isInteger(pgid) || pgid <= 0) { return false } - const members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid) + let members + try { + members = operations.processIdentities(true).filter((identity) => identity.pgid === pgid) + } catch (error) { + if (!groupState.stopped) { + throw error + } + try { + operations.signalProcess(-pgid, 'SIGCONT') + groupState.stopped = false + } catch (resumeError) { + if (resumeError?.code === 'ESRCH') { + groupState.stopped = false + } else { + throw new AggregateError( + [error, resumeError], + 'Benchmark process group recovery failed before validation' + ) + } + } + throw error + } if (members.length === 0) { return false } if (members.some((identity) => !identity.command.includes(environmentFragment))) { - throw new Error('Benchmark process group no longer belongs to this trial') + const ownershipError = new Error('Benchmark process group no longer belongs to this trial') + if (groupState.stopped) { + try { + operations.signalProcess(-pgid, 'SIGCONT') + groupState.stopped = false + } catch (resumeError) { + if (resumeError?.code === 'ESRCH') { + groupState.stopped = false + } else { + throw new AggregateError( + [ownershipError, resumeError], + 'Benchmark process group authority recovery failed' + ) + } + } + } + throw ownershipError } const anchor = members[0] - let stopped = false + let anchorNeedsResume = false + let groupNeedsResume = groupState.stopped try { operations.signalProcess(anchor.pid, 'SIGSTOP') - stopped = true + anchorNeedsResume = true const stoppedAnchor = operations .processIdentities(true) .find((identity) => identity.pid === anchor.pid) if (!sameIdentity(stoppedAnchor, anchor)) { - stopped = false throw new Error('Benchmark process group anchor changed before signaling') } operations.signalProcess(-pgid, 'SIGSTOP') + groupNeedsResume = true + groupState.stopped = true + anchorNeedsResume = false const stoppedMembers = operations .processIdentities(true) .filter((identity) => identity.pgid === pgid) @@ -204,29 +274,31 @@ export function signalValidatedProcessGroup( if (signal !== 'SIGKILL') { operations.signalProcess(-pgid, 'SIGCONT') } - stopped = false + groupNeedsResume = false + groupState.stopped = false } return true } catch (error) { const recoveryErrors = [] - if (stopped) { + if (groupNeedsResume) { try { - const recoveryMembers = operations - .processIdentities(true) - .filter( - (identity) => identity.pgid === pgid && identity.command.includes(environmentFragment) - ) - for (const member of recoveryMembers) { - try { - operations.signalProcess(member.pid, 'SIGCONT') - } catch (caught) { - if (caught?.code !== 'ESRCH') { - recoveryErrors.push(caught) - } - } - } + operations.signalProcess(-pgid, 'SIGCONT') + groupState.stopped = false } catch (caught) { - recoveryErrors.push(caught) + if (caught?.code === 'ESRCH') { + groupState.stopped = false + } else { + recoveryErrors.push(caught) + } + } + } + if (anchorNeedsResume) { + try { + operations.signalProcess(anchor.pid, 'SIGCONT') + } catch (caught) { + if (caught?.code !== 'ESRCH') { + recoveryErrors.push(caught) + } } } if (recoveryErrors.length > 0) { @@ -277,7 +349,6 @@ export function signalProcessIdentity( stopped = true const stoppedIdentity = operations.processIdentity(identity.pid) if (!sameIdentity(stoppedIdentity, identity)) { - stopped = false throw new Error('Recorded benchmark helper PID changed before signaling') } operations.signalProcess(-identity.pgid, signal) @@ -290,10 +361,7 @@ export function signalProcessIdentity( let resumeError if (stopped) { try { - const recoveryIdentity = operations.processIdentity(identity.pid) - if (sameIdentity(recoveryIdentity, identity)) { - operations.signalProcess(identity.pid, 'SIGCONT') - } + operations.signalProcess(identity.pid, 'SIGCONT') } catch (caught) { if (caught?.code !== 'ESRCH') { resumeError = caught diff --git a/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs b/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs index aa59cbbbaaa..ae799b02fc5 100644 --- a/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs +++ b/config/scripts/macos-computer-helper-owner-loss-processes.test.mjs @@ -8,9 +8,11 @@ import { killRecordedAndMatchingProcesses, killRecordedProcess, processIdentity, + runBenchmarkCleanupStages, signalProcessIdentity, signalValidatedProcessGroup, spawnBenchmarkProcess, + throwBenchmarkTrialFailures, writeProcessRecord } from './macos-computer-helper-owner-loss-processes.mjs' @@ -45,6 +47,46 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { expect(() => process.kill(result.pid, 0)).toThrow() }) + it('runs every cleanup stage before aggregating errors', () => { + const completed = [] + let thrown + + try { + runBenchmarkCleanupStages([ + () => { + completed.push(1) + throw new Error('first failure') + }, + () => { + completed.push(2) + }, + () => { + completed.push(3) + throw new Error('last failure') + } + ]) + } catch (error) { + thrown = error + } + expect(completed).toEqual([1, 2, 3]) + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual(['first failure', 'last failure']) + }) + + it('preserves both trial and cleanup failures', () => { + const trialError = new Error('trial failed with captured output') + const cleanupError = new Error('cleanup failed') + let thrown + + try { + throwBenchmarkTrialFailures(trialError, cleanupError) + } catch (error) { + thrown = error + } + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors).toEqual([trialError, cleanupError]) + }) + it('kills a timed-out trial group only after validating its environment', async () => { const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'orca-owner-benchmark-group-test-')) temporaryDirectories.add(temporaryDirectory) @@ -68,13 +110,18 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { const childPid = Number(readFileSync(childPidPath, 'utf8')) spawnedPids.add(childPid) const environmentFragment = `${environmentName}=${environmentValue}` + const groupState = { stopped: false } expect(() => signalValidatedProcessGroup(result.pid, `${environmentName}=wrong`, 'SIGSTOP') ).toThrow('Benchmark process group no longer belongs to this trial') expect(() => process.kill(childPid, 0)).not.toThrow() - expect(signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP')).toBe(true) - expect(signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL')).toBe(true) + expect( + signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGSTOP', groupState) + ).toBe(true) + expect( + signalValidatedProcessGroup(result.pid, environmentFragment, 'SIGKILL', groupState) + ).toBe(true) await expect .poll(() => { try { @@ -89,7 +136,7 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { spawnedPids.delete(childPid) }) - it('resumes marked group members after revalidation fails', () => { + it('resumes the group after post-stop revalidation fails', () => { const marker = 'ORCA_OWNER_GROUP=trial' const members = [ { pid: 41, pgid: 41, command: `/launcher ${marker}` }, @@ -99,46 +146,206 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { let scanCount = 0 expect(() => - signalValidatedProcessGroup(41, marker, 'SIGKILL', { - processIdentities: () => { - scanCount += 1 - if (scanCount === 3) { - throw new Error('transient group inspection failure') + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: false }, + { + processIdentities: () => { + scanCount += 1 + if (scanCount === 3) { + throw new Error('transient group inspection failure') + } + return members + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) } - return members - }, - signalProcess: (pid, signal) => { - signals.push([pid, signal]) } - }) + ) ).toThrow('transient group inspection failure') expect(signals).toEqual([ [41, 'SIGSTOP'], [-41, 'SIGSTOP'], - [41, 'SIGCONT'], - [42, 'SIGCONT'] + [-41, 'SIGCONT'] ]) }) - it('does not signal a replacement group anchor', () => { + it('compensates a possible stop after group anchor replacement', () => { const marker = 'ORCA_OWNER_GROUP=trial' const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` } - const replacement = { pid: 41, pgid: 41, command: '/unrelated' } + const replacement = { pid: 41, pgid: 99, command: '/unrelated' } const signals = [] let scanCount = 0 expect(() => - signalValidatedProcessGroup(41, marker, 'SIGKILL', { - processIdentities: () => { - scanCount += 1 - return scanCount === 1 ? [anchor] : [replacement] - }, - signalProcess: (pid, signal) => { - signals.push([pid, signal]) + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: false }, + { + processIdentities: () => { + scanCount += 1 + return scanCount === 1 ? [anchor] : [replacement] + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } } - }) + ) ).toThrow('Benchmark process group anchor changed before signaling') - expect(signals).toEqual([[41, 'SIGSTOP']]) + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [41, 'SIGCONT'] + ]) + }) + + it('resumes a previously frozen group when final inspection fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }] + const signals = [] + let scanCount = 0 + const groupState = { stopped: false } + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount === 4) { + throw new Error('transient final inspection failure') + } + return members + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('transient final inspection failure') + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [-41, 'SIGSTOP'], + [-41, 'SIGCONT'] + ]) + }) + + it('resumes a previously frozen group when final anchor stop fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [ + { pid: 41, pgid: 41, command: `/launcher ${marker}` }, + { pid: 42, pgid: 41, command: `/child ${marker}` } + ] + const signals = [] + let finalCall = false + const groupState = { stopped: false } + const missingProcessError = Object.assign(new Error('anchor exited'), { code: 'ESRCH' }) + const operations = { + processIdentities: () => members, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + if (finalCall && pid === 41 && signal === 'SIGSTOP') { + throw missingProcessError + } + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + finalCall = true + expect(signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations)).toBe(false) + expect(signals.at(-1)).toEqual([-41, 'SIGCONT']) + }) + + it('resumes a previously frozen group after final anchor replacement', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const anchor = { pid: 41, pgid: 41, command: `/launcher ${marker}` } + const child = { pid: 42, pgid: 41, command: `/child ${marker}` } + const replacement = { pid: 41, pgid: 99, command: '/unrelated' } + const signals = [] + let scanCount = 0 + const groupState = { stopped: false } + const operations = { + processIdentities: () => { + scanCount += 1 + return scanCount === 5 ? [replacement, child] : [anchor, child] + }, + signalProcess: (pid, signal) => { + signals.push([pid, signal]) + } + } + + expect(signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations)).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('Benchmark process group anchor changed before signaling') + expect(signals.slice(-2)).toEqual([ + [-41, 'SIGCONT'], + [41, 'SIGCONT'] + ]) + }) + + it('retains uncertain stop state across cleanup stage failures', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const members = [{ pid: 41, pgid: 41, command: `/launcher ${marker}` }] + const groupState = { stopped: false } + let scanCount = 0 + let continueAttempts = 0 + const operations = { + processIdentities: () => { + scanCount += 1 + if (scanCount >= 3) { + throw new Error( + scanCount === 3 ? 'post-stop inspection failed' : 'final inspection failed' + ) + } + return members + }, + signalProcess: (pid, signal) => { + if (pid === -41 && signal === 'SIGCONT') { + continueAttempts += 1 + if (continueAttempts === 1) { + throw new Error('first compensation failed') + } + } + } + } + + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGSTOP', groupState, operations) + ).toThrow('Benchmark process group signal recovery failed') + expect(groupState.stopped).toBe(true) + expect(() => + signalValidatedProcessGroup(41, marker, 'SIGKILL', groupState, operations) + ).toThrow('final inspection failed') + expect(continueAttempts).toBe(2) + expect(groupState.stopped).toBe(false) + }) + + it('preserves group authority errors when compensation fails', () => { + const marker = 'ORCA_OWNER_GROUP=trial' + const ownershipError = 'Benchmark process group no longer belongs to this trial' + let thrown + + try { + signalValidatedProcessGroup( + 41, + marker, + 'SIGKILL', + { stopped: true }, + { + processIdentities: () => [{ pid: 41, pgid: 41, command: '/unrelated' }], + signalProcess: () => { + throw new Error('resume denied') + } + } + ) + } catch (error) { + thrown = error + } + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual([ownershipError, 'resume denied']) }) it('kills a recorded helper in a separate process group', async () => { @@ -263,7 +470,7 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { ]) }) - it('does not signal a replacement PID after post-stop revalidation', () => { + it('compensates a possible stop after helper PID replacement', () => { const identity = { pid: 41, pgid: 41, command: '/helper marker' } const replacement = { pid: 41, pgid: 41, command: '/unrelated' } const signals = [] @@ -280,7 +487,38 @@ describeMacOS('macOS helper owner-loss benchmark process cleanup', () => { } }) ).toThrow('Recorded benchmark helper PID changed before signaling') - expect(signals).toEqual([[41, 'SIGSTOP']]) + expect(signals).toEqual([ + [41, 'SIGSTOP'], + [41, 'SIGCONT'] + ]) + }) + + it('preserves helper identity errors when compensation fails', () => { + const identity = { pid: 41, pgid: 41, command: '/helper marker' } + const replacement = { pid: 41, pgid: 41, command: '/unrelated' } + let inspectionCount = 0 + let thrown + + try { + signalProcessIdentity(identity, 'marker', 'SIGKILL', { + processIdentity: () => { + inspectionCount += 1 + return inspectionCount === 1 ? identity : replacement + }, + signalProcess: (_pid, signal) => { + if (signal === 'SIGCONT') { + throw new Error('resume denied') + } + } + }) + } catch (error) { + thrown = error + } + expect(thrown).toBeInstanceOf(AggregateError) + expect(thrown.errors.map((error) => error.message)).toEqual([ + 'Recorded benchmark helper PID changed before signaling', + 'resume denied' + ]) }) it('treats a missing PID as process exit after an identity query failure', () => { diff --git a/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs b/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs new file mode 100644 index 00000000000..61e0fef6f14 --- /dev/null +++ b/config/scripts/macos-computer-helper-owner-loss-trial-cleanup.mjs @@ -0,0 +1,55 @@ +import { closeSync, existsSync, readFileSync, rmSync } from 'node:fs' +import { + killRecordedAndMatchingProcesses, + runBenchmarkCleanupStages, + signalValidatedProcessGroup +} from './macos-computer-helper-owner-loss-processes.mjs' + +export function cleanupOwnerLossTrial(options) { + const groupState = { stopped: false } + let error + let output = '' + try { + runBenchmarkCleanupStages([ + () => { + if (options.failed) { + signalValidatedProcessGroup(options.pid, options.marker, 'SIGSTOP', groupState) + } + }, + () => { + if (options.failed) { + killRecordedAndMatchingProcesses(options.recordPath, options.helperPath, [ + options.helperPath, + options.tempDir + ]) + } + }, + () => { + if (options.failed) { + signalValidatedProcessGroup(options.pid, options.marker, 'SIGKILL', groupState) + } + }, + () => { + if (options.stderrDescriptor !== undefined) { + closeSync(options.stderrDescriptor) + } + }, + () => { + if (options.stdoutDescriptor !== undefined) { + closeSync(options.stdoutDescriptor) + } + }, + () => { + output = options.outputPaths + .filter(existsSync) + .map((outputPath) => readFileSync(outputPath, 'utf8')) + .join('') + }, + () => rmSync(options.launcherDir, { recursive: true, force: true }), + () => rmSync(options.tempDir, { recursive: true, force: true }) + ]) + } catch (caught) { + error = caught + } + return { error, output } +}