test: trim worker release checks and remove precheck suites (#41796)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-11-12 19:53:48 -08:00
committed by GitHub
co-authored by OrcaWin
parent 30fa6b4398
commit ddf4aa8d74
5 changed files with 2 additions and 327 deletions
+2 -2
View File
@@ -10904,12 +10904,12 @@
},
{
"file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts",
"assertions": ["retains a dead external terminal without closing a process"]
"assertions": ["does not claim retention succeeded after terminal close was committed"]
},
{
"file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts",
"assertions": [
"finishes a requested release after restart-style interruption within the same runtime and database",
"settles a closed terminal after restart when exact process liveness is exited",
"coalesces reconciliation joining an interactive release into one recorded recovery attempt and one terminal close"
]
},
@@ -1,44 +0,0 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import type { Repo } from '../../../shared/repo-types'
import { wslMoveCutoverFixture as cutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-cutover-fixture'
import { seedWslMoveCutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-journal-seeding'
import {
loadWslMoveFence,
resetWslMoveFenceForTests
} from '../../persistence/moving-wsl-projects/quiescence/wsl-move-scope-guard'
import { NO_DISPATCH_HOST, describeScheduledRefusal } from './dispatch-refusal'
function repo(id: string, overrides: Partial<Repo> = {}): Repo {
return { id, path: `/ws/${id}`, displayName: id, badgeColor: '#000', addedAt: 0, ...overrides }
}
const refusalFor = (target: Repo, canDispatch = true) =>
describeScheduledRefusal({ target: { ok: true, cwd: target.path, repo: target }, canDispatch })
?.error ?? null
let userDataPath: string
beforeEach(() => {
userDataPath = mkdtempSync(join(tmpdir(), 'wsl-move-automation-'))
loadWslMoveFence(userDataPath)
seedWslMoveCutover(userDataPath, cutover('move-1', 'source-fenced'))
})
afterEach(() => {
resetWslMoveFenceForTests()
rmSync(userDataPath, { recursive: true, force: true })
})
describe('describeScheduledRefusal under a WSL move', () => {
it('runs a project the move is not taking', () => {
expect(refusalFor(repo('repo-2'))).toBeNull()
expect(refusalFor(repo('repo-2'), false)).toBe(NO_DISPATCH_HOST)
})
it('runs the same repo id on another host', () => {
expect(
refusalFor(repo('repo-1', { connectionId: 'target-1', executionHostId: 'ssh:target-1' }))
).toBeNull()
})
})
@@ -1,102 +0,0 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { runAutomationPrecheck } from './precheck-runner'
const sshManagerState = vi.hoisted(() => ({
manager: null as null | {
getConnection: ReturnType<typeof vi.fn>
}
}))
vi.mock('../../ssh/targets/ssh-target-registry', () => ({
getSshConnectionManager: () => sshManagerState.manager
}))
const node = JSON.stringify(process.execPath)
function nodeCommand(script: string): string {
return `${node} -e ${JSON.stringify(script)}`
}
describe('runAutomationPrecheck', () => {
let cwd = ''
beforeEach(() => {
cwd = mkdtempSync(join(tmpdir(), 'orca-precheck-test-'))
sshManagerState.manager = null
})
afterEach(() => {
rmSync(cwd, { recursive: true, force: true })
})
it('captures exit code and output for a non-zero local precheck', async () => {
const result = await runAutomationPrecheck({
precheck: {
command: nodeCommand(
"console.log('stdout text'); console.error('stderr text'); process.exit(7)"
),
timeoutSeconds: 5
},
target: { type: 'local', cwd }
})
expect(result.exitCode).toBe(7)
expect(result.timedOut).toBe(false)
expect(result.stdout).toContain('stdout text')
expect(result.stderr).toContain('stderr text')
expect(result.error).toBeNull()
})
it('marks a local precheck as timed out', async () => {
const result = await runAutomationPrecheck({
precheck: {
command: nodeCommand('setTimeout(() => {}, 5000)'),
timeoutSeconds: 1
},
target: { type: 'local', cwd }
})
expect(result.exitCode).toBeNull()
expect(result.timedOut).toBe(true)
expect(result.error).toBe('Precheck timed out after 1s.')
})
it('uses the SSH channel exit event as the precheck exit code', async () => {
const channel = Object.assign(new EventEmitter(), {
stderr: new PassThrough(),
close: vi.fn()
})
sshManagerState.manager = {
getConnection: vi.fn(() => ({
getState: () => ({ status: 'connected' }),
exec: vi.fn(async () => channel)
}))
}
const resultPromise = runAutomationPrecheck({
precheck: {
command: "printf 'ready'",
timeoutSeconds: 5
},
target: {
type: 'ssh',
cwd: '/repo/path',
connectionId: 'ssh-1'
}
})
await Promise.resolve()
channel.emit('data', Buffer.from('ready\n'))
channel.emit('exit', 0)
channel.emit('close')
const result = await resultPromise
expect(result.exitCode).toBe(0)
expect(result.stdout).toContain('ready')
expect(result.error).toBeNull()
})
})
@@ -141,22 +141,6 @@ describe('orchestration worker release recovery', () => {
return worker
}
it('finishes a requested release after restart-style interruption', async () => {
setup()
const { dispatchId } = await startSettledWorker()
vi.mocked(runtime.closeTerminal).mockRejectedValueOnce(new Error('Multiplexer disposed'))
const interrupted = (await call('orchestration.workerRelease', { dispatch: dispatchId })) as {
state: string
}
expect(interrupted.state).toBe('release_pending')
expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('releasing')
const result = await reconcileRequestedWorkerTerminalReleases(runtime)
expect(result).toMatchObject({ attempted: 1, released: 1 })
expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released')
expect(runtime.closeTerminal).toHaveBeenCalledTimes(2)
})
it('settles a closed terminal after restart when exact process liveness is exited', async () => {
setup()
const { dispatchId } = await startSettledWorker()
@@ -198,58 +182,6 @@ describe('orchestration worker release recovery', () => {
expect(runtime.closeTerminal).toHaveBeenCalledTimes(1)
})
it('keeps a requested release pending after positive exit when no archive was committed', async () => {
setup()
const { dispatchId } = await startSettledWorker()
const requested = db.requestWorkerTerminalRelease(dispatchId)
expect(requested.disposition).toBe('requested')
expect(db.getWorkerTerminalArchive(dispatchId)).toBeUndefined()
vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
vi.mocked(runtime.getOrchestrationDispatchAuthority).mockReturnValue(null)
vi.mocked(runtime.getTerminalPaneKey).mockReturnValue(null)
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue(null)
vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited')
await expect(reconcileRequestedWorkerTerminalReleases(runtime)).resolves.toMatchObject({
attempted: 1,
released: 0,
pending: 1,
unknown: 0
})
expect(db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({
release_state: 'requested',
ownership_state: 'owned'
})
expect(runtime.closeTerminal).not.toHaveBeenCalled()
})
it('keeps the output reference when an unconfirmed release retry cannot find the terminal', async () => {
setup()
const { dispatchId } = await startSettledWorker()
vi.mocked(runtime.closeTerminal).mockResolvedValueOnce({
handle: 'term_worker',
tabId: 'tab-worker',
ptyKilled: false,
ptyStopVerdict: 'unverifiable',
ptyStopReason: 'its SSH provider is no longer registered'
})
await expect(
call('orchestration.workerRelease', { dispatch: dispatchId })
).resolves.toMatchObject({ state: 'release_unknown' })
expect(db.getWorkerTerminalArchive(dispatchId)).toBeDefined()
vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
await expect(
call('orchestration.workerRelease', { dispatch: dispatchId })
).resolves.toMatchObject({ state: 'release_unknown' })
// This worker reported no transcript, so there is no original to read.
await expect(call('orchestration.workerRead', { dispatch: dispatchId })).rejects.toMatchObject({
code: 'archive_unavailable'
})
})
it('records one recovery attempt when reconciliation joins an interactive release', async () => {
setup()
const { dispatchId } = await startSettledWorker()
@@ -7,61 +7,6 @@ describe('orchestration worker release', () => {
afterEach(() => h.cleanup())
it('is idempotent: a duplicate release returns already_released without another close', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker()
await h.call('orchestration.workerRelease', { dispatch: dispatchId })
const second = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
state: string
processAction: string
}
expect(second).toMatchObject({ state: 'already_released', processAction: 'none' })
expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1)
})
it('retains a dead external terminal the orchestration never owned', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker('succeeded', { terminal: 'term_worker' })
h.inspectProcessLiveness.mockResolvedValue('exited')
await expect(
h.call('orchestration.workerRelease', { dispatch: dispatchId })
).resolves.toMatchObject({
state: 'retained',
reason: 'external_terminal',
processAction: 'none'
})
expect(h.inspectProcessLiveness).toHaveBeenCalledWith(
'runtime_test:term_worker:1',
JSON.stringify({ kind: 'local', hostId: 'local' })
)
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({
ownership_state: 'external',
release_state: 'not_requested'
})
})
it.each([false, true])(
'keeps the output reference of a stopped worker (retained first: %s)',
async (retain) => {
h.setup()
const { dispatchId } = await h.startWorker()
await expect(
h.call('orchestration.workerStop', { dispatch: dispatchId })
).resolves.toMatchObject({ state: 'stopped' })
if (retain) {
await h.call('orchestration.workerRetain', { dispatch: dispatchId })
}
h.inspectProcessLiveness.mockResolvedValue('exited')
await expect(
h.call('orchestration.workerRelease', { dispatch: dispatchId })
).resolves.toMatchObject({ state: 'released', archive: { status: 'unavailable' } })
expect(h.db.getWorkerTerminalArchive(dispatchId)?.kind).toBe('terminal_tail')
}
)
it('does not claim retention succeeded after terminal close was committed', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker()
@@ -79,60 +24,4 @@ describe('orchestration worker release', () => {
await expect(release).resolves.toMatchObject({ state: 'released' })
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released')
})
it('keeps a resumed settled worker retained in worker-list without re-dispatch or release', async () => {
h.setup()
const { dispatchId, taskId } = await h.startSettledWorker()
const dispatch = h.db.getDispatchContextById(dispatchId)
const task = h.db.getTask(taskId)
vi.mocked(h.runtime.createTerminal).mockClear()
vi.mocked(h.runtime.sendTerminalAgentPrompt).mockClear()
vi.mocked(h.runtime.getTerminalProcessIncarnation).mockImplementation((handle) =>
handle === 'term_worker' ? 'runtime_test:term_worker:2' : null
)
await expect(
h.call('orchestration.workerRelease', { dispatch: dispatchId })
).resolves.toMatchObject({
state: 'retained',
reason: 'identity_unproven',
processAction: 'none'
})
const listed = (await h.call('orchestration.workerList', { run: h.activeRunId })) as {
workers: { dispatchId: string; terminalState: string; workerState: string }[]
}
expect(listed.workers).toEqual([
expect.objectContaining({ dispatchId, terminalState: 'retained', workerState: 'succeeded' })
])
expect(h.db.getTask(taskId)).toEqual(task)
expect(h.db.getDispatchContextById(dispatchId)).toEqual(dispatch)
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('retained')
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
expect(h.runtime.createTerminal).not.toHaveBeenCalled()
expect(h.runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
})
it('returns release_unknown when the terminal no longer resolves, then completes a retry', async () => {
h.setup()
const { dispatchId } = await h.startSettledWorker()
vi.mocked(h.runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
const receipt = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
state: string
recovery?: string
}
expect(receipt.state).toBe('release_unknown')
expect(receipt.recovery).toContain('worker-show')
expect(receipt.recovery).toContain('fresh request ID')
expect(receipt.recovery).not.toContain('same --retry-request')
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
vi.mocked(h.runtime.showTerminal).mockImplementation(
async (handle) => ({ handle, worktreeId: 'repo::worktree', status: 'running' }) as never
)
const retry = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
state: string
}
expect(retry.state).toBe('released')
expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1)
})
})