mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 20:06:10 +00:00
test: trim worker release checks and remove precheck suites (#41796)
Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
@@ -10904,12 +10904,12 @@
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts",
|
||||
"assertions": ["retains a dead external terminal without closing a process"]
|
||||
"assertions": ["does not claim retention succeeded after terminal close was committed"]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts",
|
||||
"assertions": [
|
||||
"finishes a requested release after restart-style interruption within the same runtime and database",
|
||||
"settles a closed terminal after restart when exact process liveness is exited",
|
||||
"coalesces reconciliation joining an interactive release into one recorded recovery attempt and one terminal close"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import type { Repo } from '../../../shared/repo-types'
|
||||
import { wslMoveCutoverFixture as cutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-cutover-fixture'
|
||||
import { seedWslMoveCutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-journal-seeding'
|
||||
import {
|
||||
loadWslMoveFence,
|
||||
resetWslMoveFenceForTests
|
||||
} from '../../persistence/moving-wsl-projects/quiescence/wsl-move-scope-guard'
|
||||
import { NO_DISPATCH_HOST, describeScheduledRefusal } from './dispatch-refusal'
|
||||
|
||||
function repo(id: string, overrides: Partial<Repo> = {}): Repo {
|
||||
return { id, path: `/ws/${id}`, displayName: id, badgeColor: '#000', addedAt: 0, ...overrides }
|
||||
}
|
||||
|
||||
const refusalFor = (target: Repo, canDispatch = true) =>
|
||||
describeScheduledRefusal({ target: { ok: true, cwd: target.path, repo: target }, canDispatch })
|
||||
?.error ?? null
|
||||
|
||||
let userDataPath: string
|
||||
beforeEach(() => {
|
||||
userDataPath = mkdtempSync(join(tmpdir(), 'wsl-move-automation-'))
|
||||
loadWslMoveFence(userDataPath)
|
||||
seedWslMoveCutover(userDataPath, cutover('move-1', 'source-fenced'))
|
||||
})
|
||||
afterEach(() => {
|
||||
resetWslMoveFenceForTests()
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('describeScheduledRefusal under a WSL move', () => {
|
||||
it('runs a project the move is not taking', () => {
|
||||
expect(refusalFor(repo('repo-2'))).toBeNull()
|
||||
expect(refusalFor(repo('repo-2'), false)).toBe(NO_DISPATCH_HOST)
|
||||
})
|
||||
|
||||
it('runs the same repo id on another host', () => {
|
||||
expect(
|
||||
refusalFor(repo('repo-1', { connectionId: 'target-1', executionHostId: 'ssh:target-1' }))
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -1,102 +0,0 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { runAutomationPrecheck } from './precheck-runner'
|
||||
|
||||
const sshManagerState = vi.hoisted(() => ({
|
||||
manager: null as null | {
|
||||
getConnection: ReturnType<typeof vi.fn>
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('../../ssh/targets/ssh-target-registry', () => ({
|
||||
getSshConnectionManager: () => sshManagerState.manager
|
||||
}))
|
||||
|
||||
const node = JSON.stringify(process.execPath)
|
||||
|
||||
function nodeCommand(script: string): string {
|
||||
return `${node} -e ${JSON.stringify(script)}`
|
||||
}
|
||||
|
||||
describe('runAutomationPrecheck', () => {
|
||||
let cwd = ''
|
||||
|
||||
beforeEach(() => {
|
||||
cwd = mkdtempSync(join(tmpdir(), 'orca-precheck-test-'))
|
||||
sshManagerState.manager = null
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(cwd, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('captures exit code and output for a non-zero local precheck', async () => {
|
||||
const result = await runAutomationPrecheck({
|
||||
precheck: {
|
||||
command: nodeCommand(
|
||||
"console.log('stdout text'); console.error('stderr text'); process.exit(7)"
|
||||
),
|
||||
timeoutSeconds: 5
|
||||
},
|
||||
target: { type: 'local', cwd }
|
||||
})
|
||||
|
||||
expect(result.exitCode).toBe(7)
|
||||
expect(result.timedOut).toBe(false)
|
||||
expect(result.stdout).toContain('stdout text')
|
||||
expect(result.stderr).toContain('stderr text')
|
||||
expect(result.error).toBeNull()
|
||||
})
|
||||
|
||||
it('marks a local precheck as timed out', async () => {
|
||||
const result = await runAutomationPrecheck({
|
||||
precheck: {
|
||||
command: nodeCommand('setTimeout(() => {}, 5000)'),
|
||||
timeoutSeconds: 1
|
||||
},
|
||||
target: { type: 'local', cwd }
|
||||
})
|
||||
|
||||
expect(result.exitCode).toBeNull()
|
||||
expect(result.timedOut).toBe(true)
|
||||
expect(result.error).toBe('Precheck timed out after 1s.')
|
||||
})
|
||||
|
||||
it('uses the SSH channel exit event as the precheck exit code', async () => {
|
||||
const channel = Object.assign(new EventEmitter(), {
|
||||
stderr: new PassThrough(),
|
||||
close: vi.fn()
|
||||
})
|
||||
sshManagerState.manager = {
|
||||
getConnection: vi.fn(() => ({
|
||||
getState: () => ({ status: 'connected' }),
|
||||
exec: vi.fn(async () => channel)
|
||||
}))
|
||||
}
|
||||
|
||||
const resultPromise = runAutomationPrecheck({
|
||||
precheck: {
|
||||
command: "printf 'ready'",
|
||||
timeoutSeconds: 5
|
||||
},
|
||||
target: {
|
||||
type: 'ssh',
|
||||
cwd: '/repo/path',
|
||||
connectionId: 'ssh-1'
|
||||
}
|
||||
})
|
||||
await Promise.resolve()
|
||||
channel.emit('data', Buffer.from('ready\n'))
|
||||
channel.emit('exit', 0)
|
||||
channel.emit('close')
|
||||
|
||||
const result = await resultPromise
|
||||
expect(result.exitCode).toBe(0)
|
||||
expect(result.stdout).toContain('ready')
|
||||
expect(result.error).toBeNull()
|
||||
})
|
||||
})
|
||||
-68
@@ -141,22 +141,6 @@ describe('orchestration worker release recovery', () => {
|
||||
return worker
|
||||
}
|
||||
|
||||
it('finishes a requested release after restart-style interruption', async () => {
|
||||
setup()
|
||||
const { dispatchId } = await startSettledWorker()
|
||||
vi.mocked(runtime.closeTerminal).mockRejectedValueOnce(new Error('Multiplexer disposed'))
|
||||
const interrupted = (await call('orchestration.workerRelease', { dispatch: dispatchId })) as {
|
||||
state: string
|
||||
}
|
||||
expect(interrupted.state).toBe('release_pending')
|
||||
expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('releasing')
|
||||
|
||||
const result = await reconcileRequestedWorkerTerminalReleases(runtime)
|
||||
expect(result).toMatchObject({ attempted: 1, released: 1 })
|
||||
expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released')
|
||||
expect(runtime.closeTerminal).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('settles a closed terminal after restart when exact process liveness is exited', async () => {
|
||||
setup()
|
||||
const { dispatchId } = await startSettledWorker()
|
||||
@@ -198,58 +182,6 @@ describe('orchestration worker release recovery', () => {
|
||||
expect(runtime.closeTerminal).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('keeps a requested release pending after positive exit when no archive was committed', async () => {
|
||||
setup()
|
||||
const { dispatchId } = await startSettledWorker()
|
||||
const requested = db.requestWorkerTerminalRelease(dispatchId)
|
||||
expect(requested.disposition).toBe('requested')
|
||||
expect(db.getWorkerTerminalArchive(dispatchId)).toBeUndefined()
|
||||
|
||||
vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
|
||||
vi.mocked(runtime.getOrchestrationDispatchAuthority).mockReturnValue(null)
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockReturnValue(null)
|
||||
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue(null)
|
||||
vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited')
|
||||
|
||||
await expect(reconcileRequestedWorkerTerminalReleases(runtime)).resolves.toMatchObject({
|
||||
attempted: 1,
|
||||
released: 0,
|
||||
pending: 1,
|
||||
unknown: 0
|
||||
})
|
||||
expect(db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({
|
||||
release_state: 'requested',
|
||||
ownership_state: 'owned'
|
||||
})
|
||||
expect(runtime.closeTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the output reference when an unconfirmed release retry cannot find the terminal', async () => {
|
||||
setup()
|
||||
const { dispatchId } = await startSettledWorker()
|
||||
vi.mocked(runtime.closeTerminal).mockResolvedValueOnce({
|
||||
handle: 'term_worker',
|
||||
tabId: 'tab-worker',
|
||||
ptyKilled: false,
|
||||
ptyStopVerdict: 'unverifiable',
|
||||
ptyStopReason: 'its SSH provider is no longer registered'
|
||||
})
|
||||
|
||||
await expect(
|
||||
call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({ state: 'release_unknown' })
|
||||
expect(db.getWorkerTerminalArchive(dispatchId)).toBeDefined()
|
||||
|
||||
vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
|
||||
await expect(
|
||||
call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({ state: 'release_unknown' })
|
||||
// This worker reported no transcript, so there is no original to read.
|
||||
await expect(call('orchestration.workerRead', { dispatch: dispatchId })).rejects.toMatchObject({
|
||||
code: 'archive_unavailable'
|
||||
})
|
||||
})
|
||||
|
||||
it('records one recovery attempt when reconciliation joins an interactive release', async () => {
|
||||
setup()
|
||||
const { dispatchId } = await startSettledWorker()
|
||||
|
||||
@@ -7,61 +7,6 @@ describe('orchestration worker release', () => {
|
||||
|
||||
afterEach(() => h.cleanup())
|
||||
|
||||
it('is idempotent: a duplicate release returns already_released without another close', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker()
|
||||
await h.call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
const second = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
|
||||
state: string
|
||||
processAction: string
|
||||
}
|
||||
expect(second).toMatchObject({ state: 'already_released', processAction: 'none' })
|
||||
expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('retains a dead external terminal the orchestration never owned', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker('succeeded', { terminal: 'term_worker' })
|
||||
h.inspectProcessLiveness.mockResolvedValue('exited')
|
||||
|
||||
await expect(
|
||||
h.call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({
|
||||
state: 'retained',
|
||||
reason: 'external_terminal',
|
||||
processAction: 'none'
|
||||
})
|
||||
expect(h.inspectProcessLiveness).toHaveBeenCalledWith(
|
||||
'runtime_test:term_worker:1',
|
||||
JSON.stringify({ kind: 'local', hostId: 'local' })
|
||||
)
|
||||
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
|
||||
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({
|
||||
ownership_state: 'external',
|
||||
release_state: 'not_requested'
|
||||
})
|
||||
})
|
||||
|
||||
it.each([false, true])(
|
||||
'keeps the output reference of a stopped worker (retained first: %s)',
|
||||
async (retain) => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startWorker()
|
||||
await expect(
|
||||
h.call('orchestration.workerStop', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({ state: 'stopped' })
|
||||
if (retain) {
|
||||
await h.call('orchestration.workerRetain', { dispatch: dispatchId })
|
||||
}
|
||||
h.inspectProcessLiveness.mockResolvedValue('exited')
|
||||
|
||||
await expect(
|
||||
h.call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({ state: 'released', archive: { status: 'unavailable' } })
|
||||
expect(h.db.getWorkerTerminalArchive(dispatchId)?.kind).toBe('terminal_tail')
|
||||
}
|
||||
)
|
||||
|
||||
it('does not claim retention succeeded after terminal close was committed', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker()
|
||||
@@ -79,60 +24,4 @@ describe('orchestration worker release', () => {
|
||||
await expect(release).resolves.toMatchObject({ state: 'released' })
|
||||
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released')
|
||||
})
|
||||
|
||||
it('keeps a resumed settled worker retained in worker-list without re-dispatch or release', async () => {
|
||||
h.setup()
|
||||
const { dispatchId, taskId } = await h.startSettledWorker()
|
||||
const dispatch = h.db.getDispatchContextById(dispatchId)
|
||||
const task = h.db.getTask(taskId)
|
||||
vi.mocked(h.runtime.createTerminal).mockClear()
|
||||
vi.mocked(h.runtime.sendTerminalAgentPrompt).mockClear()
|
||||
vi.mocked(h.runtime.getTerminalProcessIncarnation).mockImplementation((handle) =>
|
||||
handle === 'term_worker' ? 'runtime_test:term_worker:2' : null
|
||||
)
|
||||
|
||||
await expect(
|
||||
h.call('orchestration.workerRelease', { dispatch: dispatchId })
|
||||
).resolves.toMatchObject({
|
||||
state: 'retained',
|
||||
reason: 'identity_unproven',
|
||||
processAction: 'none'
|
||||
})
|
||||
const listed = (await h.call('orchestration.workerList', { run: h.activeRunId })) as {
|
||||
workers: { dispatchId: string; terminalState: string; workerState: string }[]
|
||||
}
|
||||
expect(listed.workers).toEqual([
|
||||
expect.objectContaining({ dispatchId, terminalState: 'retained', workerState: 'succeeded' })
|
||||
])
|
||||
expect(h.db.getTask(taskId)).toEqual(task)
|
||||
expect(h.db.getDispatchContextById(dispatchId)).toEqual(dispatch)
|
||||
expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('retained')
|
||||
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
|
||||
expect(h.runtime.createTerminal).not.toHaveBeenCalled()
|
||||
expect(h.runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns release_unknown when the terminal no longer resolves, then completes a retry', async () => {
|
||||
h.setup()
|
||||
const { dispatchId } = await h.startSettledWorker()
|
||||
vi.mocked(h.runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale'))
|
||||
const receipt = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
|
||||
state: string
|
||||
recovery?: string
|
||||
}
|
||||
expect(receipt.state).toBe('release_unknown')
|
||||
expect(receipt.recovery).toContain('worker-show')
|
||||
expect(receipt.recovery).toContain('fresh request ID')
|
||||
expect(receipt.recovery).not.toContain('same --retry-request')
|
||||
expect(h.runtime.closeTerminal).not.toHaveBeenCalled()
|
||||
|
||||
vi.mocked(h.runtime.showTerminal).mockImplementation(
|
||||
async (handle) => ({ handle, worktreeId: 'repo::worktree', status: 'running' }) as never
|
||||
)
|
||||
const retry = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as {
|
||||
state: string
|
||||
}
|
||||
expect(retry.state).toBe('released')
|
||||
expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user