diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 9eee6b4d2be..3983f6e283a 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10904,12 +10904,12 @@ }, { "file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts", - "assertions": ["retains a dead external terminal without closing a process"] + "assertions": ["does not claim retention succeeded after terminal close was committed"] }, { "file": "src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts", "assertions": [ - "finishes a requested release after restart-style interruption within the same runtime and database", + "settles a closed terminal after restart when exact process liveness is exited", "coalesces reconciliation joining an interactive release into one recorded recovery attempt and one terminal close" ] }, diff --git a/src/main/automations/dispatch/dispatch-refusal-wsl-move.test.ts b/src/main/automations/dispatch/dispatch-refusal-wsl-move.test.ts deleted file mode 100644 index e66128a7c0e..00000000000 --- a/src/main/automations/dispatch/dispatch-refusal-wsl-move.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { Repo } from '../../../shared/repo-types' -import { wslMoveCutoverFixture as cutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-cutover-fixture' -import { seedWslMoveCutover } from '../../persistence/__fixtures__/wsl-mover/wsl-move-journal-seeding' -import { - loadWslMoveFence, - resetWslMoveFenceForTests -} from '../../persistence/moving-wsl-projects/quiescence/wsl-move-scope-guard' -import { NO_DISPATCH_HOST, describeScheduledRefusal } from './dispatch-refusal' - -function repo(id: string, overrides: Partial = {}): Repo { - return { id, path: `/ws/${id}`, displayName: id, badgeColor: '#000', addedAt: 0, ...overrides } -} - -const refusalFor = (target: Repo, canDispatch = true) => - describeScheduledRefusal({ target: { ok: true, cwd: target.path, repo: target }, canDispatch }) - ?.error ?? null - -let userDataPath: string -beforeEach(() => { - userDataPath = mkdtempSync(join(tmpdir(), 'wsl-move-automation-')) - loadWslMoveFence(userDataPath) - seedWslMoveCutover(userDataPath, cutover('move-1', 'source-fenced')) -}) -afterEach(() => { - resetWslMoveFenceForTests() - rmSync(userDataPath, { recursive: true, force: true }) -}) - -describe('describeScheduledRefusal under a WSL move', () => { - it('runs a project the move is not taking', () => { - expect(refusalFor(repo('repo-2'))).toBeNull() - expect(refusalFor(repo('repo-2'), false)).toBe(NO_DISPATCH_HOST) - }) - - it('runs the same repo id on another host', () => { - expect( - refusalFor(repo('repo-1', { connectionId: 'target-1', executionHostId: 'ssh:target-1' })) - ).toBeNull() - }) -}) diff --git a/src/main/automations/dispatch/precheck-runner.test.ts b/src/main/automations/dispatch/precheck-runner.test.ts deleted file mode 100644 index 7e2ba9a3842..00000000000 --- a/src/main/automations/dispatch/precheck-runner.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { EventEmitter } from 'node:events' -import { PassThrough } from 'node:stream' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { runAutomationPrecheck } from './precheck-runner' - -const sshManagerState = vi.hoisted(() => ({ - manager: null as null | { - getConnection: ReturnType - } -})) - -vi.mock('../../ssh/targets/ssh-target-registry', () => ({ - getSshConnectionManager: () => sshManagerState.manager -})) - -const node = JSON.stringify(process.execPath) - -function nodeCommand(script: string): string { - return `${node} -e ${JSON.stringify(script)}` -} - -describe('runAutomationPrecheck', () => { - let cwd = '' - - beforeEach(() => { - cwd = mkdtempSync(join(tmpdir(), 'orca-precheck-test-')) - sshManagerState.manager = null - }) - - afterEach(() => { - rmSync(cwd, { recursive: true, force: true }) - }) - - it('captures exit code and output for a non-zero local precheck', async () => { - const result = await runAutomationPrecheck({ - precheck: { - command: nodeCommand( - "console.log('stdout text'); console.error('stderr text'); process.exit(7)" - ), - timeoutSeconds: 5 - }, - target: { type: 'local', cwd } - }) - - expect(result.exitCode).toBe(7) - expect(result.timedOut).toBe(false) - expect(result.stdout).toContain('stdout text') - expect(result.stderr).toContain('stderr text') - expect(result.error).toBeNull() - }) - - it('marks a local precheck as timed out', async () => { - const result = await runAutomationPrecheck({ - precheck: { - command: nodeCommand('setTimeout(() => {}, 5000)'), - timeoutSeconds: 1 - }, - target: { type: 'local', cwd } - }) - - expect(result.exitCode).toBeNull() - expect(result.timedOut).toBe(true) - expect(result.error).toBe('Precheck timed out after 1s.') - }) - - it('uses the SSH channel exit event as the precheck exit code', async () => { - const channel = Object.assign(new EventEmitter(), { - stderr: new PassThrough(), - close: vi.fn() - }) - sshManagerState.manager = { - getConnection: vi.fn(() => ({ - getState: () => ({ status: 'connected' }), - exec: vi.fn(async () => channel) - })) - } - - const resultPromise = runAutomationPrecheck({ - precheck: { - command: "printf 'ready'", - timeoutSeconds: 5 - }, - target: { - type: 'ssh', - cwd: '/repo/path', - connectionId: 'ssh-1' - } - }) - await Promise.resolve() - channel.emit('data', Buffer.from('ready\n')) - channel.emit('exit', 0) - channel.emit('close') - - const result = await resultPromise - expect(result.exitCode).toBe(0) - expect(result.stdout).toContain('ready') - expect(result.error).toBeNull() - }) -}) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts index b5507766e07..3bc4e18bf9d 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release-recovery.test.ts @@ -141,22 +141,6 @@ describe('orchestration worker release recovery', () => { return worker } - it('finishes a requested release after restart-style interruption', async () => { - setup() - const { dispatchId } = await startSettledWorker() - vi.mocked(runtime.closeTerminal).mockRejectedValueOnce(new Error('Multiplexer disposed')) - const interrupted = (await call('orchestration.workerRelease', { dispatch: dispatchId })) as { - state: string - } - expect(interrupted.state).toBe('release_pending') - expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('releasing') - - const result = await reconcileRequestedWorkerTerminalReleases(runtime) - expect(result).toMatchObject({ attempted: 1, released: 1 }) - expect(db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released') - expect(runtime.closeTerminal).toHaveBeenCalledTimes(2) - }) - it('settles a closed terminal after restart when exact process liveness is exited', async () => { setup() const { dispatchId } = await startSettledWorker() @@ -198,58 +182,6 @@ describe('orchestration worker release recovery', () => { expect(runtime.closeTerminal).toHaveBeenCalledTimes(1) }) - it('keeps a requested release pending after positive exit when no archive was committed', async () => { - setup() - const { dispatchId } = await startSettledWorker() - const requested = db.requestWorkerTerminalRelease(dispatchId) - expect(requested.disposition).toBe('requested') - expect(db.getWorkerTerminalArchive(dispatchId)).toBeUndefined() - - vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale')) - vi.mocked(runtime.getOrchestrationDispatchAuthority).mockReturnValue(null) - vi.mocked(runtime.getTerminalPaneKey).mockReturnValue(null) - vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue(null) - vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited') - - await expect(reconcileRequestedWorkerTerminalReleases(runtime)).resolves.toMatchObject({ - attempted: 1, - released: 0, - pending: 1, - unknown: 0 - }) - expect(db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({ - release_state: 'requested', - ownership_state: 'owned' - }) - expect(runtime.closeTerminal).not.toHaveBeenCalled() - }) - - it('keeps the output reference when an unconfirmed release retry cannot find the terminal', async () => { - setup() - const { dispatchId } = await startSettledWorker() - vi.mocked(runtime.closeTerminal).mockResolvedValueOnce({ - handle: 'term_worker', - tabId: 'tab-worker', - ptyKilled: false, - ptyStopVerdict: 'unverifiable', - ptyStopReason: 'its SSH provider is no longer registered' - }) - - await expect( - call('orchestration.workerRelease', { dispatch: dispatchId }) - ).resolves.toMatchObject({ state: 'release_unknown' }) - expect(db.getWorkerTerminalArchive(dispatchId)).toBeDefined() - - vi.mocked(runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale')) - await expect( - call('orchestration.workerRelease', { dispatch: dispatchId }) - ).resolves.toMatchObject({ state: 'release_unknown' }) - // This worker reported no transcript, so there is no original to read. - await expect(call('orchestration.workerRead', { dispatch: dispatchId })).rejects.toMatchObject({ - code: 'archive_unavailable' - }) - }) - it('records one recovery attempt when reconciliation joins an interactive release', async () => { setup() const { dispatchId } = await startSettledWorker() diff --git a/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts index 67dc1401178..6c12828fd09 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/release/worker-release.test.ts @@ -7,61 +7,6 @@ describe('orchestration worker release', () => { afterEach(() => h.cleanup()) - it('is idempotent: a duplicate release returns already_released without another close', async () => { - h.setup() - const { dispatchId } = await h.startSettledWorker() - await h.call('orchestration.workerRelease', { dispatch: dispatchId }) - const second = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as { - state: string - processAction: string - } - expect(second).toMatchObject({ state: 'already_released', processAction: 'none' }) - expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1) - }) - - it('retains a dead external terminal the orchestration never owned', async () => { - h.setup() - const { dispatchId } = await h.startSettledWorker('succeeded', { terminal: 'term_worker' }) - h.inspectProcessLiveness.mockResolvedValue('exited') - - await expect( - h.call('orchestration.workerRelease', { dispatch: dispatchId }) - ).resolves.toMatchObject({ - state: 'retained', - reason: 'external_terminal', - processAction: 'none' - }) - expect(h.inspectProcessLiveness).toHaveBeenCalledWith( - 'runtime_test:term_worker:1', - JSON.stringify({ kind: 'local', hostId: 'local' }) - ) - expect(h.runtime.closeTerminal).not.toHaveBeenCalled() - expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)).toMatchObject({ - ownership_state: 'external', - release_state: 'not_requested' - }) - }) - - it.each([false, true])( - 'keeps the output reference of a stopped worker (retained first: %s)', - async (retain) => { - h.setup() - const { dispatchId } = await h.startWorker() - await expect( - h.call('orchestration.workerStop', { dispatch: dispatchId }) - ).resolves.toMatchObject({ state: 'stopped' }) - if (retain) { - await h.call('orchestration.workerRetain', { dispatch: dispatchId }) - } - h.inspectProcessLiveness.mockResolvedValue('exited') - - await expect( - h.call('orchestration.workerRelease', { dispatch: dispatchId }) - ).resolves.toMatchObject({ state: 'released', archive: { status: 'unavailable' } }) - expect(h.db.getWorkerTerminalArchive(dispatchId)?.kind).toBe('terminal_tail') - } - ) - it('does not claim retention succeeded after terminal close was committed', async () => { h.setup() const { dispatchId } = await h.startSettledWorker() @@ -79,60 +24,4 @@ describe('orchestration worker release', () => { await expect(release).resolves.toMatchObject({ state: 'released' }) expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('released') }) - - it('keeps a resumed settled worker retained in worker-list without re-dispatch or release', async () => { - h.setup() - const { dispatchId, taskId } = await h.startSettledWorker() - const dispatch = h.db.getDispatchContextById(dispatchId) - const task = h.db.getTask(taskId) - vi.mocked(h.runtime.createTerminal).mockClear() - vi.mocked(h.runtime.sendTerminalAgentPrompt).mockClear() - vi.mocked(h.runtime.getTerminalProcessIncarnation).mockImplementation((handle) => - handle === 'term_worker' ? 'runtime_test:term_worker:2' : null - ) - - await expect( - h.call('orchestration.workerRelease', { dispatch: dispatchId }) - ).resolves.toMatchObject({ - state: 'retained', - reason: 'identity_unproven', - processAction: 'none' - }) - const listed = (await h.call('orchestration.workerList', { run: h.activeRunId })) as { - workers: { dispatchId: string; terminalState: string; workerState: string }[] - } - expect(listed.workers).toEqual([ - expect.objectContaining({ dispatchId, terminalState: 'retained', workerState: 'succeeded' }) - ]) - expect(h.db.getTask(taskId)).toEqual(task) - expect(h.db.getDispatchContextById(dispatchId)).toEqual(dispatch) - expect(h.db.getWorkerTerminalResourceByOwner(dispatchId)?.release_state).toBe('retained') - expect(h.runtime.closeTerminal).not.toHaveBeenCalled() - expect(h.runtime.createTerminal).not.toHaveBeenCalled() - expect(h.runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() - }) - - it('returns release_unknown when the terminal no longer resolves, then completes a retry', async () => { - h.setup() - const { dispatchId } = await h.startSettledWorker() - vi.mocked(h.runtime.showTerminal).mockRejectedValue(new Error('terminal_handle_stale')) - const receipt = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as { - state: string - recovery?: string - } - expect(receipt.state).toBe('release_unknown') - expect(receipt.recovery).toContain('worker-show') - expect(receipt.recovery).toContain('fresh request ID') - expect(receipt.recovery).not.toContain('same --retry-request') - expect(h.runtime.closeTerminal).not.toHaveBeenCalled() - - vi.mocked(h.runtime.showTerminal).mockImplementation( - async (handle) => ({ handle, worktreeId: 'repo::worktree', status: 'running' }) as never - ) - const retry = (await h.call('orchestration.workerRelease', { dispatch: dispatchId })) as { - state: string - } - expect(retry.state).toBe('released') - expect(h.runtime.closeTerminal).toHaveBeenCalledTimes(1) - }) })