fix(runtime): the host stops a removed project's shells and agents itself (#29964)

* fix(runtime): the host stops a removed project's terminals itself

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): a runtime-stamped project still stops the shells that run here

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): refresh the PTY inventory and sweep again after a project's removal

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): bound the project inventory refresh and recheck the WSL move fence

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): fence a project's removal sweep by its SSH connection and wait out in-flight spawns

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): bound the project stop's lease wait by the removal deadline

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): refuse spawns into a removed project and skip a stop a same-id sibling shares

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): bound each project stop by the removal deadline

Co-Authored-By: Claude <noreply@anthropic.com>

* fix(runtime): pass the removal deadline to the lease so a timed-out waiter is dequeued

Co-Authored-By: Claude <noreply@anthropic.com>

* test(runtime): type the project-removal fixtures instead of casting

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Neil
2026-10-16 21:54:47 -07:00
committed by GitHub
co-authored by Claude
parent 0d9beb603e
commit a82298ebe6
7 changed files with 386 additions and 17 deletions
@@ -56,6 +56,9 @@ export abstract class RuntimeConstructionServices {
abstract stopTerminalsForWorktree(
...args: Calls.StopTerminalsForWorktreeArgs
): Calls.StopTerminalsForWorktreeResult
abstract stopTerminalsForProject(
...args: Calls.StopTerminalsForProjectArgs
): Calls.StopTerminalsForProjectResult
abstract resolveOrchestrationWorkerServer(
...args: Calls.ResolveOrchestrationWorkerServerArgs
): Calls.ResolveOrchestrationWorkerServerResult
@@ -1,5 +1,6 @@
import type { OrcaRuntimeWithMarkPtyLivenessUnverifiable as MarkPtyLivenessUnverifiable } from '../terminal/layout/terminal-input-floor'
import type { OrcaRuntimeWithStopTerminalsForWorktree as StopTerminalsForWorktree } from '../terminal/stop/worktree-terminal-stop'
import type { OrcaRuntimeWithStopTerminalsForProject as StopTerminalsForProject } from '../terminal/stop/project-terminal-stop'
import type { OrcaRuntimeWithGetRuntimeId as GetRuntimeId } from '../workspaces/session-hosts/get-runtime-id'
import type { OrcaRuntimeWithHasExactPersistedTerminalSurfaceIdentity as HasExactPersistedTerminalSurfaceIdentity } from '../workspaces/persisted-surfaces/has-exact-persisted-terminal-surface-identity'
import type { OrcaRuntimeWithAutomationOperations as AutomationOperations } from '../workspaces/persisted-surfaces/automation-operations'
@@ -20,6 +21,12 @@ export type StopTerminalsForWorktreeArgs = Parameters<
export type StopTerminalsForWorktreeResult = ReturnType<
StopTerminalsForWorktree['stopTerminalsForWorktree']
>
export type StopTerminalsForProjectArgs = Parameters<
StopTerminalsForProject['stopTerminalsForProject']
>
export type StopTerminalsForProjectResult = ReturnType<
StopTerminalsForProject['stopTerminalsForProject']
>
export type ResolveOrchestrationWorkerServerArgs = Parameters<
GetRuntimeId['resolveOrchestrationWorkerServer']
>
@@ -314,31 +314,28 @@ export abstract class OrcaRuntimeWithPreservedBranchCleanup extends OrcaRuntimeW
}
})
protected readonly nestedRepoImport = new RuntimeNestedRepoImport({
private readonly repositoryCatalogDeps = {
getStore: () => this.store,
invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(),
invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId),
invalidateWorktreeScan: (repoId: string) => this.invalidateWorktreeScanCacheForRepo(repoId),
notifyReposChanged: () => this.notifyReposChanged()
})
}
protected readonly repositoryRegistrations = new RuntimeRepositoryRegistrationController({
getStore: () => this.store,
invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(),
invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId),
notifyReposChanged: () => this.notifyReposChanged()
})
protected readonly nestedRepoImport = new RuntimeNestedRepoImport(this.repositoryCatalogDeps)
protected readonly repositoryClones = new RuntimeRepositoryCloneController({
getStore: () => this.store,
invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(),
invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId),
notifyReposChanged: () => this.notifyReposChanged()
})
protected readonly repositoryRegistrations = new RuntimeRepositoryRegistrationController(
this.repositoryCatalogDeps
)
protected readonly repositoryClones = new RuntimeRepositoryCloneController(
this.repositoryCatalogDeps
)
protected readonly repositorySettings = new RuntimeRepositorySettingsController({
getStore: () => this.store,
resolveRepo: (selector) => this.resolveRepoSelector(selector),
forgetTerminalTopology: (repoId) => this.terminalTopologyRevisionByRepoId.delete(repoId),
stopProjectTerminals: (repo, deadline) => this.stopTerminalsForProject(repo, { deadline }),
invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(),
invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId),
notifyReposChanged: () => this.notifyReposChanged()
@@ -0,0 +1,212 @@
/**
* The host owns a project's shells, so removing the project stops them itself: a client only
* stops the workspaces it has loaded, and an unhydrated or hidden one would keep running.
*/
import { describe, expect, it, vi } from 'vitest'
import type { Repo } from '../../shared/repo-types'
import { OrcaRuntimeService } from './orca-runtime-test-mocks.spec'
import { TEST_REPO_ID, TEST_WORKTREE_ID, store } from './orca-runtime-test-fixtures.spec'
const HIDDEN_WORKTREE_ID = `${TEST_REPO_ID}::/tmp/hidden-worktree`
const OTHER_WORKTREE_ID = 'repo-2::/tmp/other'
function createRemovableRuntime(onFirstStop?: () => void): {
runtime: InstanceType<typeof OrcaRuntimeService>
stopAndWait: ReturnType<typeof vi.fn>
removeProjectForHost: ReturnType<typeof vi.fn>
stoppedBeforeRemoval: string[]
} {
const removeProjectForHost = vi.fn()
const stoppedBeforeRemoval: string[] = []
let firstStop = true
const stopAndWait = vi.fn(async (ptyId: string) => {
if (removeProjectForHost.mock.calls.length === 0) {
stoppedBeforeRemoval.push(ptyId)
}
if (firstStop) {
firstStop = false
onFirstStop?.()
}
if (ptyId === 'pty-unverifiable') {
return false
}
// Like the provider: a stopped PTY exits and leaves the runtime's table.
runtime.onPtyExit(ptyId, 0)
return true
})
const runtime = new OrcaRuntimeService({ ...store, removeProjectForHost })
runtime.setPtyController({
write: () => true,
kill: vi.fn(() => false),
stopAndWait,
getForegroundProcess: async () => null
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
return { runtime, stopAndWait, removeProjectForHost, stoppedBeforeRemoval }
}
describe('project removal on the execution host', () => {
it('stops every terminal of the project, including workspaces no client listed', async () => {
const { runtime, removeProjectForHost, stoppedBeforeRemoval } = createRemovableRuntime()
runtime.registerPty('pty-visible', TEST_WORKTREE_ID, null)
runtime.registerPty('pty-hidden', HIDDEN_WORKTREE_ID, null)
runtime.registerPty('pty-other-project', OTHER_WORKTREE_ID, null)
runtime.registerPty('pty-ssh-twin', HIDDEN_WORKTREE_ID, 'ssh-target')
await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true })
expect(stoppedBeforeRemoval.sort()).toEqual(['pty-hidden', 'pty-visible'])
expect(removeProjectForHost).toHaveBeenCalledWith(TEST_REPO_ID, 'local')
})
it('stops native shells of a project a paired client registered under a runtime stamp', async () => {
const removeProjectForHost = vi.fn()
const stopAndWait = vi.fn(async () => true)
const stampedRepo: Repo = {
...store.getRepo(TEST_REPO_ID)!,
executionHostId: 'runtime:env-self'
}
const runtime = new OrcaRuntimeService({
...store,
getRepo: (id: string) => (id === TEST_REPO_ID ? stampedRepo : undefined),
getRepos: () => [stampedRepo],
removeProjectForHost
})
runtime.setPtyController({
write: () => true,
kill: vi.fn(() => false),
stopAndWait,
getForegroundProcess: async () => null
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
runtime.registerPty('pty-native', TEST_WORKTREE_ID, null)
await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true })
expect(stopAndWait).toHaveBeenCalledWith('pty-native', expect.anything())
})
it('fences a runtime-stamped project with a nested SSH connection to that connection', async () => {
const removeProjectForHost = vi.fn()
const stopAndWait = vi.fn(async (_ptyId: string) => true)
const sshRepo: Repo = {
...store.getRepo(TEST_REPO_ID)!,
executionHostId: 'runtime:env-self',
connectionId: 'ssh-target'
}
const runtime = new OrcaRuntimeService({
...store,
getRepo: (id: string) => (id === TEST_REPO_ID ? sshRepo : undefined),
getRepos: () => [sshRepo],
removeProjectForHost
})
runtime.setPtyController({
write: () => true,
kill: vi.fn(() => false),
stopAndWait,
getForegroundProcess: async () => null
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
runtime.registerPty('pty-native-sibling', TEST_WORKTREE_ID, null)
runtime.registerPty('pty-over-ssh', TEST_WORKTREE_ID, 'ssh-target')
await runtime.removeProject(TEST_REPO_ID)
expect([...new Set(stopAndWait.mock.calls.map(([ptyId]) => ptyId))]).toEqual(['pty-over-ssh'])
})
it('waits out a spawn already in flight and stops what it registers', async () => {
const { runtime, stopAndWait } = createRemovableRuntime()
runtime.registerPty('pty-early', TEST_WORKTREE_ID, null)
const releaseSpawn = await runtime.acquireWorktreeTerminalSpawn(TEST_WORKTREE_ID)
const removal = runtime.removeProject(TEST_REPO_ID)
await new Promise((resolve) => setTimeout(resolve, 0))
runtime.registerPty('pty-in-flight', TEST_WORKTREE_ID, null)
releaseSpawn()
await removal
expect(stopAndWait.mock.calls.map(([ptyId]) => ptyId)).toContain('pty-in-flight')
})
it('refuses a spawn that reaches its lease after the project is gone', async () => {
let removed = false
const repo = store.getRepo(TEST_REPO_ID)!
const runtime = new OrcaRuntimeService({
...store,
getRepo: (id: string) => (id === TEST_REPO_ID && !removed ? repo : undefined),
getRepos: () => (removed ? [] : [repo]),
removeProjectForHost: () => {
removed = true
}
})
runtime.setPtyController({
write: () => true,
kill: vi.fn(() => false),
stopAndWait: vi.fn(async () => true),
getForegroundProcess: async () => null
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
await runtime.removeProject(TEST_REPO_ID)
await expect(runtime.acquireWorktreeTerminalSpawn(TEST_WORKTREE_ID)).rejects.toThrow(
'repo_not_found'
)
})
it('stops nothing when a same-id sibling row would claim the same PTYs', async () => {
const removeProjectForHost = vi.fn()
const stopAndWait = vi.fn(async (_ptyId: string) => true)
const localRow = store.getRepo(TEST_REPO_ID)!
const foreignRow: Repo = {
...localRow,
path: '/srv/foreign-copy',
executionHostId: 'runtime:env-other'
}
const runtime = new OrcaRuntimeService({
...store,
getRepos: () => [localRow, foreignRow],
removeProjectForHost
})
runtime.setPtyController({
write: () => true,
kill: vi.fn(() => false),
stopAndWait,
getForegroundProcess: async () => null
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
runtime.registerPty('pty-local-sibling', TEST_WORKTREE_ID, null)
await runtime.removeProject('path:/srv/foreign-copy')
expect(stopAndWait).not.toHaveBeenCalled()
expect(removeProjectForHost).toHaveBeenCalledTimes(1)
})
it('stops a terminal spawned while the first sweep ran', async () => {
const { runtime, stopAndWait } = createRemovableRuntime(() =>
runtime.registerPty('pty-late', TEST_WORKTREE_ID, null)
)
runtime.registerPty('pty-early', TEST_WORKTREE_ID, null)
await runtime.removeProject(TEST_REPO_ID)
expect(stopAndWait.mock.calls.map(([ptyId]) => ptyId)).toContain('pty-late')
})
it('still removes the project when a stop cannot be verified', async () => {
const { runtime, stopAndWait, removeProjectForHost } = createRemovableRuntime()
runtime.registerPty('pty-unverifiable', TEST_WORKTREE_ID, null)
await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true })
expect(stopAndWait).toHaveBeenCalledWith('pty-unverifiable', expect.anything())
expect(removeProjectForHost).toHaveBeenCalledTimes(1)
})
})
@@ -8,10 +8,14 @@ import { prepareLocalWorktreeRootForRepo } from '../worktree-root-preparation'
import type { RuntimeStore } from './runtime-store-contract'
import { assertWslMoveRepoWritable } from '../persistence/moving-wsl-projects/wsl-move-scope-guard'
const PROJECT_TERMINAL_STOP_BUDGET_MS = 10_000
type RuntimeRepositorySettingsDependencies = {
getStore: () => RuntimeStore | null
resolveRepo: (selector: string) => Promise<Repo>
forgetTerminalTopology: (repoId: string) => void
/** Optional so test doubles and older wiring keep removing without it. */
stopProjectTerminals?: (repo: Repo, deadline: number) => Promise<unknown>
invalidateResolvedWorktrees: () => void
invalidateWorktreeScan: (repoId: string) => void
notifyReposChanged: () => void
@@ -123,7 +127,21 @@ export class RuntimeRepositorySettingsController {
const repo = await this.deps.resolveRepo(repoSelector)
const hostId = getRepoExecutionHostId(repo)
assertWslMoveRepoWritable(store, repo)
// Why here: the host owns these shells; a client only stops the workspaces it has loaded,
// and older clients stop none. Bounded so repo.rm still answers inside the client's timeout.
const deadline = Date.now() + PROJECT_TERMINAL_STOP_BUDGET_MS
const stopProjectTerminals = async (): Promise<void> => {
await this.deps.stopProjectTerminals?.(repo, deadline).catch((error: unknown) => {
console.warn('[runtime] failed to stop project terminals on removal', error)
})
}
await stopProjectTerminals()
// Why again: a WSL move may have fenced the project while the stop was awaited.
assertWslMoveRepoWritable(store, repo)
store.removeProjectForHost(repo.id, hostId)
// Why again: a terminal spawned while the first stop ran is past that sweep's snapshot, and
// nothing can spawn into the project once it is gone.
await stopProjectTerminals()
this.deps.forgetTerminalTopology(repo.id)
this.deps.invalidateResolvedWorktrees()
this.deps.invalidateWorktreeScan(repo.id)
@@ -1,11 +1,11 @@
import type { OrcaRuntimeWithTransitionGraphReloadToTerminalState as TransitionGraphReloadToTerminalStateMethods } from './session-graph-authority'
import type { OrcaRuntimeWithBindPtyIncarnationHandle as BindPtyIncarnationHandleMethods } from '../terminal/identity/terminal-handle-bindings'
import type { OrcaRuntimeWithResolveWaiter as ResolveWaiterMethods } from '../orca-runtime-resolve-waiter'
import { OrcaRuntimeWithStopExactTerminalsForWorktree } from '../terminal/stop/worktree-terminal-stop-targets'
import { OrcaRuntimeWithStopTerminalsForProject } from '../terminal/stop/project-terminal-stop'
import type { RuntimeRendererReloadFence } from '../orca-runtime-core'
import { HEADLESS_RUNTIME_WINDOW_ID } from '../../../shared/runtime-types'
export abstract class OrcaRuntimeWithHasTerminalsForWorktree extends OrcaRuntimeWithStopExactTerminalsForWorktree {
export abstract class OrcaRuntimeWithHasTerminalsForWorktree extends OrcaRuntimeWithStopTerminalsForProject {
protected abstract assertStableReadyGraph(
...args: Parameters<TransitionGraphReloadToTerminalStateMethods['assertStableReadyGraph']>
): ReturnType<TransitionGraphReloadToTerminalStateMethods['assertStableReadyGraph']>
@@ -0,0 +1,132 @@
import { OrcaRuntimeWithStopExactTerminalsForWorktree } from './worktree-terminal-stop-targets'
import { getRepoExecutionHostId, getRepoSshConnectionId } from '../../../../shared/execution-host'
import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id'
import type { Repo } from '../../../../shared/repo-types'
const PROJECT_INVENTORY_REFRESH_BUDGET_MS = 3_000
/** The value, or null once the deadline passes first. */
async function raceDeadline<T>(value: Promise<T>, deadline: number | undefined): Promise<T | null> {
if (deadline === undefined) {
return await value
}
let timer: ReturnType<typeof setTimeout> | undefined
try {
return await Promise.race([
value,
new Promise<null>((resolve) => {
timer = setTimeout(() => resolve(null), Math.max(0, deadline - Date.now()))
})
])
} finally {
clearTimeout(timer)
}
}
/** Distinct worktree ids of a project that still hold a pane or PTY on this host. */
export function listProjectTerminalWorktreeIds(
repoId: string,
leaves: ReadonlyMap<string, { worktreeId: string; ptyId: string | null }>,
ptys: ReadonlyMap<string, { worktreeId: string }>
): string[] {
const worktreeIds = new Set<string>()
for (const leaf of leaves.values()) {
if (leaf.ptyId && getRepoIdFromWorktreeId(leaf.worktreeId) === repoId) {
worktreeIds.add(leaf.worktreeId)
}
}
for (const pty of ptys.values()) {
if (getRepoIdFromWorktreeId(pty.worktreeId) === repoId) {
worktreeIds.add(pty.worktreeId)
}
}
return [...worktreeIds]
}
export abstract class OrcaRuntimeWithStopTerminalsForProject extends OrcaRuntimeWithStopExactTerminalsForWorktree {
/** Projects this host removed; ids are UUIDs, so a remaining row means a same-id sibling. */
private readonly removedProjectIds = new Set<string>()
override async acquireWorktreeTerminalSpawn(
worktreeId?: string,
opts: { refuseSleptWorktree?: boolean } = {}
): Promise<() => void> {
const release = await super.acquireWorktreeTerminalSpawn(worktreeId, opts)
const repoId = worktreeId ? getRepoIdFromWorktreeId(worktreeId) : null
// Why under the lease: a spawn that resolved its workspace before removal queues behind the sweep.
if (repoId && this.removedProjectIds.has(repoId) && !this.store?.getRepo?.(repoId)) {
release()
throw new Error('repo_not_found')
}
return release
}
/**
* Stops every terminal this host runs for a project, found through its own PTY table rather
* than a client's workspace list, which can be unhydrated or hide workspaces.
*/
async stopTerminalsForProject(
repo: Pick<Repo, 'id' | 'connectionId' | 'executionHostId'>,
options: { deadline?: number } = {}
): Promise<{ stopped: number }> {
// Why: after a restart, surviving daemon sessions are in the provider inventory before ptysById.
// Bounded so a slow worktree scan cannot spend the budget the known PTYs' stops need.
const refreshDeadline = Math.min(
options.deadline ?? Infinity,
Date.now() + PROJECT_INVENTORY_REFRESH_BUDGET_MS
)
const projectWorktreeIds = new Set<string>()
const refresh = (async () => {
const resolvedWorktrees = [...(await this.getResolvedWorktreeMap()).values()]
for (const worktree of resolvedWorktrees) {
if (worktree.repoId === repo.id) {
projectWorktreeIds.add(worktree.id)
}
}
await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees, null, refreshDeadline)
})().catch((error: unknown) => {
console.warn('[runtime] project terminal inventory refresh failed', error)
})
await raceDeadline(refresh, refreshDeadline)
// Why not the row's stamp: a `runtime:` stamp is how paired clients address a project that runs
// here, natively or over its nested SSH connection.
const fence = { resolvedConnectionId: getRepoSshConnectionId(repo) }
this.removedProjectIds.add(repo.id)
// Why: a same-id sibling row with the same fence owns PTYs this table cannot tell apart.
const siblingSharesFence = (this.store?.getRepos?.() ?? []).some(
(row) =>
row.id === repo.id &&
getRepoExecutionHostId(row) !== getRepoExecutionHostId(repo) &&
getRepoSshConnectionId(row) === fence.resolvedConnectionId
)
if (siblingSharesFence) {
return { stopped: 0 }
}
let stopped = 0
for (const id of listProjectTerminalWorktreeIds(repo.id, this.leaves, this.ptysById)) {
projectWorktreeIds.add(id)
}
for (const id of projectWorktreeIds) {
const opts = { resolvedWorktreeId: id, ...fence, ...options }
// Why exclusive: it waits out a spawn that holds the shared lease, so its PTY is registered
// and stopped. Bounded by the deadline so repo.rm still answers; past it the stop is skipped.
// A timed-out waiter is dequeued by the lock, so it cannot hold later spawns back.
const release = await this.acquireWorktreeTerminalMutation(
id,
'exclusive',
options.deadline
).catch(() => null)
try {
// Why raced: a provider's own exit wait can outlast the budget that keeps repo.rm answering.
const result = await raceDeadline(
this.stopTerminalsForWorktree(`id:${id}`, opts),
options.deadline
)
stopped += result?.stopped ?? 0
} finally {
release?.()
}
}
return { stopped }
}
}