diff --git a/src/main/runtime/construction/runtime-construction-services.ts b/src/main/runtime/construction/runtime-construction-services.ts index 14327a368b7..c7fe2510fb8 100644 --- a/src/main/runtime/construction/runtime-construction-services.ts +++ b/src/main/runtime/construction/runtime-construction-services.ts @@ -56,6 +56,9 @@ export abstract class RuntimeConstructionServices { abstract stopTerminalsForWorktree( ...args: Calls.StopTerminalsForWorktreeArgs ): Calls.StopTerminalsForWorktreeResult + abstract stopTerminalsForProject( + ...args: Calls.StopTerminalsForProjectArgs + ): Calls.StopTerminalsForProjectResult abstract resolveOrchestrationWorkerServer( ...args: Calls.ResolveOrchestrationWorkerServerArgs ): Calls.ResolveOrchestrationWorkerServerResult diff --git a/src/main/runtime/construction/runtime-host-service-calls.ts b/src/main/runtime/construction/runtime-host-service-calls.ts index d354ec60aa5..c9728e72633 100644 --- a/src/main/runtime/construction/runtime-host-service-calls.ts +++ b/src/main/runtime/construction/runtime-host-service-calls.ts @@ -1,5 +1,6 @@ import type { OrcaRuntimeWithMarkPtyLivenessUnverifiable as MarkPtyLivenessUnverifiable } from '../terminal/layout/terminal-input-floor' import type { OrcaRuntimeWithStopTerminalsForWorktree as StopTerminalsForWorktree } from '../terminal/stop/worktree-terminal-stop' +import type { OrcaRuntimeWithStopTerminalsForProject as StopTerminalsForProject } from '../terminal/stop/project-terminal-stop' import type { OrcaRuntimeWithGetRuntimeId as GetRuntimeId } from '../workspaces/session-hosts/get-runtime-id' import type { OrcaRuntimeWithHasExactPersistedTerminalSurfaceIdentity as HasExactPersistedTerminalSurfaceIdentity } from '../workspaces/persisted-surfaces/has-exact-persisted-terminal-surface-identity' import type { OrcaRuntimeWithAutomationOperations as AutomationOperations } from '../workspaces/persisted-surfaces/automation-operations' @@ -20,6 +21,12 @@ export type StopTerminalsForWorktreeArgs = Parameters< export type StopTerminalsForWorktreeResult = ReturnType< StopTerminalsForWorktree['stopTerminalsForWorktree'] > +export type StopTerminalsForProjectArgs = Parameters< + StopTerminalsForProject['stopTerminalsForProject'] +> +export type StopTerminalsForProjectResult = ReturnType< + StopTerminalsForProject['stopTerminalsForProject'] +> export type ResolveOrchestrationWorkerServerArgs = Parameters< GetRuntimeId['resolveOrchestrationWorkerServer'] > diff --git a/src/main/runtime/construction/runtime-host-services.ts b/src/main/runtime/construction/runtime-host-services.ts index e7b88397afc..3ca424730ad 100644 --- a/src/main/runtime/construction/runtime-host-services.ts +++ b/src/main/runtime/construction/runtime-host-services.ts @@ -314,31 +314,28 @@ export abstract class OrcaRuntimeWithPreservedBranchCleanup extends OrcaRuntimeW } }) - protected readonly nestedRepoImport = new RuntimeNestedRepoImport({ + private readonly repositoryCatalogDeps = { getStore: () => this.store, invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(), - invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId), + invalidateWorktreeScan: (repoId: string) => this.invalidateWorktreeScanCacheForRepo(repoId), notifyReposChanged: () => this.notifyReposChanged() - }) + } - protected readonly repositoryRegistrations = new RuntimeRepositoryRegistrationController({ - getStore: () => this.store, - invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(), - invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId), - notifyReposChanged: () => this.notifyReposChanged() - }) + protected readonly nestedRepoImport = new RuntimeNestedRepoImport(this.repositoryCatalogDeps) - protected readonly repositoryClones = new RuntimeRepositoryCloneController({ - getStore: () => this.store, - invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(), - invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId), - notifyReposChanged: () => this.notifyReposChanged() - }) + protected readonly repositoryRegistrations = new RuntimeRepositoryRegistrationController( + this.repositoryCatalogDeps + ) + + protected readonly repositoryClones = new RuntimeRepositoryCloneController( + this.repositoryCatalogDeps + ) protected readonly repositorySettings = new RuntimeRepositorySettingsController({ getStore: () => this.store, resolveRepo: (selector) => this.resolveRepoSelector(selector), forgetTerminalTopology: (repoId) => this.terminalTopologyRevisionByRepoId.delete(repoId), + stopProjectTerminals: (repo, deadline) => this.stopTerminalsForProject(repo, { deadline }), invalidateResolvedWorktrees: () => this.invalidateResolvedWorktreeCache(), invalidateWorktreeScan: (repoId) => this.invalidateWorktreeScanCacheForRepo(repoId), notifyReposChanged: () => this.notifyReposChanged() diff --git a/src/main/runtime/project-removal-terminal-stop.test.ts b/src/main/runtime/project-removal-terminal-stop.test.ts new file mode 100644 index 00000000000..3858a739f73 --- /dev/null +++ b/src/main/runtime/project-removal-terminal-stop.test.ts @@ -0,0 +1,212 @@ +/** + * The host owns a project's shells, so removing the project stops them itself: a client only + * stops the workspaces it has loaded, and an unhydrated or hidden one would keep running. + */ +import { describe, expect, it, vi } from 'vitest' +import type { Repo } from '../../shared/repo-types' +import { OrcaRuntimeService } from './orca-runtime-test-mocks.spec' +import { TEST_REPO_ID, TEST_WORKTREE_ID, store } from './orca-runtime-test-fixtures.spec' + +const HIDDEN_WORKTREE_ID = `${TEST_REPO_ID}::/tmp/hidden-worktree` +const OTHER_WORKTREE_ID = 'repo-2::/tmp/other' + +function createRemovableRuntime(onFirstStop?: () => void): { + runtime: InstanceType + stopAndWait: ReturnType + removeProjectForHost: ReturnType + stoppedBeforeRemoval: string[] +} { + const removeProjectForHost = vi.fn() + const stoppedBeforeRemoval: string[] = [] + let firstStop = true + const stopAndWait = vi.fn(async (ptyId: string) => { + if (removeProjectForHost.mock.calls.length === 0) { + stoppedBeforeRemoval.push(ptyId) + } + if (firstStop) { + firstStop = false + onFirstStop?.() + } + if (ptyId === 'pty-unverifiable') { + return false + } + // Like the provider: a stopped PTY exits and leaves the runtime's table. + runtime.onPtyExit(ptyId, 0) + return true + }) + const runtime = new OrcaRuntimeService({ ...store, removeProjectForHost }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + return { runtime, stopAndWait, removeProjectForHost, stoppedBeforeRemoval } +} + +describe('project removal on the execution host', () => { + it('stops every terminal of the project, including workspaces no client listed', async () => { + const { runtime, removeProjectForHost, stoppedBeforeRemoval } = createRemovableRuntime() + runtime.registerPty('pty-visible', TEST_WORKTREE_ID, null) + runtime.registerPty('pty-hidden', HIDDEN_WORKTREE_ID, null) + runtime.registerPty('pty-other-project', OTHER_WORKTREE_ID, null) + runtime.registerPty('pty-ssh-twin', HIDDEN_WORKTREE_ID, 'ssh-target') + + await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true }) + + expect(stoppedBeforeRemoval.sort()).toEqual(['pty-hidden', 'pty-visible']) + expect(removeProjectForHost).toHaveBeenCalledWith(TEST_REPO_ID, 'local') + }) + + it('stops native shells of a project a paired client registered under a runtime stamp', async () => { + const removeProjectForHost = vi.fn() + const stopAndWait = vi.fn(async () => true) + const stampedRepo: Repo = { + ...store.getRepo(TEST_REPO_ID)!, + executionHostId: 'runtime:env-self' + } + const runtime = new OrcaRuntimeService({ + ...store, + getRepo: (id: string) => (id === TEST_REPO_ID ? stampedRepo : undefined), + getRepos: () => [stampedRepo], + removeProjectForHost + }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + runtime.registerPty('pty-native', TEST_WORKTREE_ID, null) + + await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true }) + + expect(stopAndWait).toHaveBeenCalledWith('pty-native', expect.anything()) + }) + + it('fences a runtime-stamped project with a nested SSH connection to that connection', async () => { + const removeProjectForHost = vi.fn() + const stopAndWait = vi.fn(async (_ptyId: string) => true) + const sshRepo: Repo = { + ...store.getRepo(TEST_REPO_ID)!, + executionHostId: 'runtime:env-self', + connectionId: 'ssh-target' + } + const runtime = new OrcaRuntimeService({ + ...store, + getRepo: (id: string) => (id === TEST_REPO_ID ? sshRepo : undefined), + getRepos: () => [sshRepo], + removeProjectForHost + }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + runtime.registerPty('pty-native-sibling', TEST_WORKTREE_ID, null) + runtime.registerPty('pty-over-ssh', TEST_WORKTREE_ID, 'ssh-target') + + await runtime.removeProject(TEST_REPO_ID) + + expect([...new Set(stopAndWait.mock.calls.map(([ptyId]) => ptyId))]).toEqual(['pty-over-ssh']) + }) + + it('waits out a spawn already in flight and stops what it registers', async () => { + const { runtime, stopAndWait } = createRemovableRuntime() + runtime.registerPty('pty-early', TEST_WORKTREE_ID, null) + const releaseSpawn = await runtime.acquireWorktreeTerminalSpawn(TEST_WORKTREE_ID) + + const removal = runtime.removeProject(TEST_REPO_ID) + await new Promise((resolve) => setTimeout(resolve, 0)) + runtime.registerPty('pty-in-flight', TEST_WORKTREE_ID, null) + releaseSpawn() + await removal + + expect(stopAndWait.mock.calls.map(([ptyId]) => ptyId)).toContain('pty-in-flight') + }) + + it('refuses a spawn that reaches its lease after the project is gone', async () => { + let removed = false + const repo = store.getRepo(TEST_REPO_ID)! + const runtime = new OrcaRuntimeService({ + ...store, + getRepo: (id: string) => (id === TEST_REPO_ID && !removed ? repo : undefined), + getRepos: () => (removed ? [] : [repo]), + removeProjectForHost: () => { + removed = true + } + }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait: vi.fn(async () => true), + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + + await runtime.removeProject(TEST_REPO_ID) + + await expect(runtime.acquireWorktreeTerminalSpawn(TEST_WORKTREE_ID)).rejects.toThrow( + 'repo_not_found' + ) + }) + + it('stops nothing when a same-id sibling row would claim the same PTYs', async () => { + const removeProjectForHost = vi.fn() + const stopAndWait = vi.fn(async (_ptyId: string) => true) + const localRow = store.getRepo(TEST_REPO_ID)! + const foreignRow: Repo = { + ...localRow, + path: '/srv/foreign-copy', + executionHostId: 'runtime:env-other' + } + const runtime = new OrcaRuntimeService({ + ...store, + getRepos: () => [localRow, foreignRow], + removeProjectForHost + }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => false), + stopAndWait, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + runtime.registerPty('pty-local-sibling', TEST_WORKTREE_ID, null) + + await runtime.removeProject('path:/srv/foreign-copy') + + expect(stopAndWait).not.toHaveBeenCalled() + expect(removeProjectForHost).toHaveBeenCalledTimes(1) + }) + + it('stops a terminal spawned while the first sweep ran', async () => { + const { runtime, stopAndWait } = createRemovableRuntime(() => + runtime.registerPty('pty-late', TEST_WORKTREE_ID, null) + ) + runtime.registerPty('pty-early', TEST_WORKTREE_ID, null) + + await runtime.removeProject(TEST_REPO_ID) + + expect(stopAndWait.mock.calls.map(([ptyId]) => ptyId)).toContain('pty-late') + }) + + it('still removes the project when a stop cannot be verified', async () => { + const { runtime, stopAndWait, removeProjectForHost } = createRemovableRuntime() + runtime.registerPty('pty-unverifiable', TEST_WORKTREE_ID, null) + + await expect(runtime.removeProject(TEST_REPO_ID)).resolves.toEqual({ removed: true }) + + expect(stopAndWait).toHaveBeenCalledWith('pty-unverifiable', expect.anything()) + expect(removeProjectForHost).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/runtime/runtime-repository-settings-controller.ts b/src/main/runtime/runtime-repository-settings-controller.ts index d6abd6aeb18..083e9ede432 100644 --- a/src/main/runtime/runtime-repository-settings-controller.ts +++ b/src/main/runtime/runtime-repository-settings-controller.ts @@ -8,10 +8,14 @@ import { prepareLocalWorktreeRootForRepo } from '../worktree-root-preparation' import type { RuntimeStore } from './runtime-store-contract' import { assertWslMoveRepoWritable } from '../persistence/moving-wsl-projects/wsl-move-scope-guard' +const PROJECT_TERMINAL_STOP_BUDGET_MS = 10_000 + type RuntimeRepositorySettingsDependencies = { getStore: () => RuntimeStore | null resolveRepo: (selector: string) => Promise forgetTerminalTopology: (repoId: string) => void + /** Optional so test doubles and older wiring keep removing without it. */ + stopProjectTerminals?: (repo: Repo, deadline: number) => Promise invalidateResolvedWorktrees: () => void invalidateWorktreeScan: (repoId: string) => void notifyReposChanged: () => void @@ -123,7 +127,21 @@ export class RuntimeRepositorySettingsController { const repo = await this.deps.resolveRepo(repoSelector) const hostId = getRepoExecutionHostId(repo) assertWslMoveRepoWritable(store, repo) + // Why here: the host owns these shells; a client only stops the workspaces it has loaded, + // and older clients stop none. Bounded so repo.rm still answers inside the client's timeout. + const deadline = Date.now() + PROJECT_TERMINAL_STOP_BUDGET_MS + const stopProjectTerminals = async (): Promise => { + await this.deps.stopProjectTerminals?.(repo, deadline).catch((error: unknown) => { + console.warn('[runtime] failed to stop project terminals on removal', error) + }) + } + await stopProjectTerminals() + // Why again: a WSL move may have fenced the project while the stop was awaited. + assertWslMoveRepoWritable(store, repo) store.removeProjectForHost(repo.id, hostId) + // Why again: a terminal spawned while the first stop ran is past that sweep's snapshot, and + // nothing can spawn into the project once it is gone. + await stopProjectTerminals() this.deps.forgetTerminalTopology(repo.id) this.deps.invalidateResolvedWorktrees() this.deps.invalidateWorktreeScan(repo.id) diff --git a/src/main/runtime/session-graph/session-graph-reload-lifecycle.ts b/src/main/runtime/session-graph/session-graph-reload-lifecycle.ts index c7dbbd4ac49..9cd412cd9dd 100644 --- a/src/main/runtime/session-graph/session-graph-reload-lifecycle.ts +++ b/src/main/runtime/session-graph/session-graph-reload-lifecycle.ts @@ -1,11 +1,11 @@ import type { OrcaRuntimeWithTransitionGraphReloadToTerminalState as TransitionGraphReloadToTerminalStateMethods } from './session-graph-authority' import type { OrcaRuntimeWithBindPtyIncarnationHandle as BindPtyIncarnationHandleMethods } from '../terminal/identity/terminal-handle-bindings' import type { OrcaRuntimeWithResolveWaiter as ResolveWaiterMethods } from '../orca-runtime-resolve-waiter' -import { OrcaRuntimeWithStopExactTerminalsForWorktree } from '../terminal/stop/worktree-terminal-stop-targets' +import { OrcaRuntimeWithStopTerminalsForProject } from '../terminal/stop/project-terminal-stop' import type { RuntimeRendererReloadFence } from '../orca-runtime-core' import { HEADLESS_RUNTIME_WINDOW_ID } from '../../../shared/runtime-types' -export abstract class OrcaRuntimeWithHasTerminalsForWorktree extends OrcaRuntimeWithStopExactTerminalsForWorktree { +export abstract class OrcaRuntimeWithHasTerminalsForWorktree extends OrcaRuntimeWithStopTerminalsForProject { protected abstract assertStableReadyGraph( ...args: Parameters ): ReturnType diff --git a/src/main/runtime/terminal/stop/project-terminal-stop.ts b/src/main/runtime/terminal/stop/project-terminal-stop.ts new file mode 100644 index 00000000000..8f2fdf0302e --- /dev/null +++ b/src/main/runtime/terminal/stop/project-terminal-stop.ts @@ -0,0 +1,132 @@ +import { OrcaRuntimeWithStopExactTerminalsForWorktree } from './worktree-terminal-stop-targets' +import { getRepoExecutionHostId, getRepoSshConnectionId } from '../../../../shared/execution-host' +import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id' +import type { Repo } from '../../../../shared/repo-types' + +const PROJECT_INVENTORY_REFRESH_BUDGET_MS = 3_000 + +/** The value, or null once the deadline passes first. */ +async function raceDeadline(value: Promise, deadline: number | undefined): Promise { + if (deadline === undefined) { + return await value + } + let timer: ReturnType | undefined + try { + return await Promise.race([ + value, + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), Math.max(0, deadline - Date.now())) + }) + ]) + } finally { + clearTimeout(timer) + } +} + +/** Distinct worktree ids of a project that still hold a pane or PTY on this host. */ +export function listProjectTerminalWorktreeIds( + repoId: string, + leaves: ReadonlyMap, + ptys: ReadonlyMap +): string[] { + const worktreeIds = new Set() + for (const leaf of leaves.values()) { + if (leaf.ptyId && getRepoIdFromWorktreeId(leaf.worktreeId) === repoId) { + worktreeIds.add(leaf.worktreeId) + } + } + for (const pty of ptys.values()) { + if (getRepoIdFromWorktreeId(pty.worktreeId) === repoId) { + worktreeIds.add(pty.worktreeId) + } + } + return [...worktreeIds] +} + +export abstract class OrcaRuntimeWithStopTerminalsForProject extends OrcaRuntimeWithStopExactTerminalsForWorktree { + /** Projects this host removed; ids are UUIDs, so a remaining row means a same-id sibling. */ + private readonly removedProjectIds = new Set() + + override async acquireWorktreeTerminalSpawn( + worktreeId?: string, + opts: { refuseSleptWorktree?: boolean } = {} + ): Promise<() => void> { + const release = await super.acquireWorktreeTerminalSpawn(worktreeId, opts) + const repoId = worktreeId ? getRepoIdFromWorktreeId(worktreeId) : null + // Why under the lease: a spawn that resolved its workspace before removal queues behind the sweep. + if (repoId && this.removedProjectIds.has(repoId) && !this.store?.getRepo?.(repoId)) { + release() + throw new Error('repo_not_found') + } + return release + } + + /** + * Stops every terminal this host runs for a project, found through its own PTY table rather + * than a client's workspace list, which can be unhydrated or hide workspaces. + */ + async stopTerminalsForProject( + repo: Pick, + options: { deadline?: number } = {} + ): Promise<{ stopped: number }> { + // Why: after a restart, surviving daemon sessions are in the provider inventory before ptysById. + // Bounded so a slow worktree scan cannot spend the budget the known PTYs' stops need. + const refreshDeadline = Math.min( + options.deadline ?? Infinity, + Date.now() + PROJECT_INVENTORY_REFRESH_BUDGET_MS + ) + const projectWorktreeIds = new Set() + const refresh = (async () => { + const resolvedWorktrees = [...(await this.getResolvedWorktreeMap()).values()] + for (const worktree of resolvedWorktrees) { + if (worktree.repoId === repo.id) { + projectWorktreeIds.add(worktree.id) + } + } + await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees, null, refreshDeadline) + })().catch((error: unknown) => { + console.warn('[runtime] project terminal inventory refresh failed', error) + }) + await raceDeadline(refresh, refreshDeadline) + // Why not the row's stamp: a `runtime:` stamp is how paired clients address a project that runs + // here, natively or over its nested SSH connection. + const fence = { resolvedConnectionId: getRepoSshConnectionId(repo) } + this.removedProjectIds.add(repo.id) + // Why: a same-id sibling row with the same fence owns PTYs this table cannot tell apart. + const siblingSharesFence = (this.store?.getRepos?.() ?? []).some( + (row) => + row.id === repo.id && + getRepoExecutionHostId(row) !== getRepoExecutionHostId(repo) && + getRepoSshConnectionId(row) === fence.resolvedConnectionId + ) + if (siblingSharesFence) { + return { stopped: 0 } + } + let stopped = 0 + for (const id of listProjectTerminalWorktreeIds(repo.id, this.leaves, this.ptysById)) { + projectWorktreeIds.add(id) + } + for (const id of projectWorktreeIds) { + const opts = { resolvedWorktreeId: id, ...fence, ...options } + // Why exclusive: it waits out a spawn that holds the shared lease, so its PTY is registered + // and stopped. Bounded by the deadline so repo.rm still answers; past it the stop is skipped. + // A timed-out waiter is dequeued by the lock, so it cannot hold later spawns back. + const release = await this.acquireWorktreeTerminalMutation( + id, + 'exclusive', + options.deadline + ).catch(() => null) + try { + // Why raced: a provider's own exit wait can outlast the budget that keeps repo.rm answering. + const result = await raceDeadline( + this.stopTerminalsForWorktree(`id:${id}`, opts), + options.deadline + ) + stopped += result?.stopped ?? 0 + } finally { + release?.() + } + } + return { stopped } + } +}