fix(mobile): preserve truncated folder deletion fences

This commit is contained in:
Brennan Benson
2026-07-31 19:11:34 -07:00
parent d2e128bc55
commit 889a4a920c
8 changed files with 310 additions and 31 deletions
+12 -3
View File
@@ -20,9 +20,10 @@ export function boundDeletedFolderTombstoneEvidence(
)
)
const retainedTabEntries = tabEntries.slice(Math.max(0, tabEntries.length - MAX_TAB_OWNERS))
const retainedHostIds = [
const candidateHostIds = [
...new Set([...tombstone.hostIds, ...retainedTabEntries.map(([hostId]) => hostId)])
].slice(-MAX_HOST_IDS)
]
const retainedHostIds = candidateHostIds.slice(-MAX_HOST_IDS)
const retainedHostIdSet = new Set(retainedHostIds)
const tabConnectionIdsByHostId: DeletedFolderWorkspaceSessionTombstone['tabConnectionIdsByHostId'] =
{}
@@ -35,7 +36,15 @@ export function boundDeletedFolderTombstoneEvidence(
[tabId]: connectionId
}
}
return { ...tombstone, hostIds: retainedHostIds, tabConnectionIdsByHostId }
return {
...tombstone,
evidenceTruncated:
tombstone.evidenceTruncated ||
tabEntries.length > MAX_TAB_OWNERS ||
candidateHostIds.length > MAX_HOST_IDS,
hostIds: retainedHostIds,
tabConnectionIdsByHostId
}
}
export function getDeletedFolderTombstoneEvictionKeys(
+58
View File
@@ -7901,6 +7901,64 @@ describe('registerPtyHandlers', () => {
})
})
it('rejects and cleans up a runtime spawn when its workspace inventory fence changes', async () => {
type RuntimeSpawnController = {
spawn(args: {
cols: number
rows: number
worktreeId: string
tabId: string
leafId: string
persistHostSessionBinding: true
acceptWorkspaceInventory: () => boolean
}): Promise<{ id: string }>
}
const provider = createAgentClaimProvider({
spawn: vi.fn(async () => ({ id: 'stale-workspace-spawn' }))
})
setLocalPtyProvider(provider as never)
const store = { persistPtyBinding: vi.fn() }
let controller: RuntimeSpawnController | null = null
const runtime = {
setPtyController: vi.fn((value) => {
controller = value
}),
createPreAllocatedTerminalHandle: vi.fn(() => 'term_stale_spawn'),
preAllocateHandleForPty: vi.fn(() => 'term_stale_spawn'),
registerPreAllocatedHandleForPty: vi.fn(),
registerPty: vi.fn(),
noteTerminalSpawnCommand: vi.fn(),
onPtySpawned: vi.fn(),
onPtyExit: vi.fn(),
onPtyData: vi.fn()
}
registerPtyHandlers(
mainWindow as never,
runtime as never,
undefined,
undefined,
undefined,
store as never
)
const acceptWorkspaceInventory = vi.fn().mockReturnValueOnce(true).mockReturnValue(false)
const leafId = '11111111-1111-4111-8111-111111111111'
await expect(
(controller as unknown as RuntimeSpawnController).spawn({
cols: 80,
rows: 24,
worktreeId: 'wt-stale-spawn',
tabId: 'tab-stale-spawn',
leafId,
persistHostSessionBinding: true,
acceptWorkspaceInventory
})
).rejects.toThrow('tab_not_found')
expect(provider.shutdown).toHaveBeenCalledWith('stale-workspace-spawn', { immediate: true })
expect(store.persistPtyBinding).not.toHaveBeenCalled()
})
it('reports lower-owner commit before rejecting an early-exited runtime incarnation', async () => {
const persistPtyBinding = vi.fn()
const onPtySpawnCommitted = vi.fn()
+24
View File
@@ -4148,6 +4148,25 @@ export function registerPtyHandlers(
throw new Error('client_disconnected')
}
}
const assertWorkspaceInventoryAccepted = (): void => {
if (args.acceptWorkspaceInventory?.() === false) {
throw new Error('tab_not_found')
}
}
const rejectStaleWorkspaceSpawn = async (candidate: PtySpawnResult): Promise<void> => {
if (args.acceptWorkspaceInventory?.() !== false) {
return
}
if (candidate.isReattach !== true) {
try {
await provider.shutdown(candidate.id, { immediate: true })
} catch (error) {
console.warn('[pty] failed to clean up stale workspace spawn:', error)
}
clearProviderPtyState(candidate.id)
}
throw new Error('tab_not_found')
}
if (args.agentSessionEnsure) {
// Why: daemon-backed claims can outlive this controller; import all
// proven owners before deciding that an identity is absent.
@@ -4169,7 +4188,9 @@ export function registerPtyHandlers(
surface: args.agentSessionEnsure.surface,
spawn: async () => {
assertClientStillConnected()
assertWorkspaceInventoryAccepted()
providerResult = await provider.spawn(spawnOptions)
await rejectStaleWorkspaceSpawn(providerResult)
rejectedRegistrationCandidate = providerResult
// Why: a successful lower-owner return proves physical work committed even if admission sees an early exit.
reportPtySpawnCommitted()
@@ -4212,9 +4233,12 @@ export function registerPtyHandlers(
incarnationId: ptyIncarnationById.get(ensured.owner.ptyId)
}
result.agentSessionEnsure = ensured
assertWorkspaceInventoryAccepted()
} else {
assertClientStillConnected()
assertWorkspaceInventoryAccepted()
result = await provider.spawn(spawnOptions)
await rejectStaleWorkspaceSpawn(result)
rejectedRegistrationCandidate = result
// Why: daemon/relay returns cross the physical commit boundary before controller admission.
reportPtySpawnCommitted()
+42
View File
@@ -5149,6 +5149,7 @@ describe('Store', () => {
'folder:partial-tombstone': {
connectionId: null,
deletedAt: undefined as unknown as number,
evidenceTruncated: undefined as unknown as boolean,
hostIds: ['local'],
tabConnectionIdsByHostId: {
local: { valid: null, invalid: 42 as unknown as string }
@@ -5173,6 +5174,7 @@ describe('Store', () => {
).toEqual({
connectionId: null,
deletedAt: expect.any(Number),
evidenceTruncated: false,
hostIds: ['local'],
tabConnectionIdsByHostId: { local: { valid: null } }
})
@@ -5192,6 +5194,7 @@ describe('Store', () => {
{
connectionId: null,
deletedAt: deletedAt + index,
evidenceTruncated: false,
hostIds: ['local'],
tabConnectionIdsByHostId: {}
}
@@ -5261,6 +5264,7 @@ describe('Store', () => {
'folder:expired-tombstone': {
connectionId: null,
deletedAt: Date.now() - 31 * 24 * 60 * 60 * 1000,
evidenceTruncated: false,
hostIds: ['local'],
tabConnectionIdsByHostId: {}
}
@@ -6903,6 +6907,7 @@ describe('Store', () => {
const tombstone = (readDataFile() as PersistedState).deletedFolderWorkspaceSessionTombstones?.[
workspaceKey
]
expect(tombstone?.evidenceTruncated).toBe(true)
expect(Object.keys(tombstone?.tabConnectionIdsByHostId[hostId] ?? {})).toHaveLength(256)
const ownerIndex = (
store as unknown as {
@@ -6910,6 +6915,25 @@ describe('Store', () => {
}
).deletedFolderOwnersByHostAndTabId
expect(ownerIndex.get(hostId)?.size).toBe(256)
const evictedTabId = 'delayed-tab-0'
const evictedPaneKey = makePaneKey(evictedTabId, TEST_LEAF_1)
store.patchWorkspaceSession(
{
terminalLayoutsByTabId: {
[evictedTabId]: {
root: { type: 'leaf', leafId: TEST_LEAF_1 },
activeLeafId: TEST_LEAF_1,
expandedLeafId: null,
ptyIdsByLeafId: { [TEST_LEAF_1]: 'evicted-delayed-pty' }
}
},
terminalPtyIncarnationsByPaneKey: { [evictedPaneKey]: 'evicted-incarnation' }
},
hostId
)
expect(store.getWorkspaceSession(hostId).terminalLayoutsByTabId).toEqual({})
expect(store.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey).toEqual({})
})
it('bounds host-partition evidence for one deleted folder', async () => {
@@ -6941,6 +6965,7 @@ describe('Store', () => {
const tombstone = (readDataFile() as PersistedState).deletedFolderWorkspaceSessionTombstones?.[
workspaceKey
]
expect(tombstone?.evidenceTruncated).toBe(true)
expect(tombstone?.hostIds).toHaveLength(32)
expect(Object.keys(tombstone?.tabConnectionIdsByHostId ?? {})).toHaveLength(32)
const ownerIndex = (
@@ -6949,6 +6974,23 @@ describe('Store', () => {
}
).deletedFolderOwnersByHostAndTabId
expect(ownerIndex.size).toBe(32)
const evictedHostId = toRuntimeExecutionHostId('host-evidence-0')
const evictedTabId = 'host-evidence-tab-0'
store.patchWorkspaceSession(
{
terminalLayoutsByTabId: {
[evictedTabId]: {
root: { type: 'leaf', leafId: TEST_LEAF_1 },
activeLeafId: TEST_LEAF_1,
expandedLeafId: null,
ptyIdsByLeafId: { [TEST_LEAF_1]: 'evicted-host-pty' }
}
}
},
evictedHostId
)
expect(store.getWorkspaceSession(evictedHostId).terminalLayoutsByTabId).toEqual({})
})
it('durably fences a delayed unified-only folder tab before a layout-only write', async () => {
+15 -5
View File
@@ -2936,6 +2936,7 @@ function normalizeDeletedFolderWorkspaceSessionTombstones(
const normalizedTombstone = boundDeletedFolderTombstoneEvidence({
connectionId,
deletedAt,
evidenceTruncated: raw.evidenceTruncated === true,
hostIds: [...hostIds],
tabConnectionIdsByHostId
})
@@ -4583,6 +4584,7 @@ export class Store {
this.setDeletedFolderWorkspaceSessionTombstone(workspaceKey, {
connectionId: connectionId ?? prior?.connectionId ?? null,
deletedAt: prior?.deletedAt ?? Date.now(),
evidenceTruncated: prior?.evidenceTruncated ?? false,
hostIds: [...new Set([...(prior?.hostIds ?? []), ...hostIds])],
tabConnectionIdsByHostId: prior?.tabConnectionIdsByHostId ?? {}
})
@@ -4730,6 +4732,7 @@ export class Store {
this.setDeletedFolderWorkspaceSessionTombstone(workspaceKey, {
connectionId: prior?.connectionId ?? connectionId,
deletedAt: prior?.deletedAt ?? Date.now(),
evidenceTruncated: prior?.evidenceTruncated ?? false,
hostIds: [
...new Set([...retainedHostIds, hostId, ...(connectionHostId ? [connectionHostId] : [])])
],
@@ -4927,7 +4930,7 @@ export class Store {
!tombstone ||
scope?.type !== 'folder' ||
liveFolderIds.has(scope.folderWorkspaceId) ||
!tombstone.hostIds.includes(hostId)
(!tombstone.hostIds.includes(hostId) && !tombstone.evidenceTruncated)
) {
continue
}
@@ -5108,6 +5111,9 @@ export class Store {
(this.state.folderWorkspaces ?? []).map((workspace) => workspace.id)
)
const deletedFolderTombstones = this.state.deletedFolderWorkspaceSessionTombstones ?? {}
const hasTruncatedDeletedFolderEvidence = Object.values(deletedFolderTombstones).some(
(tombstone) => tombstone?.evidenceTruncated
)
const ownerKeys = new Set<string>()
const ownerKeysByTabId = new Map<string, Set<string>>()
const directPtyIdsByOwnerAndTabId = new Map<string, Map<string, string>>()
@@ -5246,7 +5252,7 @@ export class Store {
session = { ...session, activeWorkspaceExecutionHostId: null }
}
const tombstonedConnectionIds = this.getDeletedFolderConnectionIdsForHost(hostId)
if (ownerKeys.size === 0) {
if (ownerKeys.size === 0 && !hasTruncatedDeletedFolderEvidence) {
return this.pruneDeletedFolderReconnectTargets(
session,
hostId,
@@ -5286,7 +5292,10 @@ export class Store {
for (const connectionId of tombstonedConnectionIds) {
removedConnectionIds.add(connectionId)
}
const pruned = removeWorkspaceSessionOwners(session, ownerKeys)!
const pruned =
ownerKeys.size > 0
? removeWorkspaceSessionOwners(session, ownerKeys)!
: cloneWorkspaceSessionState(session)
const currentDeletedOwnersByTabId = this.deletedFolderOwnersByHostAndTabId.get(hostId)
const exclusivelyDeletedTabIds = new Set<string>()
for (const tabId of tabScopedStateIds) {
@@ -5299,12 +5308,13 @@ export class Store {
const deletedOwner = [...(currentDeletedOwnersByTabId?.get(tabId)?.entries() ?? [])].find(
([workspaceKey]) => ownerKeys.has(workspaceKey)
)
if (!deletedOwner) {
if (!deletedOwner && !hasTruncatedDeletedFolderEvidence) {
continue
}
exclusivelyDeletedTabIds.add(tabId)
const ptyId = pruned.remoteSessionIdsByTabId?.[tabId]
const connectionId = (ptyId ? parseAppSshPtyId(ptyId)?.connectionId : null) ?? deletedOwner[1]
const connectionId =
(ptyId ? parseAppSshPtyId(ptyId)?.connectionId : null) ?? deletedOwner?.[1]
if (connectionId) {
removedConnectionIds.add(connectionId)
}
@@ -150,4 +150,79 @@ describe('mobile folder session mutation races', () => {
})
}
)
it('kills a pending materialization when folder routing changes during spawn', async () => {
const folder: FolderWorkspace = {
id: 'materialize-race-folder',
projectGroupId: 'materialize-race-group',
name: 'Materialize race',
folderPath: '/workspace/materialize-race',
connectionId: null,
linkedTask: null,
comment: '',
isArchived: false,
isUnread: false,
isPinned: false,
sortOrder: 0,
lastActivityAt: 1,
createdAt: 1,
updatedAt: 1
}
const group: ProjectGroup = {
id: folder.projectGroupId,
name: folder.projectGroupId,
parentPath: folder.folderPath,
connectionId: null,
parentGroupId: null,
createdFrom: 'manual',
tabOrder: 0,
isCollapsed: false,
color: null,
createdAt: 1,
updatedAt: 1
}
const folderKey = `folder:${folder.id}`
const runtime = createRuntime([folder], group)
vi.spyOn(
runtime as unknown as {
resolveTerminalWorkspaceLaunchScope: () => Promise<{
id: string
path: string
connectionId: null
repo: null
folderWorkspace: FolderWorkspace
}>
},
'resolveTerminalWorkspaceLaunchScope'
).mockImplementation(async () => ({
id: folderKey,
path: folder.folderPath,
connectionId: null,
repo: null,
folderWorkspace: folder
}))
let releaseSpawn!: (value: { id: string }) => void
const spawn = vi.fn(
() =>
new Promise<{ id: string }>((resolve) => {
releaseSpawn = resolve
})
)
const kill = vi.fn(() => true)
runtime.setPtyController({
write: () => true,
kill,
getForegroundProcess: async () => null,
listProcesses: async () => [],
spawn
})
const activation = runtime.activateMobileSessionTab(`id:${folderKey}`, TAB_ID)
await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce())
folder.folderPath = '/workspace/materialize-race-moved'
releaseSpawn({ id: 'stale-materialized-pty' })
await expect(activation).rejects.toThrow('tab_not_found')
expect(kill).toHaveBeenCalledWith('stale-materialized-pty')
})
})
+83 -23
View File
@@ -1291,6 +1291,7 @@ type TerminalCreateOptions = {
agentSessionClaim?: AgentSessionExecutionClaim
agentSessionCreateOperationId?: string
signal?: AbortSignal
acceptWorkspaceInventory?: () => boolean
// Why: idempotent create operations must retain their fence after the PTY
// exists, even if later runtime publication fails.
onPtySpawnCommitted?: () => void
@@ -1543,6 +1544,7 @@ type RuntimePtyController = {
}
agentSessionCreateOperationId?: string
signal?: AbortSignal
acceptWorkspaceInventory?: () => boolean
onPtySpawnCommitted?: () => void
}): Promise<{
id: string
@@ -4934,6 +4936,55 @@ export class OrcaRuntimeService {
throw new Error('tab_not_found')
}
private getFolderWorkspaceInventoryFingerprint(
workspace: FolderWorkspace,
inventory = {
repos: this.store?.getRepos?.() ?? [],
projectGroups: this.store?.getProjectGroups?.() ?? []
}
): string {
let connectionId: string | null | 'ambiguous' = 'ambiguous'
let hostId: ExecutionHostId | 'ambiguous' = 'ambiguous'
try {
connectionId = this.resolveFolderWorkspaceConnectionId(workspace, inventory)
hostId = this.getWorkspaceSessionHostIdForFolderWorkspace(workspace, inventory)
} catch {
// Why: a routing error is stable until the catalog fields below change.
}
return JSON.stringify([
workspace.projectGroupId,
workspace.folderPath,
workspace.connectionId,
workspace.executionHostId,
connectionId,
hostId
])
}
private createMobileSessionFolderWorkspaceMutationFence(
worktreeId: string
): (() => boolean) | undefined {
const scope = parseWorkspaceKey(worktreeId)
if (scope?.type !== 'folder' || !this.store?.getFolderWorkspaces) {
return undefined
}
const workspace = this.store
.getFolderWorkspaces()
.find((candidate) => candidate.id === scope.folderWorkspaceId)
if (!workspace) {
return () => false
}
const expectedFingerprint = this.getFolderWorkspaceInventoryFingerprint(workspace)
return () => {
const current = this.store
?.getFolderWorkspaces?.()
.find((candidate) => candidate.id === scope.folderWorkspaceId)
return Boolean(
current && this.getFolderWorkspaceInventoryFingerprint(current) === expectedFingerprint
)
}
}
private createFolderWorkspaceInventoryFence(
folderWorkspaces: readonly FolderWorkspace[]
): () => boolean {
@@ -4946,27 +4997,10 @@ export class OrcaRuntimeService {
projectGroups: this.store?.getProjectGroups?.() ?? []
}
return new Map(
workspaces.map((workspace) => {
let connectionId: string | null | 'ambiguous' = 'ambiguous'
let hostId: ExecutionHostId | 'ambiguous' = 'ambiguous'
try {
connectionId = this.resolveFolderWorkspaceConnectionId(workspace, inventory)
hostId = this.getWorkspaceSessionHostIdForFolderWorkspace(workspace, inventory)
} catch {
// Why: a routing error is itself stable until the underlying inventory changes.
}
return [
workspace.id,
JSON.stringify([
workspace.projectGroupId,
workspace.folderPath,
workspace.connectionId,
workspace.executionHostId,
connectionId,
hostId
])
]
})
workspaces.map((workspace) => [
workspace.id,
this.getFolderWorkspaceInventoryFingerprint(workspace, inventory)
])
)
}
const expectedById = capture(folderWorkspaces)
@@ -8018,9 +8052,15 @@ export class OrcaRuntimeService {
const explicitWorktreeId = this.getValidatedExplicitWorktreeIdSelector(worktreeSelector)
const worktreeId =
explicitWorktreeId ?? (await this.resolveWorktreeSelector(worktreeSelector)).id
this.assertMobileSessionFolderWorkspaceExists(worktreeId)
const acceptWorkspaceInventory =
this.createMobileSessionFolderWorkspaceMutationFence(worktreeId)
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
await this.refreshMobileSessionPtyRecords(worktreeId)
this.assertMobileSessionFolderWorkspaceExists(worktreeId)
if (acceptWorkspaceInventory?.() === false) {
throw new Error('tab_not_found')
}
const snapshot = this.mobileSessionTabsByWorktree.get(worktreeId)
const directTab = snapshot?.tabs.find((candidate) => candidate.id === tabId)
const tab = leafId
@@ -8093,7 +8133,8 @@ export class OrcaRuntimeService {
startupCommandDelivery: agentStartup.startupCommandDelivery,
launchConfig: agentStartup.launchConfig,
launchAgent: tab.launchAgent,
targetGroupId
targetGroupId,
acceptWorkspaceInventory
})
} catch (err) {
if (sessionId && parseAppSshPtyId(sessionId)) {
@@ -8363,9 +8404,15 @@ export class OrcaRuntimeService {
const explicitWorktreeId = this.getValidatedExplicitWorktreeIdSelector(worktreeSelector)
const worktreeId =
explicitWorktreeId ?? (await this.resolveWorktreeSelector(worktreeSelector)).id
this.assertMobileSessionFolderWorkspaceExists(worktreeId)
const acceptWorkspaceInventory =
this.createMobileSessionFolderWorkspaceMutationFence(worktreeId)
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
const observedPtyIds = await this.refreshMobileSessionPtyRecords()
this.assertMobileSessionFolderWorkspaceExists(worktreeId)
if (acceptWorkspaceInventory?.() === false) {
throw new Error('tab_not_found')
}
if (graphEpoch !== null) {
this.assertStableReadyGraph(graphEpoch)
}
@@ -25250,6 +25297,9 @@ export class OrcaRuntimeService {
if (launchOpts.signal?.aborted) {
throw new Error('client_disconnected')
}
if (launchOpts.acceptWorkspaceInventory?.() === false) {
throw new Error('tab_not_found')
}
const result = await this.ptyController.spawn({
cols: 120,
rows: 40,
@@ -25290,6 +25340,9 @@ export class OrcaRuntimeService {
? { agentSessionCreateOperationId: launchOpts.agentSessionCreateOperationId }
: {}),
...(launchOpts.signal ? { signal: launchOpts.signal } : {}),
...(launchOpts.acceptWorkspaceInventory
? { acceptWorkspaceInventory: launchOpts.acceptWorkspaceInventory }
: {}),
...(launchOpts.onPtySpawnCommitted ? { onPtySpawnCommitted: reportPtySpawnCommitted } : {}),
...(launchOpts.sessionId ? { sessionId: launchOpts.sessionId } : {}),
// Why: a headless-created pane has no renderer session writer. Persist
@@ -25301,6 +25354,11 @@ export class OrcaRuntimeService {
? { persistHostSessionBinding: true }
: {})
})
if (launchOpts.acceptWorkspaceInventory?.() === false) {
this.ptyController.kill(result.id)
this.releaseRejectedPtyRegistrationFence(result.id, result.incarnationId)
throw new Error('tab_not_found')
}
reportPtySpawnCommitted()
if (result.agentSessionEnsure) {
const canonicalSurface = result.agentSessionEnsure.owner.surface
@@ -26037,6 +26095,7 @@ export class OrcaRuntimeService {
targetGroupId?: string
launchConfig?: SleepingAgentLaunchConfig
signal?: AbortSignal
acceptWorkspaceInventory?: () => boolean
} = {}
): Promise<RuntimeMobileSessionCreateTerminalResult> {
const workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${worktreeId}`)
@@ -26066,7 +26125,8 @@ export class OrcaRuntimeService {
persistHostSessionBinding: true,
// Why: this method publishes the authoritative snapshot below; skip the intermediate publish to avoid a wrong-group flash.
deferMobileSessionPublish: true,
signal: opts.signal
signal: opts.signal,
acceptWorkspaceInventory: opts.acceptWorkspaceInventory
})
const livePty = this.getLivePtyForHandle(terminal.handle)
if (!livePty) {
+1
View File
@@ -3567,6 +3567,7 @@ export type PersistedMobileClientTabSelections = Record<
export type DeletedFolderWorkspaceSessionTombstone = {
connectionId: string | null
deletedAt: number
evidenceTruncated: boolean
hostIds: ExecutionHostId[]
tabConnectionIdsByHostId: Partial<Record<ExecutionHostId, Record<string, string | null>>>
}