Merge pull request #407 from nwparker/lc2/lc2-gone-cloud-records

fix(cloud): sandboxes deleted elsewhere leave no 'Orca Cloud · Couldn't start' row (LC-2)

src/main/hosted-sandbox/cloud-account-backfill.ts

@@ -24,9 +24,10 @@ function sandboxIdOf(runtime: EphemeralVmRuntimeRecord): string | null {
 export type CloudAccountBackfill = {
   /**
    * Lists the active team's sandboxes when a live record is unfiled, the team changed, or the last
-   * list is older than `relistMs` (the first run, at startup, always lists).
+   * list is older than `relistMs` (the first run, at startup, always lists). `force` lists now
+   * (a cloud list refreshed, or a sandbox read said not found: LC-2).
    */
-  run(): Promise<void>
+  run(options?: { force?: boolean }): Promise<void>
 }
 
 export function createCloudAccountBackfill(deps: {
@@ -45,7 +46,7 @@ export function createCloudAccountBackfill(deps: {
   let lastKey: string | null = null
   let listedAt = Number.NEGATIVE_INFINITY
   return {
-    async run() {
+    async run(options) {
       const account = deps.activeAccountId()
       if (!account) {
         return
@@ -54,7 +55,7 @@ export function createCloudAccountBackfill(deps: {
       // A team switch re-lists once even with nothing unfiled: it corrects a misfiled create.
       const key = keyOf(account, before)
       const stale = deps.relistMs !== undefined && now() - listedAt >= deps.relistMs
-      if (before.length === 0 || (key === lastKey && !stale)) {
+      if (before.length === 0 || (key === lastKey && !stale && !options?.force)) {
         return
       }
       let listed: ReadonlySet<string>

src/main/hosted-sandbox/cloud-gone-prune-request.test.ts

@@ -0,0 +1,42 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { requestCloudGonePrune, setCloudGonePruneRunner } from './cloud-gone-prune-request'
+
+afterEach(() => setCloudGonePruneRunner(null))
+
+describe('requestCloudGonePrune (LC-2)', () => {
+  it('runs the prune now instead of at the next relist, at most once per 30 s', async () => {
+    const run = vi.fn(async () => {})
+    setCloudGonePruneRunner(run)
+    await requestCloudGonePrune(1_000)
+    await requestCloudGonePrune(2_000)
+    await requestCloudGonePrune(20_000)
+    expect(run).toHaveBeenCalledTimes(1)
+    await requestCloudGonePrune(31_001)
+    expect(run).toHaveBeenCalledTimes(2)
+  })
+
+  it('is single-flight and survives a failing run', async () => {
+    let release!: () => void
+    const run = vi.fn(
+      () =>
+        new Promise<void>((resolve) => {
+          release = resolve
+        })
+    )
+    setCloudGonePruneRunner(run)
+    const first = requestCloudGonePrune(1_000)
+    void requestCloudGonePrune(100_000)
+    expect(run).toHaveBeenCalledTimes(1)
+    release()
+    await first
+
+    setCloudGonePruneRunner(async () => {
+      throw new Error('offline')
+    })
+    await expect(requestCloudGonePrune(200_000)).resolves.toBeUndefined()
+  })
+
+  it('does nothing before the cloud is installed (no control plane)', async () => {
+    await expect(requestCloudGonePrune(1_000)).resolves.toBeUndefined()
+  })
+})

src/main/hosted-sandbox/cloud-gone-prune-request.ts

@@ -0,0 +1,31 @@
+// Ask for the gone-sandbox prune now instead of at the next 10-minute relist (LC-2). A sandbox
+// deleted elsewhere (Home, another desktop, the CLI, the control plane) otherwise kept its row as
+// "Orca Cloud · Couldn't start" until then, or for good when the app didn't stay open that long.
+// The prune itself still needs proof: a list miss plus one confirming read (not found/destroyed).
+
+const MIN_INTERVAL_MS = 30_000
+
+let runner: (() => Promise<void>) | null = null
+let lastRequestAt = Number.NEGATIVE_INFINITY
+let running: Promise<void> | null = null
+
+/** hosted-sandbox-install registers the forced backfill + prune pass. */
+export function setCloudGonePruneRunner(next: (() => Promise<void>) | null): void {
+  runner = next
+  lastRequestAt = Number.NEGATIVE_INFINITY
+}
+
+/** A cloud list refreshed, or a sandbox read said not found: prune soon (throttled, single-flight). */
+export function requestCloudGonePrune(now: number = Date.now()): Promise<void> {
+  if (!runner || running || now - lastRequestAt < MIN_INTERVAL_MS) {
+    return running ?? Promise.resolve()
+  }
+  lastRequestAt = now
+  const run = runner
+  running = run()
+    .catch((error: unknown) => console.warn('[orca-cloud] gone-sandbox prune failed', error))
+    .finally(() => {
+      running = null
+    })
+  return running
+}

src/main/hosted-sandbox/cloud-sandbox-gone-prune.test.ts

@@ -182,4 +182,24 @@ describe('Orca Cloud list reconcile', () => {
     await backfill.run()
     expect(onListed).toHaveBeenCalledTimes(2)
   })
+
+  it('a forced run lists now, so a sandbox deleted elsewhere is pruned without waiting (LC-2)', async () => {
+    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-gone-force-'))
+    dirs.push(userDataPath)
+    upsertEphemeralVmRuntime(userDataPath, record('lumpfish'))
+    const onListed = vi.fn(async () => {})
+    const backfill = createCloudAccountBackfill({
+      userDataPath,
+      activeAccountId: () => TEAM,
+      list: async () => [],
+      onListed,
+      relistMs: 10 * 60_000,
+      now: () => 0
+    })
+    await backfill.run()
+    await backfill.run()
+    expect(onListed).toHaveBeenCalledTimes(1)
+    await backfill.run({ force: true })
+    expect(onListed).toHaveBeenCalledTimes(2)
+  })
 })

src/main/hosted-sandbox/hosted-sandbox-install.ts

@@ -39,6 +39,7 @@ import { orcaCloudAccountId } from '../../shared/hosted-sandbox/orca-cloud-accou
 import { setOrcaCloudActiveAccountResolver } from './orca-cloud-active-account'
 import { createCloudAccountBackfill } from './cloud-account-backfill'
 import { createCloudSandboxGonePrune } from './cloud-sandbox-gone-prune'
+import { setCloudGonePruneRunner } from './cloud-gone-prune-request'
 import { retireGoneCloudSandbox } from './cloud-sandbox-gone-retire'
 import type { RemovedServerStore } from '../ipc/runtime-environments/runtime-environment-removal-cleanup'
 import { createUsageAlertRefreshScheduler } from '../billing/usage-alert-refresh'
@@ -128,6 +129,8 @@ export function installHostedSandbox(
     onListed: (account, listed) => gonePrune.observeList(account, listed),
     relistMs: 10 * 60_000
   })
+  // A cloud list refresh or a not-found status read asks for the prune now (LC-2).
+  setCloudGonePruneRunner(() => accountBackfill.run({ force: true }))
   installLocalCommands(userDataPath)
   const mcpSync = installCloudMcpSync(userDataPath, {
     baseUrl: config.controlPlaneUrl,

src/main/hosted-sandbox/hosted-workspace-service.ts

@@ -1,5 +1,7 @@
 // Orca Cloud workspace surfaces for the window: status panel, Restart, archive/unarchive, wake
 // progress and OOM notices. Maps the renderer's workspace id to its Orca Cloud runtime record.
+import { requestCloudGonePrune } from './cloud-gone-prune-request'
+import { isEnvironmentProviderError } from '../environment-providers/environment-provider-driver'
 import { failQueuedCloudPrompts } from './cloud-offline-seam'
 import { ipcMain, type BrowserWindow } from 'electron'
 import { listTeamCloudRuntimes } from './orca-cloud-active-account'
@@ -194,6 +196,10 @@ export function installHostedWorkspaceService(deps: {
       oom.observe(args.workspaceId, found.sandboxId, status)
       return { ok: true, value: status }
     } catch (error) {
+      // Gone server-side: prune it now rather than show "Couldn't start" until the next relist.
+      if (isEnvironmentProviderError(error, 'not_found')) {
+        void requestCloudGonePrune()
+      }
       return failure(error)
     }
   })

src/main/ipc/cloud-workspaces-pairings.test.ts

@@ -0,0 +1,59 @@
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, describe, expect, it, vi } from 'vitest'
+import {
+  listEphemeralVmRuntimes,
+  upsertEphemeralVmRuntime
+} from '../../shared/ephemeral-vm-runtime-store'
+import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes'
+import { HOSTED_SANDBOX_DRIVER_ID } from '../hosted-sandbox/hosted-sandbox-driver'
+
+const userData = vi.hoisted(() => ({ path: '' }))
+vi.mock('electron', () => ({
+  app: { getPath: () => userData.path },
+  BrowserWindow: { getAllWindows: () => [] }
+}))
+userData.path = mkdtempSync(join(tmpdir(), 'orca-pairings-'))
+afterAll(() => rmSync(userData.path, { recursive: true, force: true }))
+
+const { createCloudWorkspacesPairings } = await import('./cloud-workspaces-pairings')
+
+function owned(id: string, sandboxId: string): EphemeralVmRuntimeRecord {
+  return {
+    id,
+    recipeId: 'cloud',
+    recipe: { id: 'cloud', name: 'Orca Cloud', create: 'c', provider: 'orca-cloud' },
+    workspaceId: `ws-${id}`,
+    status: 'running',
+    cleanupStatus: 'not_started',
+    createdAt: 1,
+    updatedAt: 1,
+    recipeResult: {
+      schemaVersion: 1,
+      connection: { type: 'orca-server', pairingCode: 'code', projectRoot: '/p' },
+      userData: {
+        environmentDriver: {
+          driverId: HOSTED_SANDBOX_DRIVER_ID,
+          environmentId: sandboxId,
+          driverState: { sandboxId }
+        }
+      }
+    }
+  }
+}
+
+describe('Delete from Home (LC-2)', () => {
+  it("retires this desktop's own record of the deleted sandbox at once, and only that one", async () => {
+    upsertEphemeralVmRuntime(userData.path, owned('lumpfish', 'sbx_aaaaaaaaaaaaaaaaaaaaaaaa'))
+    upsertEphemeralVmRuntime(userData.path, owned('gudgeon', 'sbx_bbbbbbbbbbbbbbbbbbbbbbbb'))
+    const { settleDeleted } = createCloudWorkspacesPairings(async () => null, undefined)
+
+    await settleDeleted('sbx_aaaaaaaaaaaaaaaaaaaaaaaa')
+
+    const byId = Object.fromEntries(
+      listEphemeralVmRuntimes(userData.path).map((runtime) => [runtime.id, runtime.status])
+    )
+    expect(byId).toEqual({ lumpfish: 'cleaned', gudgeon: 'running' })
+  })
+})

src/main/ipc/cloud-workspaces-pairings.ts

@@ -23,6 +23,10 @@ import {
 } from './runtime-environments/runtime-environment-removal-cleanup'
 import { invalidateRuntimeEnvironmentTransport } from './runtime-environments/runtime-environments'
 import { publishRuntimeEnvironmentCatalogChanged } from './runtime-environments/runtime-environment-catalog-broadcast'
+import { listEphemeralVmRuntimes } from '../../shared/ephemeral-vm-runtime-store'
+import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
+import { HOSTED_SANDBOX_DRIVER_ID } from '../hosted-sandbox/hosted-sandbox-driver'
+import { retireGoneCloudSandbox } from '../hosted-sandbox/cloud-sandbox-gone-retire'
 
 const Id = z.string().regex(/^sbx_[0-9a-f]{24}$/)
 const GrantId = z.string().regex(/^grt_[0-9a-f]{24}$/)
@@ -35,6 +39,32 @@ export type CloudWorkspacesPairings = {
   updateBindings: (
     change: (current: Record<string, Binding>) => Record<string, Binding>
   ) => Promise<void>
+  /** The sandbox was deleted from here: its pairing and this desktop's own record go now (LC-2). */
+  settleDeleted: (sandboxId: string) => Promise<void>
+}
+
+const RETIRED_STATUSES = new Set(['cleaned', 'cleanup_pending'])
+
+/**
+ * This desktop's own record of a sandbox deleted from Home or the API: retired the way the
+ * sidebar's Delete retires it, so its row never lingers as "Orca Cloud · Couldn't start" (LC-2).
+ */
+async function retireOwnedRecords(
+  sandboxId: string,
+  removedServerStore: RemovedServerStore | undefined
+): Promise<void> {
+  const userDataPath = app.getPath('userData')
+  for (const runtime of listEphemeralVmRuntimes(userDataPath)) {
+    if (runtime.recipe?.provider !== 'orca-cloud' || RETIRED_STATUSES.has(runtime.status)) {
+      continue
+    }
+    const handle = runtime.recipeResult
+      ? environmentHandleFromRecipeResult(runtime.recipeResult, HOSTED_SANDBOX_DRIVER_ID)
+      : null
+    if (handle?.driverState.sandboxId === sandboxId || handle?.environmentId === sandboxId) {
+      await retireGoneCloudSandbox(userDataPath, removedServerStore, runtime)
+    }
+  }
 }
 
 const bindingsFile = () => join(app.getPath('userData'), 'cloud-workspaces-environments.json')
@@ -136,6 +166,12 @@ export function createCloudWorkspacesPairings(
     },
     log: (line) => console.warn(line)
   })
-  pairings = { prune, readBindings, updateBindings }
+  const settleDeleted = async (sandboxId: string): Promise<void> => {
+    await prune.forget(sandboxId).catch(() => undefined)
+    await retireOwnedRecords(sandboxId, removedServerStore).catch((error: unknown) =>
+      console.warn('[cloud-workspaces] could not retire a deleted sandbox record', error)
+    )
+  }
+  pairings = { prune, readBindings, updateBindings, settleDeleted }
   return pairings
 }

src/main/ipc/cloud-workspaces.ts

@@ -14,6 +14,7 @@ import {
 import { setCloudReaderKeyRevoker } from '../cloud-workspaces/cloud-reader-key-store'
 import { profileReaderKey, profileReadPins } from '../cloud-workspaces/cloud-read-profile-access'
 import { setCloudPairingVerifiedListener } from '../cloud-workspaces/cloud-pairing-seam'
+import { requestCloudGonePrune } from '../hosted-sandbox/cloud-gone-prune-request'
 import { registerTeamPresenceHandlers } from './cloud-workspaces-team-presence'
 import { registerCloudWorkspacePrefsHandlers } from './cloud-workspaces-prefs'
 import type { CloudReadSnapshotState } from '../../shared/cloud-workspaces/read-snapshot-state'
@@ -146,7 +147,7 @@ const INVALID = { ok: false as const, error: 'invalid_request' }
 export function registerCloudWorkspacesHandlers(
   options: { removedServerStore?: RemovedServerStore } = {}
 ): void {
-  const { prune, readBindings, updateBindings } = createCloudWorkspacesPairings(
+  const { prune, readBindings, updateBindings, settleDeleted } = createCloudWorkspacesPairings(
     getService,
     options.removedServerStore
   )
@@ -162,9 +163,10 @@ export function registerCloudWorkspacesHandlers(
       return !s ? NOT_CONFIGURED : INVALID
     }
     const result = await s.list(query.data)
-    // Each successful refresh also checks this device's pairings (throttled, in the background).
+    // Each successful refresh also checks this device's pairings and own records (throttled).
     if (result.ok) {
       void prune.run()
+      void requestCloudGonePrune()
     }
     return result
   })
@@ -236,9 +238,9 @@ export function registerCloudWorkspacesHandlers(
       return !s ? NOT_CONFIGURED : INVALID
     }
     const result = await s.destroy(parsed.data.id)
-    // Deleted from here: its pairing, server and projects go now, not at the next list.
+    // Deleted from here: its pairing or own record, server and projects go now (LC-1, LC-2).
     if (result.ok) {
-      await prune.forget(parsed.data.id).catch(() => undefined)
+      await settleDeleted(parsed.data.id)
     }
     return result
   })