From 8b22f044f5901a75de3c5caece09ede5f992dd62 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:04:54 -0700 Subject: [PATCH] fix(vm): preserve runtime sidecar rollback compatibility (#14444) * test(vm): reproduce runtime store rollback poisoning * fix(vm): keep runtime sidecar rollback-readable * fix(vm): harden rollback-compatible runtime persistence * fix(vm): publish rollback lifecycle authority first * test(vm): harden rollback compatibility coverage --- .github/workflows/pr.yml | 24 ++ config/reliability-gates.jsonc | 124 +++++++- ...ral-vm-runtime-store-cross-version.test.ts | 184 +++++++++++ config/scripts/pr-e2e-gate-contract.test.mjs | 10 + ...emeral-vm-runtime-store-rollback-repro.mjs | 118 +++++++ src/main/ephemeral-vm-failed-start-cleanup.ts | 24 +- ...ral-vm-runtime-provisioning-persistence.ts | 82 +++++ src/main/ephemeral-vm-runtime-service.test.ts | 152 ++++++++- src/main/ephemeral-vm-runtime-service.ts | 31 +- src/shared/bounded-secure-json-file.ts | 5 +- .../ephemeral-vm-runtime-feature-store.ts | 247 +++++++++++++++ ...phemeral-vm-runtime-rollback-projection.ts | 44 +++ ...phemeral-vm-runtime-store-rollback.test.ts | 288 ++++++++++++++++++ src/shared/ephemeral-vm-runtime-store.ts | 184 ++++++++--- src/shared/ephemeral-vm-runtimes.ts | 32 +- 15 files changed, 1483 insertions(+), 66 deletions(-) create mode 100644 config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts create mode 100644 config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs create mode 100644 src/main/ephemeral-vm-runtime-provisioning-persistence.ts create mode 100644 src/shared/ephemeral-vm-runtime-feature-store.ts create mode 100644 src/shared/ephemeral-vm-runtime-rollback-projection.ts create mode 100644 src/shared/ephemeral-vm-runtime-store-rollback.test.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f36bb405e28..293ff008020 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -50,6 +50,27 @@ jobs: - name: Check reliability gate manifest run: pnpm run check:reliability-gates + - name: Check VM runtime rollback compatibility + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if git diff --quiet --merge-base "$BASE_SHA" "$HEAD_SHA" -- \ + src/shared/ephemeral-vm-runtime-store.ts \ + src/shared/ephemeral-vm-runtime-feature-store.ts \ + src/shared/ephemeral-vm-runtime-rollback-projection.ts \ + src/shared/ephemeral-vm-runtimes.ts \ + src/shared/ephemeral-vm-recipes.ts \ + src/shared/orca-yaml-hook-types.ts \ + src/main/ephemeral-vm-runtime-service.ts \ + src/main/ephemeral-vm-runtime-provisioning-persistence.ts \ + src/main/ephemeral-vm-failed-start-cleanup.ts; then + echo "VM runtime persistence is unchanged." + exit 0 + fi + node config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs \ + config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts + - name: Enforce max-lines ratchet run: pnpm run check:max-lines-ratchet @@ -479,6 +500,9 @@ jobs: HEAD="${{ github.event.pull_request.head.sha }}" CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")" TEST_FILES="$(printf '%s\n' "$CHANGED" | grep -E '^tests/e2e/.*\.spec\.ts$' || true)" + if printf '%s\n' "$CHANGED" | grep -Eq '^src/(main/ephemeral-vm-(runtime-(service|provisioning-persistence)|failed-start-cleanup)|shared/(ephemeral-vm-runtime-(store|feature-store|rollback-projection|runtimes)|ephemeral-vm-recipes|orca-yaml-hook-types))\.ts$'; then + TEST_FILES="$(printf '%s\n%s\n' "$TEST_FILES" 'tests/e2e/ephemeral-vm-provisioned-root.spec.ts' | sort -u)" + fi TEST_FILES_JSON="$(printf '%s\n' "$TEST_FILES" | jq --raw-input --slurp --compact-output 'split("\n") | map(select(length > 0))')" echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT" if [ "$TEST_FILES_JSON" != '[]' ]; then diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 6d7292f3bf3..a390b1b3bd3 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-08-13", + "updatedAt": "2026-08-14", "policy": { "maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"], "blockingPromotion": { @@ -10,6 +10,128 @@ } }, "gates": [ + { + "id": "ephemeral-vm-runtime.rollback-readable-sidecar", + "title": "VM lifecycle records remain readable across provisioned-root rollback", + "maturity": "experimental", + "protection": "partial", + "owner": "ephemeral-vm-runtime-store", + "layer": "desktop-main-persistence", + "surfaces": ["VM runtime sidecar", "recipe lifecycle", "provisioned-root direct SSH"], + "platforms": ["macos", "linux", "windows"], + "providers": ["orca-server", "direct-ssh"], + "coveredPlatforms": ["macos", "linux", "windows"], + "coveredProviders": ["orca-server", "direct-ssh"], + "coverageNotes": "A pinned exact-source harness runs the pre-#14352 reader and lifecycle service against baseline, affected-main, candidate, and fix-reverted stores. Platform-neutral store tests cover v1 byte stability, mixed records, current-main migration, downgrade mutation, malformed/future metadata preservation, bounded writes, and provider userData round-tripping. The direct-SSH provisioned-root Electron journey remains the live transport and destroy proof; this desktop sidecar is not mobile-facing and changes no RPC or remote wire payload.", + "motivatingLinks": [ + "https://linear.app/stably/issue/STA-4274", + "https://github.com/stablyai/orca/pull/14352", + "https://github.com/stablyai/orca/issues/13044" + ], + "invariant": "A sidecar written by the new build cannot make the previous production build lose otherwise compatible VM runtimes. Downgrade lifecycle writes remain authoritative after re-upgrade, provisioned-root metadata remains recoverable, and corrupt or future feature metadata is preserved without poisoning the rollback-readable v1 store.", + "oracle": "Write one ordinary schema-v1 runtime and one provisioned-root runtime. Require the exact pre-#14352 reader to return both, run the old destroy lifecycle against the projected provisioned record, then require the candidate to retain the cleaned state while restoring schema-v2 and checkout-mode metadata. The same old reader must reject affected-main and fix-reverted bytes. Malformed, oversized, unknown-record, and future-version feature sidecars must leave rollback-compatible records accessible and must not be overwritten during compatible lifecycle mutations.", + "commands": [ + "node config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/ephemeral-vm-runtime-store.test.ts src/shared/ephemeral-vm-runtime-store-rollback.test.ts src/main/ephemeral-vm-runtime-service.test.ts src/main/ephemeral-vm-recipe-runner.test.ts src/main/ephemeral-vm-runtime-ssh-cleanup.test.ts src/main/ipc/ephemeral-vm-runtime-handler-cleanup.test.ts src/main/ipc/ephemeral-vm.test.ts src/main/provisioned-root-ssh-adoption.test.ts", + "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ephemeral-vm-provisioned-root.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "SKIP_BUILD=1 ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ephemeral-vm-provisioned-root.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" + ], + "testFiles": [ + "config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts", + "config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs", + "src/shared/ephemeral-vm-runtime-store-rollback.test.ts", + "src/shared/ephemeral-vm-runtime-store.test.ts", + "src/main/ephemeral-vm-runtime-service.test.ts", + "src/main/ephemeral-vm-recipe-runner.test.ts", + "src/main/ephemeral-vm-runtime-ssh-cleanup.test.ts", + "src/main/ipc/ephemeral-vm-runtime-handler-cleanup.test.ts", + "src/main/ipc/ephemeral-vm.test.ts", + "src/main/provisioned-root-ssh-adoption.test.ts", + "tests/e2e/ephemeral-vm-provisioned-root.spec.ts" + ], + "assertionRefs": [ + { + "file": "config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts", + "assertions": [ + "the exact rollback reader accepts both projected records", + "the rollback lifecycle service destroys the provisioned resource using schema-v1 provider data", + "re-upgrade restores provisioned-root metadata without reverting lifecycle state" + ] + }, + { + "file": "src/shared/ephemeral-vm-runtime-store-rollback.test.ts", + "assertions": [ + "ordinary runtime bytes remain unchanged and create no feature sidecar", + "malformed, future, oversized, and unknown feature data is preserved without poisoning v1 records", + "current-main poisoned bytes migrate to a rollback-readable projection", + "unchanged compatibility metadata is not rewritten during lifecycle updates" + ] + }, + { + "file": "src/main/ephemeral-vm-runtime-service.test.ts", + "assertions": [ + "an unreadable companion rejects checkout-mode recipes before their create command runs", + "post-create persistence failure destroys the provider resource when possible", + "failed destroy persists a durable rollback-readable cleanup recovery record" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts", + "result": "passed", + "durationSeconds": 11, + "summary": "The pinned baseline and affected-main oracles passed, candidate downgrade ran the exact old destroy service, candidate re-upgrade retained lifecycle state, and both affected-main and fix-reverted bytes failed the old reader as expected." + }, + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/ephemeral-vm-runtime-store.test.ts src/shared/ephemeral-vm-runtime-store-rollback.test.ts src/main/ephemeral-vm-runtime-service.test.ts src/main/ephemeral-vm-recipe-runner.test.ts src/main/ephemeral-vm-runtime-ssh-cleanup.test.ts src/main/ipc/ephemeral-vm-runtime-handler-cleanup.test.ts src/main/ipc/ephemeral-vm.test.ts src/main/provisioned-root-ssh-adoption.test.ts", + "result": "passed", + "durationSeconds": 2, + "summary": "Sixty-two focused store, recipe, lifecycle, cleanup-handler, provisioning-recovery, and provisioned-root adoption tests passed." + }, + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ephemeral-vm-provisioned-root.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 21, + "summary": "The final-tree full-build run completed the Docker Linux SSH journey: create, provisioned-root adoption, terminal, workspace removal, destroy, and provider-container cleanup." + } + ], + "runtimeBudget": { + "p95Seconds": 180, + "scope": "pinned cross-version source oracle, focused lifecycle contracts, and one Docker SSH Electron journey" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "The deterministic contract runs are stable. Two local Electron attempts reached successful adoption and terminal use, then hit the existing final context-menu detach; the immediate build-reuse rerun completed removal and destroy. CI soak history has not started." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "The same pinned oracle passes the pre-#14352 ordinary store, fails the affected-main mixed store with file-is-invalid, passes the candidate mixed store plus old-service destroy and re-upgrade, and fails again when the candidate compatibility files are reverted to affected main." + }, + "performanceBudget": { + "required": true, + "evidence": "Ordinary writes remain byte-identical and create no feature sidecar. Reads add one bounded feature-file existence check; sorted feature comparisons avoid unchanged rewrites, and durable companion writes occur only when compatibility metadata changes. There is no polling, timer, renderer work, RPC traffic, or remote fanout." + }, + "promotionCriteria": [ + "Keep the pinned baseline/latest/candidate/revert oracle green in CI.", + "Keep the Docker provisioned-root create/adopt/terminal/remove/destroy journey green.", + "Collect soak history without unexplained lifecycle or sidecar flakes." + ], + "knownGaps": [ + "No packaged-build downgrade installer journey; the exact released source reader and service are executed in-process.", + "Live Docker SSH evidence runs locally on macOS and in the changed-file PR E2E lane on Ubuntu; Windows desktop behavior is covered by the platform-neutral filesystem codec and package job." + ], + "demotionRule": "Demote if a new write fails the pinned rollback reader, a downgrade lifecycle mutation is lost after re-upgrade, malformed/future metadata overwrites recoverable bytes, ordinary v1 bytes change, or the focused gate flakes without an identified harness defect." + }, { "id": "terminal-session.shell-ready-exec-prompt-fallback", "title": "Startup exec falls back to the identified shell's line editor", diff --git a/config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts b/config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts new file mode 100644 index 00000000000..ce44812cf95 --- /dev/null +++ b/config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts @@ -0,0 +1,184 @@ +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterAll, expect, test } from 'vitest' + +const targetRoot = process.env.STA_4274_TARGET_ROOT +const operation = process.env.STA_4274_OPERATION +const ownedDirs: string[] = [] + +function makeOwnedDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)) + ownedDirs.push(dir) + return dir +} + +afterAll(() => { + for (const dir of ownedDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +const ordinary = { + id: 'ordinary-runtime', + recipeId: 'ordinary-recipe', + recipe: { + id: 'ordinary-recipe', + name: 'Ordinary VM', + create: './ordinary-create.sh', + destroy: './ordinary-destroy.sh' + }, + status: 'running' as const, + cleanupStatus: 'not_started' as const, + createdAt: 1_000, + updatedAt: 1_000, + recipeResult: { + schemaVersion: 1 as const, + connection: { + type: 'ssh' as const, + projectRoot: '/workspace/ordinary', + target: { + label: 'Ordinary VM', + host: 'ordinary.example.com', + port: 22, + username: 'developer' + } + }, + userData: { resourceId: 'ordinary-resource' } + } +} + +const provisionedRoot = { + id: 'provisioned-root-runtime', + recipeId: 'provisioned-root-recipe', + recipe: { + id: 'provisioned-root-recipe', + name: 'Provisioned Root VM', + create: './provisioned-root-create.sh', + destroy: './provisioned-root-destroy.sh', + checkoutMode: 'provisioned-root' as const + }, + status: 'running' as const, + cleanupStatus: 'not_started' as const, + createdAt: 2_000, + updatedAt: 2_000, + recipeResult: { + schemaVersion: 2 as const, + checkoutMode: 'provisioned-root' as const, + connection: { + type: 'ssh' as const, + projectRoot: '/workspace/provisioned', + target: { + label: 'Provisioned Root VM', + host: 'provisioned.example.com', + port: 22, + username: 'developer' + } + }, + userData: { resourceId: 'provisioned-resource' } + } +} + +test.skipIf(!targetRoot || !operation)(`STA-4274 ${operation ?? 'disabled'}`, async () => { + if (!targetRoot || !operation) { + throw new Error('STA_4274_TARGET_ROOT and STA_4274_OPERATION are required') + } + const userDataPath = process.env.STA_4274_USER_DATA_PATH ?? makeOwnedDir('sta-4274-') + const moduleUrl = pathToFileURL( + resolve(targetRoot, 'src/shared/ephemeral-vm-runtime-store.ts') + ).href + const store = await import(/* @vite-ignore */ moduleUrl) + + if (operation === 'write-legacy') { + store.upsertEphemeralVmRuntime(userDataPath, ordinary) + return + } + if (operation === 'write-mixed') { + store.upsertEphemeralVmRuntime(userDataPath, ordinary) + store.upsertEphemeralVmRuntime(userDataPath, provisionedRoot) + return + } + if (operation === 'read') { + const runtimes = store.listEphemeralVmRuntimes(userDataPath) + expect(runtimes.map((record: { id: string }) => record.id)).toEqual([ + 'provisioned-root-runtime', + 'ordinary-runtime' + ]) + return + } + if (operation === 'read-rollback-projection') { + const runtimes = store.listEphemeralVmRuntimes(userDataPath) + expect(runtimes).toHaveLength(2) + expect(runtimes[0].recipe).not.toHaveProperty('checkoutMode') + expect(runtimes[0].recipeResult).toMatchObject({ schemaVersion: 1 }) + return + } + if (operation === 'mutate-lifecycle') { + store.updateEphemeralVmRuntimeStatus(userDataPath, 'ordinary-runtime', { + status: 'suspended', + updatedAt: 3_000 + }) + const repoPath = makeOwnedDir('sta-4274-cleanup-') + const cleanupPath = join(repoPath, 'cleanup.js') + const proofPath = join(repoPath, 'cleanup-proof') + writeFileSync( + cleanupPath, + [ + "let input = ''", + "process.stdin.on('data', (chunk) => { input += chunk })", + "process.stdin.on('end', () => {", + ' const payload = JSON.parse(input)', + ' if (payload.recipeResult.schemaVersion !== 1) process.exit(12)', + " if (payload.recipeResult.userData.resourceId !== 'provisioned-resource') process.exit(13)", + ` require('fs').writeFileSync(${JSON.stringify(proofPath)}, 'destroyed')`, + '})' + ].join('\n') + ) + const serviceUrl = pathToFileURL( + resolve(targetRoot, 'src/main/ephemeral-vm-runtime-service.ts') + ).href + const service = await import(/* @vite-ignore */ serviceUrl) + const result = await service.cleanupEphemeralVmRuntime({ + userDataPath, + repoPath, + runtimeId: 'provisioned-root-runtime', + recipe: { + id: 'provisioned-root-recipe', + name: 'Provisioned Root VM', + create: './provisioned-root-create.sh', + destroy: `${JSON.stringify(process.execPath)} ${JSON.stringify(cleanupPath)}` + }, + now: 3_000 + }) + expect(result).toMatchObject({ + ok: true, + runtime: { status: 'cleaned', cleanupStatus: 'succeeded' } + }) + expect(existsSync(proofPath)).toBe(true) + return + } + if (operation === 'read-after-downgrade') { + expect(store.listEphemeralVmRuntimes(userDataPath)).toEqual([ + expect.objectContaining({ + id: 'provisioned-root-runtime', + status: 'cleaned', + cleanupStatus: 'succeeded', + recipe: expect.objectContaining({ checkoutMode: 'provisioned-root' }), + recipeResult: expect.objectContaining({ + schemaVersion: 2, + checkoutMode: 'provisioned-root' + }) + }), + expect.objectContaining({ id: 'ordinary-runtime', status: 'suspended' }) + ]) + return + } + if (operation === 'read-legacy') { + expect( + store.listEphemeralVmRuntimes(userDataPath).map((record: { id: string }) => record.id) + ).toEqual(['ordinary-runtime']) + return + } + throw new Error(`Unknown operation: ${operation}`) +}) diff --git a/config/scripts/pr-e2e-gate-contract.test.mjs b/config/scripts/pr-e2e-gate-contract.test.mjs index c600d233437..e8b7c19c5d8 100644 --- a/config/scripts/pr-e2e-gate-contract.test.mjs +++ b/config/scripts/pr-e2e-gate-contract.test.mjs @@ -14,6 +14,9 @@ const sshDockerRunner = readFileSync( const filterStep = prWorkflow.jobs['e2e-paths'].steps.find( (step) => step.name === 'Filter changed E2E specs' ) +const rollbackStep = prWorkflow.jobs.static_analysis.steps.find( + (step) => step.name === 'Check VM runtime rollback compatibility' +) const verifyStep = prWorkflow.jobs.verify.steps.find( (step) => step.name === 'Require successful checks' ) @@ -121,4 +124,11 @@ describe('PR E2E gate contract', () => { expect(filterStep.run).toContain('--merge-base "$BASE" "$HEAD"') expect(filterStep.run).toContain('set -euo pipefail') }) + + it('scopes the VM rollback oracle to the PR range and recipe schema authorities', () => { + expect(rollbackStep.run).toContain('--merge-base "$BASE_SHA" "$HEAD_SHA"') + expect(rollbackStep.run).toContain('src/shared/ephemeral-vm-recipes.ts') + expect(rollbackStep.run).toContain('src/shared/orca-yaml-hook-types.ts') + expect(filterStep.run).toContain('ephemeral-vm-recipes|orca-yaml-hook-types') + }) }) diff --git a/config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs b/config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs new file mode 100644 index 00000000000..9b5926639f2 --- /dev/null +++ b/config/scripts/run-ephemeral-vm-runtime-store-rollback-repro.mjs @@ -0,0 +1,118 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { spawnSync } from 'node:child_process' +import process from 'node:process' + +const BASELINE_COMMIT = 'bf0c77d5bc800e19117084c27fd1441eda9134ad' +const AFFECTED_MAIN_COMMIT = '25abb9368d98ad84a174f530e02f4228d2269062' +const root = process.cwd() +const driver = process.argv[2] ?? 'config/scripts/ephemeral-vm-runtime-store-cross-version.test.ts' +const config = 'config/vitest.config.ts' +const tempRoot = mkdtempSync(join(tmpdir(), 'orca-sta-4274-repro-')) + +try { + const baselineRoot = extractSource(BASELINE_COMMIT, 'baseline') + const affectedRoot = extractSource(AFFECTED_MAIN_COMMIT, 'affected-main') + const revertedRoot = extractSource('HEAD', 'candidate-reverted') + restoreAffectedStoreFiles(revertedRoot) + + const baselineData = makeDataDir('baseline-data') + runOracle('baseline write', baselineRoot, 'write-legacy', baselineData) + runOracle('baseline read', baselineRoot, 'read-legacy', baselineData) + + const affectedData = makeDataDir('affected-data') + runOracle('affected write', affectedRoot, 'write-mixed', affectedData) + runOracle('affected current read', affectedRoot, 'read', affectedData) + runOracle('affected rollback read', baselineRoot, 'read', affectedData, { + expectFailure: 'file is invalid' + }) + + const candidateData = makeDataDir('candidate-data') + runOracle('candidate write', root, 'write-mixed', candidateData) + runOracle( + 'candidate rollback projection', + baselineRoot, + 'read-rollback-projection', + candidateData + ) + runOracle('rollback lifecycle mutation', baselineRoot, 'mutate-lifecycle', candidateData) + runOracle('candidate re-upgrade', root, 'read-after-downgrade', candidateData) + + const revertedData = makeDataDir('reverted-data') + runOracle('reverted write', revertedRoot, 'write-mixed', revertedData) + runOracle('reverted rollback read', baselineRoot, 'read', revertedData, { + expectFailure: 'file is invalid' + }) +} finally { + rmSync(tempRoot, { recursive: true, force: true }) +} + +function extractSource(commit, name) { + const destination = join(tempRoot, name) + const archive = join(tempRoot, `${name}.tar`) + mkdirSync(destination) + run('git', ['archive', '--format=tar', `--output=${archive}`, commit, 'src/main', 'src/shared']) + run('tar', ['-xf', archive, '-C', destination]) + return destination +} + +function restoreAffectedStoreFiles(destination) { + for (const relativePath of [ + 'src/shared/ephemeral-vm-runtime-store.ts', + 'src/shared/ephemeral-vm-runtimes.ts' + ]) { + const result = run('git', ['show', `${AFFECTED_MAIN_COMMIT}:${relativePath}`], { + encoding: 'utf8' + }) + writeFileSync(join(destination, relativePath), result.stdout) + } +} + +function makeDataDir(name) { + const destination = join(tempRoot, name) + mkdirSync(destination) + return destination +} + +function runOracle(label, targetRoot, operation, userDataPath, options = {}) { + const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' + const result = spawnSync(pnpm, ['exec', 'vitest', 'run', driver, '--config', config], { + cwd: root, + encoding: 'utf8', + env: { + ...process.env, + STA_4274_TARGET_ROOT: resolve(targetRoot), + STA_4274_OPERATION: operation, + STA_4274_USER_DATA_PATH: userDataPath + } + }) + const output = `${result.stdout ?? ''}${result.stderr ?? ''}` + if (options.expectFailure) { + if (result.status === 0 || !output.includes(options.expectFailure)) { + throw new Error( + `${label} did not fail with ${JSON.stringify(options.expectFailure)}\n${output}` + ) + } + process.stdout.write(`EXPECTED_FAIL ${label}\n`) + return + } + if (result.status !== 0) { + throw new Error(`${label} failed\n${output}`) + } + process.stdout.write(`PASS ${label}\n`) +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: root, + encoding: options.encoding, + maxBuffer: 16 * 1024 * 1024 + }) + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(' ')} failed\n${String(result.stdout ?? '')}${String(result.stderr ?? '')}` + ) + } + return result +} diff --git a/src/main/ephemeral-vm-failed-start-cleanup.ts b/src/main/ephemeral-vm-failed-start-cleanup.ts index c7d3d991b2e..4b56a43e52b 100644 --- a/src/main/ephemeral-vm-failed-start-cleanup.ts +++ b/src/main/ephemeral-vm-failed-start-cleanup.ts @@ -2,7 +2,11 @@ import { getEphemeralVmRecipeResultConnection, type EphemeralVmRecipeResult } from '../shared/ephemeral-vm-recipes' -import { upsertEphemeralVmRuntime } from '../shared/ephemeral-vm-runtime-store' +import type { EphemeralVmRuntimeRecord } from '../shared/ephemeral-vm-runtimes' +import { + upsertEphemeralVmRuntime, + upsertEphemeralVmRuntimeRollbackRecovery +} from '../shared/ephemeral-vm-runtime-store' import type { ProvisionEphemeralVmRuntimeArgs } from './ephemeral-vm-runtime-service' import { runEphemeralVmRecipeCleanup, @@ -17,15 +21,15 @@ type FailedStart = { export async function cleanupFailedEphemeralVmStart( args: ProvisionEphemeralVmRuntimeArgs, start: FailedStart -): Promise { +): Promise { const cleanupError = await getCleanupError(args, start) if (cleanupError === null) { - return + return true } const now = args.now ?? Date.now() const connection = getEphemeralVmRecipeResultConnection(start.recipeResult) - upsertEphemeralVmRuntime(args.userDataPath, { + const recovery: EphemeralVmRuntimeRecord = { id: start.context.instanceId ?? start.context.recipeId, recipeId: args.recipe.id, recipe: args.recipe, @@ -42,7 +46,17 @@ export async function cleanupFailedEphemeralVmStart( createdAt: now, updatedAt: now, recipeResult: start.recipeResult - }) + } + try { + upsertEphemeralVmRuntime(args.userDataPath, recovery) + } catch (error) { + if (!args.recipe.checkoutMode) { + throw error + } + // Why: cleanup retry metadata must survive even when its feature companion is unreadable. + upsertEphemeralVmRuntimeRollbackRecovery(args.userDataPath, recovery) + } + return false } async function getCleanupError( diff --git a/src/main/ephemeral-vm-runtime-provisioning-persistence.ts b/src/main/ephemeral-vm-runtime-provisioning-persistence.ts new file mode 100644 index 00000000000..ee1d6e5418b --- /dev/null +++ b/src/main/ephemeral-vm-runtime-provisioning-persistence.ts @@ -0,0 +1,82 @@ +import { randomUUID } from 'node:crypto' +import { assertEphemeralVmRuntimeCheckoutModeCanPersist } from '../shared/ephemeral-vm-runtime-feature-store' +import { getEphemeralVmRecipeResultConnection } from '../shared/ephemeral-vm-recipes' +import { + listEphemeralVmRuntimes, + removeEphemeralVmRuntime, + upsertEphemeralVmRuntime +} from '../shared/ephemeral-vm-runtime-store' +import type { EphemeralVmRuntimeRecord } from '../shared/ephemeral-vm-runtimes' +import type { + ProvisionEphemeralVmRuntimeArgs, + ProvisionEphemeralVmRuntimeResult +} from './ephemeral-vm-runtime-service' +import { cleanupFailedEphemeralVmStart } from './ephemeral-vm-failed-start-cleanup' + +export type EphemeralVmCompatibilityPersistence = { + instanceId: string + createdAt: number +} + +export function prepareEphemeralVmCompatibilityPersistence( + args: ProvisionEphemeralVmRuntimeArgs +): EphemeralVmCompatibilityPersistence | null { + if (!args.recipe.checkoutMode) { + return null + } + listEphemeralVmRuntimes(args.userDataPath) + const compatibility = { + instanceId: `orca-${randomUUID()}`, + createdAt: args.now ?? Date.now() + } + assertEphemeralVmRuntimeCheckoutModeCanPersist(args.userDataPath, { + id: compatibility.instanceId, + recipeId: args.recipe.id, + createdAt: compatibility.createdAt, + checkoutMode: args.recipe.checkoutMode + }) + return compatibility +} + +export async function persistProvisionedEphemeralVmRuntime( + args: ProvisionEphemeralVmRuntimeArgs, + start: Extract['start'], + compatibility: EphemeralVmCompatibilityPersistence | null +): Promise { + const now = compatibility?.createdAt ?? args.now ?? Date.now() + const connection = getEphemeralVmRecipeResultConnection(start.result) + try { + return upsertEphemeralVmRuntime(args.userDataPath, { + id: start.context.instanceId ?? start.context.recipeId, + recipeId: args.recipe.id, + recipe: args.recipe, + ...(args.repoId ? { repoId: args.repoId } : {}), + ...(args.projectId ? { projectId: args.projectId } : {}), + ...(args.workspaceId ? { workspaceId: args.workspaceId } : {}), + ...(args.workspaceName ? { workspaceName: args.workspaceName } : {}), + status: 'running', + connectionMode: connection.type, + cleanupStatus: args.recipe.destroyDisabled ? 'disabled' : 'not_started', + ...(args.recipe.destroyDisabled ? { cleanupDisabled: true } : {}), + createdAt: now, + updatedAt: now, + recipeResult: start.result + }) + } catch (error) { + if (compatibility) { + const cleaned = await cleanupFailedEphemeralVmStart(args, { + context: start.context, + recipeResult: start.result + }) + if ( + cleaned && + listEphemeralVmRuntimes(args.userDataPath).some( + (runtime) => runtime.id === compatibility.instanceId + ) + ) { + removeEphemeralVmRuntime(args.userDataPath, compatibility.instanceId) + } + } + throw error + } +} diff --git a/src/main/ephemeral-vm-runtime-service.test.ts b/src/main/ephemeral-vm-runtime-service.test.ts index b3516d0e791..9280be2ba90 100644 --- a/src/main/ephemeral-vm-runtime-service.test.ts +++ b/src/main/ephemeral-vm-runtime-service.test.ts @@ -1,12 +1,17 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { encodePairingOffer, PAIRING_OFFER_VERSION } from '../shared/pairing' import { + getEphemeralVmRuntimeStorePath, listEphemeralVmRuntimes, upsertEphemeralVmRuntime } from '../shared/ephemeral-vm-runtime-store' +import { + getEphemeralVmRuntimeFeatureStorePath, + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES +} from '../shared/ephemeral-vm-runtime-feature-store' import { cleanupEphemeralVmRuntime, provisionEphemeralVmRuntime, @@ -179,6 +184,151 @@ describe('ephemeral VM runtime service', () => { expect(listEphemeralVmRuntimes(userDataPath)).toEqual([]) }) + it('rejects an unwritable feature store before a checkout-mode recipe creates resources', async () => { + const userDataPath = makeDir('orca-ephemeral-vm-service-user-data-') + const repoPath = makeDir('orca-ephemeral-vm-service-repo-') + const startPath = join(repoPath, 'start.js') + const markerPath = join(repoPath, 'create-ran.txt') + writeFileSync(startPath, `require('fs').writeFileSync(${JSON.stringify(markerPath)}, 'yes')`) + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + writeFileSync(featurePath, '{}') + truncateSync(featurePath, MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES + 1) + + await expect( + provisionEphemeralVmRuntime({ + userDataPath, + repoPath, + recipe: { + id: 'cloud-sandbox', + name: 'Cloud Sandbox', + checkoutMode: 'provisioned-root', + create: nodeCommand(startPath), + destroyDisabled: true + } + }) + ).rejects.toThrow('Could not preserve ephemeral VM runtime compatibility metadata') + expect(existsSync(markerPath)).toBe(false) + }) + + it('rejects an unreadable lifecycle store before a checkout-mode recipe creates resources', async () => { + const userDataPath = makeDir('orca-ephemeral-vm-service-user-data-') + const repoPath = makeDir('orca-ephemeral-vm-service-repo-') + const startPath = join(repoPath, 'start.js') + const markerPath = join(repoPath, 'create-ran.txt') + writeFileSync(startPath, `require('fs').writeFileSync(${JSON.stringify(markerPath)}, 'yes')`) + writeFileSync(getEphemeralVmRuntimeStorePath(userDataPath), '{') + + await expect( + provisionEphemeralVmRuntime({ + userDataPath, + repoPath, + recipe: { + id: 'cloud-sandbox', + name: 'Cloud Sandbox', + checkoutMode: 'provisioned-root', + create: nodeCommand(startPath), + destroyDisabled: true + } + }) + ).rejects.toThrow('file is invalid') + expect(existsSync(markerPath)).toBe(false) + }) + + it('destroys a checkout-mode resource when compatibility persistence fails after create', async () => { + const userDataPath = makeDir('orca-ephemeral-vm-service-user-data-') + const repoPath = makeDir('orca-ephemeral-vm-service-repo-') + const startPath = join(repoPath, 'start.js') + const cleanupPath = join(repoPath, 'cleanup.js') + const cleanupMarkerPath = join(repoPath, 'cleanup-ran.txt') + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + writeFileSync( + startPath, + [ + `require('fs').writeFileSync(${JSON.stringify(featurePath)}, '{')`, + 'console.log(JSON.stringify({', + ' schemaVersion: 2,', + ' checkoutMode: "provisioned-root",', + ' connection: {', + ' type: "ssh",', + ' projectRoot: "/workspace/repo",', + ' target: { label: "VM", host: "host", port: 22, username: "orca" }', + ' }', + '}))' + ].join('\n') + ) + writeFileSync( + cleanupPath, + `require('fs').writeFileSync(${JSON.stringify(cleanupMarkerPath)}, 'yes')` + ) + + await expect( + provisionEphemeralVmRuntime({ + userDataPath, + repoPath, + recipe: { + id: 'cloud-sandbox', + name: 'Cloud Sandbox', + checkoutMode: 'provisioned-root', + create: nodeCommand(startPath), + destroy: nodeCommand(cleanupPath) + } + }) + ).rejects.toThrow('Could not preserve ephemeral VM runtime compatibility metadata') + expect(readFileSync(cleanupMarkerPath, 'utf8')).toBe('yes') + expect(listEphemeralVmRuntimes(userDataPath)).toEqual([]) + }) + + it('keeps rollback-readable cleanup recovery when post-create destroy also fails', async () => { + const userDataPath = makeDir('orca-ephemeral-vm-service-user-data-') + const repoPath = makeDir('orca-ephemeral-vm-service-repo-') + const startPath = join(repoPath, 'start.js') + const cleanupPath = join(repoPath, 'cleanup.js') + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + writeFileSync( + startPath, + [ + `require('fs').writeFileSync(${JSON.stringify(featurePath)}, '{')`, + 'console.log(JSON.stringify({', + ' schemaVersion: 2,', + ' checkoutMode: "provisioned-root",', + ' connection: {', + ' type: "ssh",', + ' projectRoot: "/workspace/repo",', + ' target: { label: "VM", host: "host", port: 22, username: "orca" }', + ' },', + ' userData: { providerResourceId: "paid-vm" }', + '}))' + ].join('\n') + ) + writeFileSync(cleanupPath, 'process.exit(1)') + + await expect( + provisionEphemeralVmRuntime({ + userDataPath, + repoPath, + recipe: { + id: 'cloud-sandbox', + name: 'Cloud Sandbox', + checkoutMode: 'provisioned-root', + create: nodeCommand(startPath), + destroy: nodeCommand(cleanupPath) + }, + now: 1_000 + }) + ).rejects.toThrow('Could not preserve ephemeral VM runtime compatibility metadata') + expect(listEphemeralVmRuntimes(userDataPath)).toEqual([ + expect.objectContaining({ + status: 'cleanup_failed', + cleanupStatus: 'failed', + recipe: expect.not.objectContaining({ checkoutMode: expect.anything() }), + recipeResult: expect.objectContaining({ + schemaVersion: 1, + userData: { providerResourceId: 'paid-vm' } + }) + }) + ]) + }) + it('destroys a provisioned resource when its checkout handshake is incompatible', async () => { const userDataPath = makeDir('orca-ephemeral-vm-service-user-data-') const repoPath = makeDir('orca-ephemeral-vm-service-repo-') diff --git a/src/main/ephemeral-vm-runtime-service.ts b/src/main/ephemeral-vm-runtime-service.ts index ed419b2f950..a6dd1933f89 100644 --- a/src/main/ephemeral-vm-runtime-service.ts +++ b/src/main/ephemeral-vm-runtime-service.ts @@ -1,11 +1,9 @@ import type { OrcaVmRecipe } from '../shared/orca-yaml-hook-types' import { listEphemeralVmRuntimes, - updateEphemeralVmRuntimeStatus, - upsertEphemeralVmRuntime + updateEphemeralVmRuntimeStatus } from '../shared/ephemeral-vm-runtime-store' import type { EphemeralVmRuntimeRecord } from '../shared/ephemeral-vm-runtimes' -import { getEphemeralVmRecipeResultConnection } from '../shared/ephemeral-vm-recipes' import { runEphemeralVmRecipeCleanup, runEphemeralVmRecipeResume, @@ -16,6 +14,10 @@ import { type EphemeralVmRecipeStartSuccess } from './ephemeral-vm-recipe-runner' import { cleanupFailedEphemeralVmStart } from './ephemeral-vm-failed-start-cleanup' +import { + persistProvisionedEphemeralVmRuntime, + prepareEphemeralVmCompatibilityPersistence +} from './ephemeral-vm-runtime-provisioning-persistence' import { provisionedRootChangedDuringResume } from './ephemeral-vm-resume-integrity' export type ProvisionEphemeralVmRuntimeArgs = { @@ -99,6 +101,7 @@ const cleanupInFlight = new Map export async function provisionEphemeralVmRuntime( args: ProvisionEphemeralVmRuntimeArgs ): Promise { + const compatibility = prepareEphemeralVmCompatibilityPersistence(args) const start = await runEphemeralVmRecipeStart({ repoPath: args.repoPath, recipe: args.recipe, @@ -109,7 +112,8 @@ export async function provisionEphemeralVmRuntime( repoUrl: args.repoUrl, branch: args.branch, ref: args.ref, - orcaVersion: args.orcaVersion + orcaVersion: args.orcaVersion, + ...(compatibility ? { instanceId: compatibility.instanceId } : {}) }, signal: args.signal, onStdout: args.onStdout, @@ -125,24 +129,7 @@ export async function provisionEphemeralVmRuntime( return { ok: false, start } } - const now = args.now ?? Date.now() - const connection = getEphemeralVmRecipeResultConnection(start.result) - const runtime = upsertEphemeralVmRuntime(args.userDataPath, { - id: start.context.instanceId ?? start.context.recipeId, - recipeId: args.recipe.id, - recipe: args.recipe, - ...(args.repoId ? { repoId: args.repoId } : {}), - ...(args.projectId ? { projectId: args.projectId } : {}), - ...(args.workspaceId ? { workspaceId: args.workspaceId } : {}), - ...(args.workspaceName ? { workspaceName: args.workspaceName } : {}), - status: 'running', - connectionMode: connection.type, - cleanupStatus: args.recipe.destroyDisabled ? 'disabled' : 'not_started', - ...(args.recipe.destroyDisabled ? { cleanupDisabled: true } : {}), - createdAt: now, - updatedAt: now, - recipeResult: start.result - }) + const runtime = await persistProvisionedEphemeralVmRuntime(args, start, compatibility) return { ok: true, start, runtime } } diff --git a/src/shared/bounded-secure-json-file.ts b/src/shared/bounded-secure-json-file.ts index c9da2b9ce6f..75f048b3480 100644 --- a/src/shared/bounded-secure-json-file.ts +++ b/src/shared/bounded-secure-json-file.ts @@ -4,7 +4,8 @@ import { writeSecureFile } from './secure-file' export function writeSecureJsonFileWithinLimit( targetPath: string, value: unknown, - maxBytes: number + maxBytes: number, + options: { durable?: boolean } = {} ): void { - writeSecureFile(targetPath, stringifyJsonWithinByteLimit(value, maxBytes).serialized) + writeSecureFile(targetPath, stringifyJsonWithinByteLimit(value, maxBytes).serialized, options) } diff --git a/src/shared/ephemeral-vm-runtime-feature-store.ts b/src/shared/ephemeral-vm-runtime-feature-store.ts new file mode 100644 index 00000000000..4678c5b6a24 --- /dev/null +++ b/src/shared/ephemeral-vm-runtime-feature-store.ts @@ -0,0 +1,247 @@ +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import { z } from 'zod' +import { readNodeFileSyncWithinLimit } from './node-bounded-file-reader' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from './node-bounded-json-stringify' +import { writeSecureJsonFileWithinLimit } from './bounded-secure-json-file' +import { hardenExistingSecureFile } from './secure-file' +import { + EphemeralVmRuntimeRecordSchema, + type EphemeralVmRuntimeRecord +} from './ephemeral-vm-runtimes' + +const EPHEMERAL_VM_RUNTIME_FEATURES_FILE = 'orca-ephemeral-vm-runtime-features.json' +export const MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES = 1024 * 1024 + +const EphemeralVmRuntimeFeatureEntrySchema = z + .object({ + id: z.string().min(1), + recipeId: z.string().min(1), + createdAt: z.number().finite(), + recipeCheckoutMode: z.enum(['orca-worktree', 'provisioned-root']).optional(), + resultCheckoutMode: z.literal('provisioned-root').optional() + }) + .strict() + +export type EphemeralVmRuntimeFeatureEntry = z.infer + +const EphemeralVmRuntimeFeatureStoreSchema = z + .object({ + version: z.literal(1), + records: z.array(z.unknown()) + }) + .strict() + +export type EphemeralVmRuntimeFeatureStoreSnapshot = + | { + writable: true + features: EphemeralVmRuntimeFeatureEntry[] + retainedRecords: unknown[] + } + | { + writable: false + features: [] + retainedRecords: [] + } + +export function getEphemeralVmRuntimeFeatureStorePath(userDataPath: string): string { + return join(userDataPath, EPHEMERAL_VM_RUNTIME_FEATURES_FILE) +} + +export function assertEphemeralVmRuntimeCheckoutModeCanPersist( + userDataPath: string, + args: { + id: string + recipeId: string + createdAt: number + checkoutMode: NonNullable['checkoutMode']> + } +): void { + const snapshot = readEphemeralVmRuntimeFeatureStore(userDataPath) + if (!snapshot.writable) { + throw new Error('Could not preserve ephemeral VM runtime compatibility metadata.') + } + const required: EphemeralVmRuntimeFeatureEntry = { + id: args.id, + recipeId: args.recipeId, + createdAt: args.createdAt, + recipeCheckoutMode: args.checkoutMode, + ...(args.checkoutMode === 'provisioned-root' ? { resultCheckoutMode: 'provisioned-root' } : {}) + } + try { + assertFeatureStoreCanPersist(snapshot, mergeFeatureEntries(snapshot.features, [required])) + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new Error( + 'Could not preserve ephemeral VM runtime compatibility metadata; the feature store exceeds its durable capacity.' + ) + } + throw error + } +} + +export function readEphemeralVmRuntimeFeatureStore( + userDataPath: string +): EphemeralVmRuntimeFeatureStoreSnapshot { + const path = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + if (!existsSync(path)) { + return { writable: true, features: [], retainedRecords: [] } + } + try { + hardenExistingSecureFile(path) + const parsed = EphemeralVmRuntimeFeatureStoreSchema.parse( + JSON.parse( + readNodeFileSyncWithinLimit( + path, + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES + ).buffer.toString('utf8') + ) + ) + return parseFeatureRecords(parsed.records) + } catch { + return { writable: false, features: [], retainedRecords: [] } + } +} + +export function writeEphemeralVmRuntimeFeatureStore( + userDataPath: string, + snapshot: EphemeralVmRuntimeFeatureStoreSnapshot, + features: EphemeralVmRuntimeFeatureEntry[] +): void { + if (!snapshot.writable) { + throw new Error('The ephemeral VM runtime feature store is not writable.') + } + writeSecureJsonFileWithinLimit( + getEphemeralVmRuntimeFeatureStorePath(userDataPath), + runtimeFeatureStoreValue(snapshot, features), + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES, + { durable: true } + ) +} + +function assertFeatureStoreCanPersist( + snapshot: EphemeralVmRuntimeFeatureStoreSnapshot, + features: EphemeralVmRuntimeFeatureEntry[] +): void { + if (!snapshot.writable) { + throw new Error('The ephemeral VM runtime feature store is not writable.') + } + stringifyJsonWithinByteLimit( + runtimeFeatureStoreValue(snapshot, features), + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES + ) +} + +function mergeFeatureEntries( + existing: readonly EphemeralVmRuntimeFeatureEntry[], + required: readonly EphemeralVmRuntimeFeatureEntry[] +): EphemeralVmRuntimeFeatureEntry[] { + const merged = new Map(existing.map((entry) => [featureIdentity(entry), entry])) + for (const entry of required) { + merged.set(featureIdentity(entry), entry) + } + return sortFeatures([...merged.values()]) +} + +export function featureEntryFromRuntime( + runtime: EphemeralVmRuntimeRecord +): EphemeralVmRuntimeFeatureEntry | null { + const recipeCheckoutMode = runtime.recipe?.checkoutMode + const resultCheckoutMode = + runtime.recipeResult.schemaVersion === 2 ? runtime.recipeResult.checkoutMode : undefined + if (!recipeCheckoutMode && !resultCheckoutMode) { + return null + } + return { + id: runtime.id, + recipeId: runtime.recipeId, + createdAt: runtime.createdAt, + ...(recipeCheckoutMode ? { recipeCheckoutMode } : {}), + ...(resultCheckoutMode ? { resultCheckoutMode } : {}) + } +} + +export function restoreRuntimeFeatures( + runtime: EphemeralVmRuntimeRecord, + features: readonly EphemeralVmRuntimeFeatureEntry[] +): EphemeralVmRuntimeRecord { + const feature = features.find((entry) => featureIdentity(entry) === featureIdentity(runtime)) + if (!feature) { + return runtime + } + return EphemeralVmRuntimeRecordSchema.parse({ + ...runtime, + ...(runtime.recipe && feature.recipeCheckoutMode + ? { recipe: { ...runtime.recipe, checkoutMode: feature.recipeCheckoutMode } } + : {}), + ...(feature.resultCheckoutMode + ? { + recipeResult: { + ...runtime.recipeResult, + schemaVersion: 2, + checkoutMode: feature.resultCheckoutMode + } + } + : {}) + }) +} + +export function runtimeFeaturesEqual( + left: EphemeralVmRuntimeRecord, + right: EphemeralVmRuntimeRecord +): boolean { + return ( + JSON.stringify(featureEntryFromRuntime(left)) === JSON.stringify(featureEntryFromRuntime(right)) + ) +} + +export function featureIdentity( + value: Pick +): string { + return `${value.id}\0${value.recipeId}\0${value.createdAt}` +} + +function parseFeatureRecords(records: unknown[]): EphemeralVmRuntimeFeatureStoreSnapshot { + const features: EphemeralVmRuntimeFeatureEntry[] = [] + const retainedRecords: unknown[] = [] + const identities = new Map() + for (const record of records) { + const parsed = EphemeralVmRuntimeFeatureEntrySchema.safeParse(record) + if (!parsed.success) { + retainedRecords.push(record) + continue + } + const identity = featureIdentity(parsed.data) + const serialized = JSON.stringify(parsed.data) + const existing = identities.get(identity) + if (existing && existing !== serialized) { + return { writable: false, features: [], retainedRecords: [] } + } + if (!existing) { + identities.set(identity, serialized) + features.push(parsed.data) + } + } + return { writable: true, features: sortFeatures(features), retainedRecords } +} + +function sortFeatures( + features: readonly EphemeralVmRuntimeFeatureEntry[] +): EphemeralVmRuntimeFeatureEntry[] { + return [...features].sort((left, right) => + featureIdentity(left).localeCompare(featureIdentity(right)) + ) +} + +function runtimeFeatureStoreValue( + snapshot: Extract, + features: EphemeralVmRuntimeFeatureEntry[] +): { version: 1; records: unknown[] } { + return { + version: 1, + records: [...sortFeatures(features), ...snapshot.retainedRecords] + } +} diff --git a/src/shared/ephemeral-vm-runtime-rollback-projection.ts b/src/shared/ephemeral-vm-runtime-rollback-projection.ts new file mode 100644 index 00000000000..40dc37429a0 --- /dev/null +++ b/src/shared/ephemeral-vm-runtime-rollback-projection.ts @@ -0,0 +1,44 @@ +import { featureIdentity } from './ephemeral-vm-runtime-feature-store' +import { + RollbackEphemeralVmRuntimeRecordSchema, + type EphemeralVmRuntimeRecord +} from './ephemeral-vm-runtimes' + +export function projectRuntimeForRollback( + runtime: EphemeralVmRuntimeRecord +): EphemeralVmRuntimeRecord { + const recipe = runtime.recipe + ? (({ checkoutMode: _checkoutMode, ...rollbackRecipe }) => rollbackRecipe)(runtime.recipe) + : undefined + const recipeResult = + runtime.recipeResult.schemaVersion === 2 + ? (({ checkoutMode: _checkoutMode, ...rollbackResult }) => ({ + ...rollbackResult, + schemaVersion: 1 as const + }))(runtime.recipeResult) + : runtime.recipeResult + return RollbackEphemeralVmRuntimeRecordSchema.parse({ + ...runtime, + ...(recipe ? { recipe } : {}), + recipeResult + }) +} + +export function mergeRuntimeFeatures( + existing: readonly T[], + required: readonly T[] +): T[] { + const merged = new Map(existing.map((entry) => [featureIdentity(entry), entry])) + for (const entry of required) { + merged.set(featureIdentity(entry), entry) + } + return [...merged.values()].sort((left, right) => + featureIdentity(left).localeCompare(featureIdentity(right)) + ) +} + +export function runtimeFeatureListsEqual< + T extends { id: string; recipeId: string; createdAt: number } +>(left: readonly T[], right: readonly T[]): boolean { + return JSON.stringify(left) === JSON.stringify(right) +} diff --git a/src/shared/ephemeral-vm-runtime-store-rollback.test.ts b/src/shared/ephemeral-vm-runtime-store-rollback.test.ts new file mode 100644 index 00000000000..47c2ffb1f83 --- /dev/null +++ b/src/shared/ephemeral-vm-runtime-store-rollback.test.ts @@ -0,0 +1,288 @@ +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + truncateSync, + utimesSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + getEphemeralVmRuntimeFeatureStorePath, + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES +} from './ephemeral-vm-runtime-feature-store' +import { + EphemeralVmRuntimeStoreError, + getEphemeralVmRuntimeStorePath, + listEphemeralVmRuntimes, + updateEphemeralVmRuntimeStatus, + upsertEphemeralVmRuntime +} from './ephemeral-vm-runtime-store' +import { + EphemeralVmRuntimeStoreSchema, + RollbackEphemeralVmRuntimeStoreSchema, + type EphemeralVmRuntimeRecord +} from './ephemeral-vm-runtimes' + +function runtimeRecord( + overrides: Partial = {} +): EphemeralVmRuntimeRecord { + return { + id: 'ordinary-runtime', + recipeId: 'ordinary-recipe', + recipe: { + id: 'ordinary-recipe', + name: 'Ordinary VM', + create: './create.sh', + destroy: './destroy.sh' + }, + status: 'running', + cleanupStatus: 'not_started', + createdAt: 1_000, + updatedAt: 1_000, + recipeResult: { + schemaVersion: 1, + connection: { + type: 'ssh', + projectRoot: '/workspace/ordinary', + target: { + label: 'Ordinary VM', + host: 'ordinary.example.com', + port: 22, + username: 'developer' + } + }, + userData: { resourceId: 'ordinary-resource' } + }, + ...overrides + } +} + +function provisionedRootRecord(): EphemeralVmRuntimeRecord { + return runtimeRecord({ + id: 'provisioned-runtime', + recipeId: 'provisioned-recipe', + recipe: { + id: 'provisioned-recipe', + name: 'Provisioned VM', + create: './create.sh', + destroy: './destroy.sh', + checkoutMode: 'provisioned-root' + }, + createdAt: 2_000, + updatedAt: 2_000, + recipeResult: { + schemaVersion: 2, + checkoutMode: 'provisioned-root', + connection: { + type: 'ssh', + projectRoot: '/workspace/provisioned', + target: { + label: 'Provisioned VM', + host: 'provisioned.example.com', + port: 22, + username: 'developer' + } + }, + userData: { resourceId: 'provisioned-resource' } + } + }) +} + +describe('ephemeral VM runtime store rollback projection', () => { + const tempDirs: string[] = [] + + afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + }) + + function makeUserDataPath(): string { + const path = mkdtempSync(join(tmpdir(), 'orca-vm-rollback-store-')) + tempDirs.push(path) + return path + } + + it('keeps a mixed store readable by the rollback schema and restores new fields', () => { + const userDataPath = makeUserDataPath() + const ordinary = upsertEphemeralVmRuntime(userDataPath, runtimeRecord()) + const provisioned = upsertEphemeralVmRuntime(userDataPath, provisionedRootRecord()) + + const persisted = JSON.parse(readFileSync(getEphemeralVmRuntimeStorePath(userDataPath), 'utf8')) + expect(RollbackEphemeralVmRuntimeStoreSchema.parse(persisted).runtimes).toHaveLength(2) + expect(persisted.runtimes[0].recipe).not.toHaveProperty('checkoutMode') + expect(persisted.runtimes[0].recipeResult).toMatchObject({ schemaVersion: 1 }) + expect(listEphemeralVmRuntimes(userDataPath)).toEqual([provisioned, ordinary]) + }) + + it('keeps ordinary v1 bytes and sidecar behavior unchanged', () => { + const userDataPath = makeUserDataPath() + const runtime = runtimeRecord() + const expected = JSON.stringify( + EphemeralVmRuntimeStoreSchema.parse({ version: 1, runtimes: [runtime] }) + ) + + upsertEphemeralVmRuntime(userDataPath, runtime) + + expect(readFileSync(getEphemeralVmRuntimeStorePath(userDataPath), 'utf8')).toBe(expected) + expect(existsSync(getEphemeralVmRuntimeFeatureStorePath(userDataPath))).toBe(false) + }) + + it('projects an explicit ordinary checkout mode without changing its current meaning', () => { + const userDataPath = makeUserDataPath() + const runtime = runtimeRecord({ + recipe: { ...runtimeRecord().recipe!, checkoutMode: 'orca-worktree' } + }) + + upsertEphemeralVmRuntime(userDataPath, runtime) + + const persisted = JSON.parse(readFileSync(getEphemeralVmRuntimeStorePath(userDataPath), 'utf8')) + expect(RollbackEphemeralVmRuntimeStoreSchema.safeParse(persisted).success).toBe(true) + expect(listEphemeralVmRuntimes(userDataPath)).toEqual([runtime]) + }) + + it('does not rewrite unchanged features when runtime order differs from feature order', () => { + const userDataPath = makeUserDataPath() + const older = { + ...provisionedRootRecord(), + id: 'a-runtime', + recipeId: 'a-recipe', + createdAt: 1_000 + } + const newer = { + ...provisionedRootRecord(), + id: 'z-runtime', + recipeId: 'z-recipe', + createdAt: 2_000 + } + upsertEphemeralVmRuntime(userDataPath, older) + upsertEphemeralVmRuntime(userDataPath, newer) + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + const oldTimestamp = new Date('2020-01-01T00:00:00.000Z') + utimesSync(featurePath, oldTimestamp, oldTimestamp) + const beforeBytes = readFileSync(featurePath, 'utf8') + const beforeMtime = statSync(featurePath).mtimeMs + + updateEphemeralVmRuntimeStatus(userDataPath, newer.id, { status: 'suspended' }) + + expect(readFileSync(featurePath, 'utf8')).toBe(beforeBytes) + expect(statSync(featurePath).mtimeMs).toBe(beforeMtime) + }) + + it('migrates current-main poisoned bytes when they are first read', () => { + const userDataPath = makeUserDataPath() + const poisoned = { + version: 1 as const, + runtimes: [provisionedRootRecord(), runtimeRecord()] + } + writeFileSync( + getEphemeralVmRuntimeStorePath(userDataPath), + JSON.stringify(EphemeralVmRuntimeStoreSchema.parse(poisoned)) + ) + + expect(listEphemeralVmRuntimes(userDataPath)).toEqual(poisoned.runtimes) + expect( + RollbackEphemeralVmRuntimeStoreSchema.safeParse( + JSON.parse(readFileSync(getEphemeralVmRuntimeStorePath(userDataPath), 'utf8')) + ).success + ).toBe(true) + }) + + it('carries rollback lifecycle mutations through re-upgrade', () => { + const userDataPath = makeUserDataPath() + upsertEphemeralVmRuntime(userDataPath, runtimeRecord()) + upsertEphemeralVmRuntime(userDataPath, provisionedRootRecord()) + const path = getEphemeralVmRuntimeStorePath(userDataPath) + const rollback = RollbackEphemeralVmRuntimeStoreSchema.parse( + JSON.parse(readFileSync(path, 'utf8')) + ) + writeFileSync( + path, + JSON.stringify({ + version: 1, + runtimes: rollback.runtimes.map((runtime) => + runtime.id === 'provisioned-runtime' + ? { ...runtime, status: 'cleaned', cleanupStatus: 'succeeded', updatedAt: 3_000 } + : { ...runtime, status: 'suspended', updatedAt: 3_000 } + ) + }) + ) + + expect(listEphemeralVmRuntimes(userDataPath)).toEqual([ + expect.objectContaining({ + id: 'provisioned-runtime', + status: 'cleaned', + cleanupStatus: 'succeeded', + recipe: expect.objectContaining({ checkoutMode: 'provisioned-root' }), + recipeResult: expect.objectContaining({ + schemaVersion: 2, + checkoutMode: 'provisioned-root' + }) + }), + expect.objectContaining({ id: 'ordinary-runtime', status: 'suspended' }) + ]) + }) + + it('preserves unknown feature records while valid siblings remain usable', () => { + const userDataPath = makeUserDataPath() + upsertEphemeralVmRuntime(userDataPath, runtimeRecord()) + upsertEphemeralVmRuntime(userDataPath, provisionedRootRecord()) + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + const featureStore = JSON.parse(readFileSync(featurePath, 'utf8')) + const futureRecord = { kind: 'future-runtime-feature', payload: { version: 3 } } + writeFileSync( + featurePath, + JSON.stringify({ ...featureStore, records: [...featureStore.records, futureRecord] }) + ) + + expect(listEphemeralVmRuntimes(userDataPath)).toHaveLength(2) + updateEphemeralVmRuntimeStatus(userDataPath, 'ordinary-runtime', { status: 'suspended' }) + expect(JSON.parse(readFileSync(featurePath, 'utf8')).records).toContainEqual(futureRecord) + }) + + it.each([ + ['malformed', '{ nope'], + ['future-version', JSON.stringify({ version: 2, records: [] })] + ])( + 'preserves an unreadable %s feature sidecar while keeping v1 records accessible', + (_, bytes) => { + const userDataPath = makeUserDataPath() + upsertEphemeralVmRuntime(userDataPath, runtimeRecord()) + upsertEphemeralVmRuntime(userDataPath, provisionedRootRecord()) + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + writeFileSync(featurePath, bytes) + + expect(listEphemeralVmRuntimes(userDataPath).map((runtime) => runtime.id)).toEqual([ + 'provisioned-runtime', + 'ordinary-runtime' + ]) + updateEphemeralVmRuntimeStatus(userDataPath, 'ordinary-runtime', { status: 'suspended' }) + expect(readFileSync(featurePath, 'utf8')).toBe(bytes) + } + ) + + it('publishes lifecycle authority before an unreadable feature companion', () => { + const userDataPath = makeUserDataPath() + upsertEphemeralVmRuntime(userDataPath, runtimeRecord()) + const featurePath = getEphemeralVmRuntimeFeatureStorePath(userDataPath) + writeFileSync(featurePath, '{}') + truncateSync(featurePath, MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES + 1) + expect(() => upsertEphemeralVmRuntime(userDataPath, provisionedRootRecord())).toThrow( + EphemeralVmRuntimeStoreError + ) + const persisted = JSON.parse(readFileSync(getEphemeralVmRuntimeStorePath(userDataPath), 'utf8')) + expect(RollbackEphemeralVmRuntimeStoreSchema.parse(persisted).runtimes).toHaveLength(2) + expect(readFileSync(featurePath, 'utf8')).toHaveLength( + MAX_EPHEMERAL_VM_RUNTIME_FEATURE_STORE_FILE_BYTES + 1 + ) + expect(listEphemeralVmRuntimes(userDataPath).map((runtime) => runtime.id)).toEqual([ + 'provisioned-runtime', + 'ordinary-runtime' + ]) + }) +}) diff --git a/src/shared/ephemeral-vm-runtime-store.ts b/src/shared/ephemeral-vm-runtime-store.ts index ad26bb2950e..722e45caf3c 100644 --- a/src/shared/ephemeral-vm-runtime-store.ts +++ b/src/shared/ephemeral-vm-runtime-store.ts @@ -4,9 +4,24 @@ import { JsonStringifyByteLimitError } from './node-bounded-json-stringify' import { readNodeFileSyncWithinLimit } from './node-bounded-file-reader' import { writeSecureJsonFileWithinLimit } from './bounded-secure-json-file' import { hardenExistingSecureFile } from './secure-file' +import { + featureEntryFromRuntime, + featureIdentity, + readEphemeralVmRuntimeFeatureStore, + restoreRuntimeFeatures, + runtimeFeaturesEqual, + writeEphemeralVmRuntimeFeatureStore, + type EphemeralVmRuntimeFeatureStoreSnapshot +} from './ephemeral-vm-runtime-feature-store' +import { + mergeRuntimeFeatures, + projectRuntimeForRollback, + runtimeFeatureListsEqual +} from './ephemeral-vm-runtime-rollback-projection' import { EphemeralVmRuntimeRecordSchema, EphemeralVmRuntimeStoreSchema, + RollbackEphemeralVmRuntimeStoreSchema, type EphemeralVmCleanupStatus, type EphemeralVmRuntimeRecord, type EphemeralVmRuntimeStatus, @@ -33,7 +48,7 @@ export function getEphemeralVmRuntimeStorePath(userDataPath: string): string { } export function listEphemeralVmRuntimes(userDataPath: string): EphemeralVmRuntimeRecord[] { - return readEphemeralVmRuntimeStore(userDataPath).runtimes + return readEphemeralVmRuntimeStore(userDataPath).store.runtimes } export function upsertEphemeralVmRuntime( @@ -41,16 +56,61 @@ export function upsertEphemeralVmRuntime( record: EphemeralVmRuntimeRecord ): EphemeralVmRuntimeRecord { const parsed = EphemeralVmRuntimeRecordSchema.parse(record) - const store = readEphemeralVmRuntimeStore(userDataPath) - writeEphemeralVmRuntimeStore(userDataPath, { - version: 1, - runtimes: [...store.runtimes.filter((entry) => entry.id !== parsed.id), parsed].sort( - compareRuntimeRecords + const loaded = readEphemeralVmRuntimeStore(userDataPath) + const previous = loaded.store.runtimes.find((entry) => entry.id === parsed.id) + if ( + previous && + featureIdentity(previous) === featureIdentity(parsed) && + !runtimeFeaturesEqual(previous, parsed) + ) { + throw new EphemeralVmRuntimeStoreError( + 'invalid_argument', + `Cannot change compatibility features for ephemeral VM runtime: ${parsed.id}` ) - }) + } + writeEphemeralVmRuntimeStore( + userDataPath, + { + version: 1, + runtimes: [...loaded.store.runtimes.filter((entry) => entry.id !== parsed.id), parsed].sort( + compareRuntimeRecords + ) + }, + loaded.features + ) return parsed } +export function upsertEphemeralVmRuntimeRollbackRecovery( + userDataPath: string, + record: EphemeralVmRuntimeRecord +): void { + const parsed = EphemeralVmRuntimeRecordSchema.parse(record) + const loaded = readEphemeralVmRuntimeStore(userDataPath) + const path = getEphemeralVmRuntimeStorePath(userDataPath) + try { + writeSecureJsonFileWithinLimit( + path, + RollbackEphemeralVmRuntimeStoreSchema.parse({ + version: 1, + runtimes: [...loaded.store.runtimes.filter((entry) => entry.id !== parsed.id), parsed] + .sort(compareRuntimeRecords) + .map(projectRuntimeForRollback) + }), + MAX_EPHEMERAL_VM_RUNTIME_STORE_FILE_BYTES, + { durable: true } + ) + } catch (error) { + if (error instanceof JsonStringifyByteLimitError) { + throw new EphemeralVmRuntimeStoreError( + 'runtime_error', + `Could not write Orca ephemeral VM runtimes at ${path}; the store exceeds its durable capacity.` + ) + } + throw error + } +} + export function updateEphemeralVmRuntimeStatus( userDataPath: string, id: string, @@ -68,8 +128,8 @@ export function updateEphemeralVmRuntimeStatus( updatedAt?: number } ): EphemeralVmRuntimeRecord { - const store = readEphemeralVmRuntimeStore(userDataPath) - const existing = store.runtimes.find((entry) => entry.id === id) + const loaded = readEphemeralVmRuntimeStore(userDataPath) + const existing = loaded.store.runtimes.find((entry) => entry.id === id) if (!existing) { throw new EphemeralVmRuntimeStoreError( 'invalid_argument', @@ -105,12 +165,16 @@ export function updateEphemeralVmRuntimeStatus( ...(args.recipeResult ? { recipeResult: args.recipeResult } : {}), updatedAt: args.updatedAt ?? Date.now() }) - writeEphemeralVmRuntimeStore(userDataPath, { - version: 1, - runtimes: store.runtimes - .map((entry) => (entry.id === id ? next : entry)) - .sort(compareRuntimeRecords) - }) + writeEphemeralVmRuntimeStore( + userDataPath, + { + version: 1, + runtimes: loaded.store.runtimes + .map((entry) => (entry.id === id ? next : entry)) + .sort(compareRuntimeRecords) + }, + loaded.features + ) return next } @@ -118,42 +182,61 @@ export function removeEphemeralVmRuntime( userDataPath: string, id: string ): EphemeralVmRuntimeRecord { - const store = readEphemeralVmRuntimeStore(userDataPath) - const existing = store.runtimes.find((entry) => entry.id === id) + const loaded = readEphemeralVmRuntimeStore(userDataPath) + const existing = loaded.store.runtimes.find((entry) => entry.id === id) if (!existing) { throw new EphemeralVmRuntimeStoreError( 'invalid_argument', `Unknown ephemeral VM runtime: ${id}` ) } - writeEphemeralVmRuntimeStore(userDataPath, { - version: 1, - runtimes: store.runtimes.filter((entry) => entry.id !== id) - }) + writeEphemeralVmRuntimeStore( + userDataPath, + { + version: 1, + runtimes: loaded.store.runtimes.filter((entry) => entry.id !== id) + }, + loaded.features + ) return existing } -function readEphemeralVmRuntimeStore(userDataPath: string): EphemeralVmRuntimeStore { +type LoadedEphemeralVmRuntimeStore = { + store: EphemeralVmRuntimeStore + features: EphemeralVmRuntimeFeatureStoreSnapshot +} + +function readEphemeralVmRuntimeStore(userDataPath: string): LoadedEphemeralVmRuntimeStore { const path = getEphemeralVmRuntimeStorePath(userDataPath) if (!existsSync(path)) { - return { version: 1, runtimes: [] } + return { + store: { version: 1, runtimes: [] }, + features: readEphemeralVmRuntimeFeatureStore(userDataPath) + } } try { hardenExistingSecureFile(path) - const parsed = EphemeralVmRuntimeStoreSchema.parse( - JSON.parse( - readNodeFileSyncWithinLimit( - path, - MAX_EPHEMERAL_VM_RUNTIME_STORE_FILE_BYTES - ).buffer.toString('utf8') + const persisted = JSON.parse( + readNodeFileSyncWithinLimit(path, MAX_EPHEMERAL_VM_RUNTIME_STORE_FILE_BYTES).buffer.toString( + 'utf8' ) ) - return { + const parsed = EphemeralVmRuntimeStoreSchema.parse(persisted) + const features = readEphemeralVmRuntimeFeatureStore(userDataPath) + const store: EphemeralVmRuntimeStore = { version: 1, runtimes: parsed.runtimes - .map((entry) => EphemeralVmRuntimeRecordSchema.parse(entry)) + .map((entry) => restoreRuntimeFeatures(entry, features.features)) .sort(compareRuntimeRecords) } + if (features.writable && !RollbackEphemeralVmRuntimeStoreSchema.safeParse(persisted).success) { + try { + writeEphemeralVmRuntimeStore(userDataPath, store, features) + } catch { + // Why: a failed migration must not block cleanup through the still-readable current shape. + } + } + return { store, features } } catch { throw new EphemeralVmRuntimeStoreError( 'runtime_error', @@ -162,14 +245,47 @@ function readEphemeralVmRuntimeStore(userDataPath: string): EphemeralVmRuntimeSt } } -function writeEphemeralVmRuntimeStore(userDataPath: string, store: EphemeralVmRuntimeStore): void { +function writeEphemeralVmRuntimeStore( + userDataPath: string, + store: EphemeralVmRuntimeStore, + features: EphemeralVmRuntimeFeatureStoreSnapshot +): void { const path = getEphemeralVmRuntimeStorePath(userDataPath) try { + const parsed = EphemeralVmRuntimeStoreSchema.parse(store) + const requiredFeatures = mergeRuntimeFeatures( + [], + parsed.runtimes.flatMap((entry) => { + const feature = featureEntryFromRuntime(entry) + return feature ? [feature] : [] + }) + ) + const preparedFeatures = mergeRuntimeFeatures(features.features, requiredFeatures) writeSecureJsonFileWithinLimit( path, - EphemeralVmRuntimeStoreSchema.parse(store), - MAX_EPHEMERAL_VM_RUNTIME_STORE_FILE_BYTES + RollbackEphemeralVmRuntimeStoreSchema.parse({ + version: 1, + runtimes: parsed.runtimes.map(projectRuntimeForRollback) + }), + MAX_EPHEMERAL_VM_RUNTIME_STORE_FILE_BYTES, + { durable: preparedFeatures.length > 0 || features.features.length > 0 } ) + if (!features.writable && requiredFeatures.length > 0) { + throw new EphemeralVmRuntimeStoreError( + 'runtime_error', + 'Could not preserve ephemeral VM runtime compatibility metadata.' + ) + } + if (features.writable && !runtimeFeatureListsEqual(features.features, preparedFeatures)) { + writeEphemeralVmRuntimeFeatureStore(userDataPath, features, preparedFeatures) + } + if (features.writable && !runtimeFeatureListsEqual(preparedFeatures, requiredFeatures)) { + try { + writeEphemeralVmRuntimeFeatureStore(userDataPath, features, requiredFeatures) + } catch { + // Stale feature records do not match any persisted runtime identity. + } + } } catch (error) { if (error instanceof JsonStringifyByteLimitError) { throw new EphemeralVmRuntimeStoreError( diff --git a/src/shared/ephemeral-vm-runtimes.ts b/src/shared/ephemeral-vm-runtimes.ts index c8bb859f06e..97a1a48fbf2 100644 --- a/src/shared/ephemeral-vm-runtimes.ts +++ b/src/shared/ephemeral-vm-runtimes.ts @@ -1,5 +1,9 @@ import { z } from 'zod' -import { EphemeralVmRecipeResultSchema } from './ephemeral-vm-recipes' +import { + EphemeralVmRecipeConnectionResultSchema, + EphemeralVmRecipeLegacyResultSchema, + EphemeralVmRecipeResultSchema +} from './ephemeral-vm-recipes' export const EphemeralVmRuntimeStatusSchema = z.enum([ 'provisioning', @@ -72,3 +76,29 @@ export const EphemeralVmRuntimeStoreSchema = z.object({ }) export type EphemeralVmRuntimeStore = z.infer + +const RollbackEphemeralVmRuntimeRecipeSchema = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + create: z.string().min(1), + description: z.string().min(1).optional(), + suspend: z.string().min(1).optional(), + resume: z.string().min(1).optional(), + destroy: z.string().min(1).optional(), + destroyDisabled: z.boolean().optional() + }) + .strict() + +export const RollbackEphemeralVmRuntimeRecordSchema = EphemeralVmRuntimeRecordSchema.extend({ + recipe: RollbackEphemeralVmRuntimeRecipeSchema.optional(), + recipeResult: z.union([ + EphemeralVmRecipeLegacyResultSchema, + EphemeralVmRecipeConnectionResultSchema + ]) +}) + +export const RollbackEphemeralVmRuntimeStoreSchema = z.object({ + version: z.literal(1), + runtimes: z.array(RollbackEphemeralVmRuntimeRecordSchema) +})