mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
feat(acp): a managed runtime installs only when asked, with progress and cancel (#29023)
* feat(acp): a managed runtime installs only when asked, with progress and cancel One installation per runtime owns every change to its releases: it reclaims old ones once, first, then installs one at a time. An install checks free space, downloads with byte progress, extracts, hashes each member (abortable), writes the receipt, then publishes by rename, the commit point: a cancel after it reports success. Reinstall rehashes the pinned release without downloading. A status read waits, bounded, for a newer revision and rereads the installed release from disk. * fix(runtime): Install repairs a damaged copy, says when unzip is missing before downloading, and reads a failed download plainly - Install over a damaged pinned copy downloads and verifies a fresh one, then swaps the damaged folder into staging, so there is no dead end; status reports the damaged fact. - A host that cannot unpack the archive (no unzip) reports it in status and refuses Install without downloading. - A network failure while downloading reads "Couldn't download …"; the raw error goes to the log. - Reclamation skips the whole pass when a release folder could not be read. * fix(runtime): a status read that can't read the installed files fails instead of saying not installed, and extraction uses the extractor the preflight found * fix(runtime): a host that can't report free disk space still installs, and says so in the log * fix(runtime): a download that can't be fixed by retrying says so instead of blaming the network * fix(runtime): Install unpacks the release itself, checks it starts before publishing, and Cancel answers with the outcome - The managed install unzips in-process (fflate, bundled into main), so a host without unzip can install; the unzip preflight and the missingProgram status field are gone. The Node runtime download keeps its own extractor. - A runtime may provide validate(): Install starts the downloaded copy once before publishing it, and an existing copy before reusing it; a failure reads "<agent> couldn't start on this host". - No 'cancelling' phase: Cancel resolves once the work has unwound and returns cancelled, or succeeded when it had already committed. - Byte progress every 250 ms; status rereads disk and PATH only on a first read or a phase change. - The declared download size is checked only for an uncompressed body. - An existing copy whose files hash correctly but whose receipt is missing gets its receipt rewritten instead of reading as damaged forever. * refactor(runtime): move the steps that put one release on disk out of the installer The installer class now holds only its lifecycle: the shared operation state, the one-at-a-time queue, install, cancel, status, reclaim and progress publishing. Reusing a whole copy, the free space check, the download, unpacking, checking each file, the start check, the receipt, the publishing rename and the failure sentences move unchanged into managed-runtime-release-install.ts, which gets the pin, folder, label, signal and two callbacks (progress, and the commit point right after the rename) instead of the class. No behavior change; this keeps the installer well under the file-length limit once Remove lands in it. * chore(deps): lock fflate for the in-process runtime unzip * fix(runtime): a failed download blames the release only for a proven mismatch, so a refused proxy reads as a network problem * fix(runtime): runtime downloads skip the HTTP cache, so Remove frees the space and nothing is stored twice
This commit is contained in:
@@ -20,6 +20,7 @@ const BUNDLED_MAIN_DEPENDENCIES = new Set([
|
||||
'@xterm/addon-serialize',
|
||||
'tldts',
|
||||
'smol-toml',
|
||||
'fflate',
|
||||
// Why: Windows NSIS deploys app.asar before external resources; bootstrap must
|
||||
// not race the later resources/node_modules copy.
|
||||
'zod'
|
||||
|
||||
@@ -192,6 +192,7 @@
|
||||
"@xterm/headless": "6.1.0-beta.303",
|
||||
"agent-browser": "~0.27.0",
|
||||
"electron-updater": "^6.8.10",
|
||||
"fflate": "0.8.3",
|
||||
"i18next": "^26.4.2",
|
||||
"jsonc-parser": "^3.3.1",
|
||||
"node-pty": "^1.1.0",
|
||||
|
||||
Generated
+8
@@ -214,6 +214,9 @@ importers:
|
||||
electron-updater:
|
||||
specifier: ^6.8.10
|
||||
version: 6.8.10(supports-color@7.2.0)
|
||||
fflate:
|
||||
specifier: 0.8.3
|
||||
version: 0.8.3
|
||||
i18next:
|
||||
specifier: ^26.4.2
|
||||
version: 26.4.2(typescript@7.0.2)
|
||||
@@ -5120,6 +5123,9 @@ packages:
|
||||
picomatch:
|
||||
optional: true
|
||||
|
||||
fflate@0.8.3:
|
||||
resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==}
|
||||
|
||||
figures@6.1.0:
|
||||
resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -12149,6 +12155,8 @@ snapshots:
|
||||
optionalDependencies:
|
||||
picomatch: 4.0.7
|
||||
|
||||
fflate@0.8.3: {}
|
||||
|
||||
figures@6.1.0:
|
||||
dependencies:
|
||||
is-unicode-supported: 2.1.0
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { strToU8, zipSync } from 'fflate'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import type * as ReleaseModule from './managed-runtime-release'
|
||||
import { reclaimRuntimeReleases, RUNTIME_RELEASE_RECEIPT } from './managed-runtime-release'
|
||||
import { ManagedRuntimeInstallation } from './managed-runtime-installation'
|
||||
|
||||
vi.mock('./managed-runtime-release', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof ReleaseModule>()
|
||||
return { ...actual, reclaimRuntimeReleases: vi.fn(actual.reclaimRuntimeReleases) }
|
||||
})
|
||||
|
||||
const sha256 = (bytes: string | Uint8Array) => createHash('sha256').update(bytes).digest('hex')
|
||||
const ARCHIVE = zipSync({ server: strToU8('server bytes') })
|
||||
const roots: string[] = []
|
||||
afterEach(() => Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true }))))
|
||||
|
||||
async function rig(bodies: ('hang' | 'whole')[]) {
|
||||
const dataDirectory = await mkdtemp(join(tmpdir(), 'orca-runtime-install-'))
|
||||
roots.push(dataDirectory)
|
||||
const root = join(dataDirectory, 'agent-runtimes', 'agent')
|
||||
const pin = {
|
||||
version: '1.0',
|
||||
url: 'https://example.test/agent-1.0.zip',
|
||||
archiveSha256: sha256(ARCHIVE),
|
||||
archiveBytes: ARCHIVE.length,
|
||||
members: [{ path: 'server', sha256: sha256('server bytes'), bytes: 12 }]
|
||||
}
|
||||
const runtime = { label: 'Agent', root: () => root, pin: () => pin }
|
||||
const fetcher = async () => {
|
||||
const body = bodies.shift()
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(ARCHIVE.slice(0, 5))
|
||||
if (body === 'whole') {
|
||||
controller.enqueue(ARCHIVE.slice(5))
|
||||
controller.close()
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
const host = () =>
|
||||
new ManagedRuntimeInstallation('agent', runtime, {
|
||||
dataDirectory,
|
||||
findExternal: async () => false,
|
||||
fetcher
|
||||
})
|
||||
const phase = async (installation: ManagedRuntimeInstallation) =>
|
||||
(await installation.status()).operation?.phase
|
||||
return { root, host, phase }
|
||||
}
|
||||
|
||||
it('installs only when asked, with progress, cancel, retry, and a status read from disk', async () => {
|
||||
const { root, host, phase } = await rig(['hang', 'whole'])
|
||||
const installation = host()
|
||||
expect(await installation.status()).toMatchObject({ installedVersion: null, source: null })
|
||||
const started = installation.install()
|
||||
await vi.waitFor(async () =>
|
||||
expect((await installation.status()).operation?.downloadedBytes).toBe(5)
|
||||
)
|
||||
expect(installation.install().id).toBe(started.id)
|
||||
expect(await installation.cancel(started.id)).toMatchObject({ phase: 'cancelled' })
|
||||
expect(await readdir(root)).toEqual([])
|
||||
|
||||
expect(installation.install().id).not.toBe(started.id)
|
||||
await vi.waitFor(async () => expect(await phase(installation)).toBe('succeeded'))
|
||||
expect(await host().status()).toMatchObject({
|
||||
installedVersion: '1.0',
|
||||
installedCurrent: true,
|
||||
source: 'managed',
|
||||
operation: null
|
||||
})
|
||||
expect(await readdir(root)).toEqual(['1.0'])
|
||||
|
||||
// Whole files without a receipt looped as damaged; Install now rewrites it, with no download.
|
||||
await rm(join(root, '1.0', RUNTIME_RELEASE_RECEIPT))
|
||||
expect(await installation.status()).toMatchObject({ damaged: true })
|
||||
const rewrite = installation.install()
|
||||
await vi.waitFor(async () =>
|
||||
expect((await installation.status()).operation).toMatchObject({
|
||||
id: rewrite.id,
|
||||
phase: 'succeeded'
|
||||
})
|
||||
)
|
||||
expect(await installation.status()).toMatchObject({ installedCurrent: true })
|
||||
})
|
||||
|
||||
it('publishes an update only once the old releases were reclaimed', async () => {
|
||||
const { root, host, phase } = await rig(['whole'])
|
||||
for (const version of ['0.8', '0.9']) {
|
||||
await mkdir(join(root, version), { recursive: true })
|
||||
await writeFile(join(root, version, 'server'), 'old server', { mode: 0o755 })
|
||||
const members = [{ path: 'server', bytes: 10 }]
|
||||
const receipt = { releaseId: sha256(version), version, members }
|
||||
await writeFile(
|
||||
join(root, version, RUNTIME_RELEASE_RECEIPT),
|
||||
JSON.stringify({ ...receipt, platform: process.platform, arch: process.arch })
|
||||
)
|
||||
}
|
||||
let resume = (): void => {}
|
||||
const reclaim = vi.mocked(reclaimRuntimeReleases)
|
||||
const paused = new Promise<void>((resolve) => (resume = resolve))
|
||||
const { reclaimRuntimeReleases: reclaimNow } = await vi.importActual<typeof ReleaseModule>(
|
||||
'./managed-runtime-release'
|
||||
)
|
||||
reclaim.mockImplementationOnce(async (...args) => paused.then(() => reclaimNow(...args)))
|
||||
const installation = host()
|
||||
installation.install()
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
expect((await installation.status()).operation).toMatchObject({ downloadedBytes: 0 })
|
||||
resume()
|
||||
await vi.waitFor(async () => expect(await phase(installation)).toBe('succeeded'))
|
||||
expect((await readdir(root)).sort()).toEqual(['0.9', '1.0'])
|
||||
})
|
||||
|
||||
it('repairs a damaged copy when Install is clicked again', async () => {
|
||||
const { root, host, phase } = await rig(['whole', 'whole'])
|
||||
const installation = host()
|
||||
installation.install()
|
||||
await vi.waitFor(async () => expect(await phase(installation)).toBe('succeeded'))
|
||||
await writeFile(join(root, '1.0', 'server'), 'cut')
|
||||
expect(await installation.status()).toMatchObject({ installedVersion: null, damaged: true })
|
||||
|
||||
installation.install()
|
||||
await vi.waitFor(async () => expect(await phase(installation)).toBe('succeeded'))
|
||||
const repaired = await installation.status()
|
||||
expect(repaired).toMatchObject({ installedVersion: '1.0', installedCurrent: true })
|
||||
expect(repaired.damaged).toBeUndefined()
|
||||
expect(await readdir(root)).toEqual(['1.0'])
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
// Installs a managed runtime when the person asks, on the host that runs its chats. One instance per
|
||||
// runtime owns every change to its releases: it reclaims old ones once, first, then installs one at a
|
||||
// time. Publishing the release by rename is the commit point; a cancel after it changes nothing.
|
||||
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import type {
|
||||
ManagedRuntimeInstallState,
|
||||
ManagedRuntimeOperation
|
||||
} from '../../shared/managed-runtime-install'
|
||||
import type { MainHttpClient } from '../network/http-client'
|
||||
import { RevisionedState } from '../runtime/revisioned-state'
|
||||
import {
|
||||
reclaimRuntimeReleases,
|
||||
resolveInstalledRuntimeRelease,
|
||||
type ManagedRuntime,
|
||||
type ManagedRuntimePin
|
||||
} from './managed-runtime-release'
|
||||
import { installFailureText, installRuntimeRelease } from './managed-runtime-release-install'
|
||||
|
||||
const STATUS_WAIT_MS = 25_000
|
||||
const DEADLINE_MS = 45 * 60_000
|
||||
|
||||
export type ManagedRuntimeInstallationDeps = {
|
||||
dataDirectory: string
|
||||
/** Whether a start would run a copy found outside Orca's releases (PATH, install folders). */
|
||||
findExternal: () => Promise<boolean>
|
||||
fetcher?: MainHttpClient['fetch']
|
||||
}
|
||||
|
||||
type Run = { id: string; controller: AbortController; committed: boolean }
|
||||
type InstalledFacts = Omit<ManagedRuntimeInstallState, 'agent' | 'revision' | 'operation'>
|
||||
|
||||
export class ManagedRuntimeInstallation {
|
||||
private readonly state = new RevisionedState<ManagedRuntimeOperation | null>(null)
|
||||
private readonly root: string
|
||||
private running: Run | null = null
|
||||
private mutation: Promise<void>
|
||||
private facts: { phase: string; read: Promise<InstalledFacts> } | null = null
|
||||
|
||||
constructor(
|
||||
private readonly agent: string,
|
||||
private readonly runtime: ManagedRuntime,
|
||||
private readonly deps: ManagedRuntimeInstallationDeps
|
||||
) {
|
||||
this.root = runtime.root(deps.dataDirectory)
|
||||
this.mutation = this.reclaim()
|
||||
}
|
||||
|
||||
/** At once without `shown`, else once the revision passes it or after a bounded wait. Disk and
|
||||
* PATH are reread on a first read and when the phase changes, never for byte progress alone. */
|
||||
async status(shown?: number, signal?: AbortSignal): Promise<ManagedRuntimeInstallState> {
|
||||
await this.state.waitForChange(shown, STATUS_WAIT_MS, signal)
|
||||
const { value: operation, revision } = this.state.get()
|
||||
const phase = `${operation?.id}:${operation?.phase}`
|
||||
let facts = this.facts
|
||||
if (shown === undefined || facts?.phase !== phase) {
|
||||
const read = this.readInstalledFacts()
|
||||
facts = { phase, read }
|
||||
this.facts = facts
|
||||
// A failed read is retried by the next status, not replayed.
|
||||
read.catch(() => {
|
||||
this.facts = this.facts?.read === read ? null : this.facts
|
||||
})
|
||||
}
|
||||
return { agent: this.agent, revision, ...(await facts.read), operation }
|
||||
}
|
||||
|
||||
private async readInstalledFacts(): Promise<InstalledFacts> {
|
||||
const pin = this.runtime.pin()
|
||||
const { release, broken, unreadable } = resolveInstalledRuntimeRelease(this.root, pin)
|
||||
// Unknown is not absent: a failed read keeps each client's last answer.
|
||||
if (unreadable && !release?.current) {
|
||||
throw new Error(`Couldn't read the installed ${this.runtime.label} files on this host.`)
|
||||
}
|
||||
const source = (await this.deps.findExternal()) ? 'external' : release ? 'managed' : null
|
||||
return {
|
||||
available: pin !== null,
|
||||
version: pin?.version ?? null,
|
||||
installedVersion: release?.version ?? null,
|
||||
installedCurrent: release?.current ?? false,
|
||||
...(pin && !release?.current && broken.includes(pin.version) && { damaged: true }),
|
||||
source
|
||||
}
|
||||
}
|
||||
|
||||
/** Joins the running install; one being cancelled finishes its cleanup before the next starts. */
|
||||
install(): ManagedRuntimeOperation {
|
||||
const { label } = this.runtime
|
||||
const pin = this.runtime.pin()
|
||||
if (!pin) {
|
||||
throw new Error(`${label} is not available on ${process.platform} ${process.arch}.`)
|
||||
}
|
||||
const current = this.state.get().value
|
||||
const { running } = this
|
||||
if (current && current.id === running?.id && !running.controller.signal.aborted) {
|
||||
return current
|
||||
}
|
||||
const run: Run = { id: randomUUID(), controller: new AbortController(), committed: false }
|
||||
const operation: ManagedRuntimeOperation = {
|
||||
id: run.id,
|
||||
kind: 'install',
|
||||
phase: 'downloading',
|
||||
downloadedBytes: 0,
|
||||
totalBytes: pin.archiveBytes
|
||||
}
|
||||
this.running = run
|
||||
this.state.set(operation)
|
||||
this.mutation = this.mutation.then(() => this.runInstall(pin, run))
|
||||
return operation
|
||||
}
|
||||
|
||||
/** Resolves with the outcome once the work has unwound: `cancelled`, or `succeeded` when it had
|
||||
* already committed. No install queues behind a running one, so `mutation` ends with it. */
|
||||
async cancel(operationId: string): Promise<ManagedRuntimeOperation | null> {
|
||||
if (this.state.get().value?.id !== operationId) {
|
||||
throw new Error(`That ${this.runtime.label} installation is no longer running.`)
|
||||
}
|
||||
const run = this.running
|
||||
if (run?.id === operationId && !run.committed) {
|
||||
run.controller.abort()
|
||||
}
|
||||
await this.mutation
|
||||
return this.state.get().value
|
||||
}
|
||||
|
||||
private publish(run: Run, patch: Partial<ManagedRuntimeOperation>): void {
|
||||
const operation = this.state.get().value
|
||||
// Once aborted, only the outcome moves it: progress from the unwinding work is stale.
|
||||
const settles =
|
||||
patch.phase === 'succeeded' || patch.phase === 'cancelled' || patch.phase === 'failed'
|
||||
if (operation?.id === run.id && (settles || !run.controller.signal.aborted)) {
|
||||
this.state.set({ ...operation, ...patch })
|
||||
}
|
||||
}
|
||||
|
||||
private async runInstall(pin: ManagedRuntimePin, run: Run): Promise<void> {
|
||||
const deadline = AbortSignal.timeout(DEADLINE_MS)
|
||||
try {
|
||||
await installRuntimeRelease({
|
||||
agent: this.agent,
|
||||
runtime: this.runtime,
|
||||
root: this.root,
|
||||
pin,
|
||||
fetcher: this.deps.fetcher,
|
||||
signal: AbortSignal.any([run.controller.signal, deadline]),
|
||||
publish: (patch) => this.publish(run, patch),
|
||||
commit: () => {
|
||||
run.committed = true
|
||||
}
|
||||
})
|
||||
this.publish(run, { phase: 'succeeded' })
|
||||
} catch (error) {
|
||||
if (run.committed) {
|
||||
console.warn(`[managed-runtime] ${this.agent} installed; staging not removed:`, error)
|
||||
this.publish(run, { phase: 'succeeded' })
|
||||
} else if (run.controller.signal.aborted) {
|
||||
this.publish(run, { phase: 'cancelled' })
|
||||
} else {
|
||||
this.publish(run, {
|
||||
phase: 'failed',
|
||||
error: installFailureText(this.runtime.label, error, deadline)
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
this.running = this.running === run ? null : this.running
|
||||
}
|
||||
}
|
||||
|
||||
private async reclaim(): Promise<void> {
|
||||
const { release, unreadable } = resolveInstalledRuntimeRelease(this.root, this.runtime.pin())
|
||||
if (unreadable) {
|
||||
// Which release a launch runs is unknown, so nothing is provably stale.
|
||||
console.warn(`[managed-runtime] ${this.agent} releases unreadable; not reclaiming this run`)
|
||||
return
|
||||
}
|
||||
const failed = await reclaimRuntimeReleases(this.root, release?.version ?? null).catch(
|
||||
(error: unknown) => [String(error)]
|
||||
)
|
||||
if (failed.length > 0) {
|
||||
console.warn(`[managed-runtime] old ${this.agent} releases not removed:`, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Puts one pinned release of a managed runtime on disk: reuses a whole copy, else downloads, unpacks,
|
||||
// verifies and starts it in a staging folder, then publishes it by rename. The caller owns the
|
||||
// lifecycle (one install at a time, cancel, status); `commit` marks the point a cancel can't undo.
|
||||
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { chmod, mkdir, rename, rm, statfs } from 'node:fs/promises'
|
||||
import { basename, join } from 'node:path'
|
||||
import type { ManagedRuntimeOperation } from '../../shared/managed-runtime-install'
|
||||
import { getMainHttpClient, type MainHttpClient } from '../network/http-client'
|
||||
import {
|
||||
resolveInstalledRuntimeRelease,
|
||||
RUNTIME_RELEASE_RECEIPT,
|
||||
writeRuntimeReleaseReceipt,
|
||||
type ManagedRuntime,
|
||||
type ManagedRuntimePin
|
||||
} from './managed-runtime-release'
|
||||
import { verifyFileSha256 } from './pinned-runtime-materializer'
|
||||
import { extractPinnedZipMembers } from './pinned-zip-members'
|
||||
import {
|
||||
downloadFailureMessage,
|
||||
downloadVerifiedArchive,
|
||||
RUNTIME_ARCHIVE_RETRY_DELAYS_MS
|
||||
} from './runtime-archive-download'
|
||||
|
||||
const FREE_SPACE_MARGIN_BYTES = 256 * 1024 * 1024
|
||||
const PROGRESS_INTERVAL_MS = 250
|
||||
|
||||
export type RuntimeReleaseInstall = {
|
||||
/** Names the agent in log lines. */
|
||||
agent: string
|
||||
runtime: Pick<ManagedRuntime, 'label' | 'validate'>
|
||||
/** The folder that holds only this runtime's releases. */
|
||||
root: string
|
||||
pin: ManagedRuntimePin
|
||||
fetcher?: MainHttpClient['fetch']
|
||||
signal: AbortSignal
|
||||
publish: (patch: Partial<ManagedRuntimeOperation>) => void
|
||||
/** Called right after the release is in place, before any cleanup that may still fail. */
|
||||
commit: () => void
|
||||
}
|
||||
|
||||
export async function installRuntimeRelease(input: RuntimeReleaseInstall): Promise<void> {
|
||||
const { agent, root, pin, signal, publish } = input
|
||||
const { label } = input.runtime
|
||||
const folder = join(root, pin.version)
|
||||
await mkdir(root, { recursive: true })
|
||||
if (existsSync(folder)) {
|
||||
publish({ phase: 'verifying' })
|
||||
const whole = await verifyMembers(folder, pin, label, signal).then(
|
||||
() => true,
|
||||
(error: unknown) => {
|
||||
signal.throwIfAborted()
|
||||
console.warn(`[managed-runtime] ${agent} copy is damaged; repairing:`, error)
|
||||
return false
|
||||
}
|
||||
)
|
||||
if (whole) {
|
||||
await validate(input, folder)
|
||||
signal.throwIfAborted()
|
||||
if (!resolveInstalledRuntimeRelease(root, pin).release?.current) {
|
||||
// Whole members without a valid receipt read as damaged; rewriting it ends that loop.
|
||||
await rm(join(folder, RUNTIME_RELEASE_RECEIPT), { force: true })
|
||||
await writeRuntimeReleaseReceipt(folder, pin)
|
||||
}
|
||||
input.commit()
|
||||
return
|
||||
}
|
||||
publish({ phase: 'downloading' })
|
||||
}
|
||||
// Bookkeeping only: a host that can't report free space still installs.
|
||||
const space = await statfs(root).catch((error: unknown) => {
|
||||
console.warn(`[managed-runtime] ${agent} free space unknown; not checked:`, error)
|
||||
})
|
||||
const needed = pin.members.reduce((sum, m) => sum + m.bytes, pin.archiveBytes)
|
||||
if (space && space.bavail * space.bsize < needed + FREE_SPACE_MARGIN_BYTES) {
|
||||
const megabytes = Math.ceil((needed + FREE_SPACE_MARGIN_BYTES) / 1024 ** 2)
|
||||
throw new Error(`${label} needs ${megabytes} MB of free disk space to install.`)
|
||||
}
|
||||
const staging = join(root, `.download-${process.pid}-${randomUUID()}`)
|
||||
const release = join(staging, 'release')
|
||||
try {
|
||||
await mkdir(release, { recursive: true })
|
||||
const archive = join(staging, basename(new URL(pin.url).pathname))
|
||||
let shownAt = 0
|
||||
await downloadVerifiedArchive(
|
||||
{
|
||||
label,
|
||||
url: pin.url,
|
||||
archiveSha256: pin.archiveSha256,
|
||||
maxArchiveBytes: pin.archiveBytes
|
||||
},
|
||||
archive,
|
||||
input.fetcher ?? getMainHttpClient().fetch,
|
||||
signal,
|
||||
RUNTIME_ARCHIVE_RETRY_DELAYS_MS,
|
||||
(bytes) => {
|
||||
if (Date.now() - shownAt >= PROGRESS_INTERVAL_MS) {
|
||||
shownAt = Date.now()
|
||||
publish({ downloadedBytes: bytes })
|
||||
}
|
||||
}
|
||||
).catch((error: unknown) => {
|
||||
if (signal.aborted) {
|
||||
throw error
|
||||
}
|
||||
console.warn(`[managed-runtime] ${agent} download failed:`, error)
|
||||
throw new Error(downloadFailureMessage(label, error), { cause: error })
|
||||
})
|
||||
publish({ phase: 'extracting', downloadedBytes: pin.archiveBytes })
|
||||
await extractPinnedZipMembers(archive, release, pin.members, label, signal)
|
||||
publish({ phase: 'verifying' })
|
||||
await verifyMembers(release, pin, label, signal)
|
||||
await validate(input, release)
|
||||
await writeRuntimeReleaseReceipt(release, pin)
|
||||
signal.throwIfAborted()
|
||||
// A damaged copy moves into staging, which is deleted below or reclaimed on the next start.
|
||||
if (existsSync(folder)) {
|
||||
await rename(folder, join(staging, 'damaged'))
|
||||
}
|
||||
await rename(release, folder)
|
||||
input.commit()
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** The sentence a failed install shows; `deadline` is the install's overall time limit. */
|
||||
export function installFailureText(label: string, error: unknown, deadline: AbortSignal): string {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
if (deadline.aborted) {
|
||||
return `Installing ${label} took longer than 45 minutes. Try again.`
|
||||
}
|
||||
return isChecksumMismatch(error)
|
||||
? `The ${label} download did not match its pinned checksum, so nothing was installed.`
|
||||
: message
|
||||
}
|
||||
|
||||
function isChecksumMismatch(error: unknown): boolean {
|
||||
return error instanceof Error && /checksum mismatch/.test(error.message)
|
||||
}
|
||||
|
||||
async function validate(input: RuntimeReleaseInstall, folder: string): Promise<void> {
|
||||
const { runtime, pin, signal } = input
|
||||
await runtime.validate?.(folder, pin, signal).catch((error: unknown) => {
|
||||
signal.throwIfAborted()
|
||||
console.warn(`[managed-runtime] ${input.agent} did not start from ${folder}:`, error)
|
||||
throw new Error(
|
||||
`${runtime.label} couldn't start on this host, so it wasn't installed. This host may not support it.`,
|
||||
{ cause: error }
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async function verifyMembers(
|
||||
root: string,
|
||||
pin: ManagedRuntimePin,
|
||||
label: string,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
for (const member of pin.members) {
|
||||
const path = join(root, ...member.path.split('/'))
|
||||
await verifyFileSha256(path, member.sha256, `${label} ${member.path}`, signal)
|
||||
if (process.platform !== 'win32') {
|
||||
await chmod(path, 0o755)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,18 +3,19 @@
|
||||
// cheap enough for every launch: a full hash is only ever taken by an install.
|
||||
|
||||
import { lstatSync, readFileSync, readdirSync, realpathSync, statSync } from 'node:fs'
|
||||
import { readdir, rm } from 'node:fs/promises'
|
||||
import { readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { z } from 'zod'
|
||||
import { isDefinitiveAbsence } from '../../shared/definitive-filesystem-absence'
|
||||
|
||||
export type ManagedRuntimePin = {
|
||||
version: string
|
||||
/** A zip holding exactly the members. */
|
||||
url: string
|
||||
archiveSha256: string
|
||||
/** The decoded archive's size. */
|
||||
archiveBytes: number
|
||||
/** Archive-relative path, sha256 and size of every executable the release must hold. */
|
||||
/** Name at the archive's top level, sha256 and size of every executable the release holds. */
|
||||
members: readonly { path: string; sha256: string; bytes: number }[]
|
||||
}
|
||||
|
||||
@@ -26,6 +27,9 @@ export type ManagedRuntime = {
|
||||
root(dataDirectory: string): string
|
||||
/** This host's pinned release; null where none is published. */
|
||||
pin(): ManagedRuntimePin | null
|
||||
/** Starts the server in `folder` once, without the person's sign-in, and rejects unless it
|
||||
* answers as the pinned release; Install publishes and reuses only a copy that passes. */
|
||||
validate?(folder: string, pin: ManagedRuntimePin, signal: AbortSignal): Promise<void>
|
||||
}
|
||||
|
||||
export const RUNTIME_RELEASE_RECEIPT = '.install-complete.json'
|
||||
@@ -108,8 +112,24 @@ export function copyBesidePinnedMembers(
|
||||
}
|
||||
}
|
||||
|
||||
/** Removes everything under `root` but the release named `keep`: called only while the caller
|
||||
* owns every install and launch of this runtime, so nothing else runs from or publishes into it.
|
||||
/** Written into a verified release before it is renamed into place: the release is whole once the
|
||||
* rename lands, so a folder with this receipt was never published partly. */
|
||||
export async function writeRuntimeReleaseReceipt(
|
||||
folder: string,
|
||||
pin: ManagedRuntimePin
|
||||
): Promise<void> {
|
||||
const receipt: z.infer<typeof runtimeReleaseReceiptSchema> = {
|
||||
releaseId: pin.archiveSha256,
|
||||
version: pin.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
members: pin.members.map(({ path, bytes }) => ({ path, bytes }))
|
||||
}
|
||||
await writeFile(join(folder, RUNTIME_RELEASE_RECEIPT), JSON.stringify(receipt), { flag: 'wx' })
|
||||
}
|
||||
|
||||
/** Removes everything under `root` but the release named `keep`. Safe only when `keep` is what a
|
||||
* launch resolves and the caller is the only writer, so no chat runs from what it removes.
|
||||
* Answers the entries it could not remove. */
|
||||
export async function reclaimRuntimeReleases(root: string, keep: string | null): Promise<string[]> {
|
||||
const names = await readdir(root).catch((error: NodeJS.ErrnoException) =>
|
||||
|
||||
@@ -24,6 +24,8 @@ export type PinnedRuntimeArchive = {
|
||||
label: string
|
||||
url: string
|
||||
archiveSha256: string
|
||||
/** The download's size limit; 200 MiB by default. */
|
||||
maxArchiveBytes?: number
|
||||
}
|
||||
|
||||
export type PinnedRuntimeExecutable = PinnedRuntimeArchive & {
|
||||
@@ -207,18 +209,20 @@ export function materializeCachedNodeRuntime(
|
||||
export async function verifyFileSha256(
|
||||
path: string,
|
||||
expected: string,
|
||||
label: string
|
||||
label: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const actual = await fileSha256(path)
|
||||
const actual = await fileSha256(path, signal)
|
||||
signal?.throwIfAborted()
|
||||
if (actual !== expected) {
|
||||
throw new Error(`${label} checksum mismatch: expected ${expected}, got ${actual ?? 'missing'}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function fileSha256(path: string): Promise<string | null> {
|
||||
export async function fileSha256(path: string, signal?: AbortSignal): Promise<string | null> {
|
||||
try {
|
||||
const hash = createHash('sha256')
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
for await (const chunk of createReadStream(path, { signal })) {
|
||||
hash.update(chunk)
|
||||
}
|
||||
return hash.digest('hex')
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
// Unpacks a hash-checked runtime zip in-process, so installing never depends on the host having
|
||||
// `unzip`. Only the pinned members land, at their pinned sizes; any other entry fails.
|
||||
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { open, type FileHandle } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { Unzip, UnzipInflate } from 'fflate'
|
||||
import { writeAll } from './runtime-archive-download'
|
||||
|
||||
export async function extractPinnedZipMembers(
|
||||
archivePath: string,
|
||||
directory: string,
|
||||
members: readonly { path: string; bytes: number }[],
|
||||
label: string,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
const expected = new Map(members.map((member) => [member.path, member.bytes]))
|
||||
const written = new Map<string, number>()
|
||||
const pending: { name: string; data: Uint8Array }[] = []
|
||||
const failures: string[] = []
|
||||
const unzip = new Unzip((file) => {
|
||||
const bytes = expected.get(file.name)
|
||||
if (bytes === undefined || written.has(file.name)) {
|
||||
failures.push(`has an unexpected entry ${JSON.stringify(file.name)}`)
|
||||
return
|
||||
}
|
||||
written.set(file.name, 0)
|
||||
file.ondata = (error, data) => {
|
||||
if (error) {
|
||||
failures.push(`has an unreadable ${file.name}: ${error.message}`)
|
||||
return
|
||||
}
|
||||
const total = (written.get(file.name) ?? 0) + data.byteLength
|
||||
if (total > bytes) {
|
||||
failures.push(`has a ${file.name} larger than expected`)
|
||||
return
|
||||
}
|
||||
written.set(file.name, total)
|
||||
pending.push({ name: file.name, data })
|
||||
}
|
||||
file.start()
|
||||
})
|
||||
unzip.register(UnzipInflate)
|
||||
const handles = new Map<string, FileHandle>()
|
||||
// Each read's output is written before the next read, which bounds memory.
|
||||
const push = async (chunk: Uint8Array, final: boolean): Promise<void> => {
|
||||
unzip.push(chunk, final)
|
||||
for (const { name, data } of pending.splice(0)) {
|
||||
const handle = handles.get(name) ?? (await open(join(directory, name), 'wx', 0o700))
|
||||
handles.set(name, handle)
|
||||
await writeAll(handle, data)
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
throw new Error(`The ${label} archive ${failures[0]}.`)
|
||||
}
|
||||
}
|
||||
try {
|
||||
for await (const chunk of createReadStream(archivePath, { signal })) {
|
||||
await push(chunk, false)
|
||||
}
|
||||
await push(new Uint8Array(0), true)
|
||||
} finally {
|
||||
await Promise.all([...handles.values()].map((handle) => handle.close()))
|
||||
}
|
||||
if (members.some((member) => written.get(member.path) !== member.bytes)) {
|
||||
throw new Error(`The ${label} archive is missing a file or has one cut short.`)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { PinnedRuntimeArchive } from './pinned-runtime-materializer'
|
||||
import { downloadVerifiedArchive } from './runtime-archive-download'
|
||||
import { downloadFailureMessage, downloadVerifiedArchive } from './runtime-archive-download'
|
||||
|
||||
const BODY = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])
|
||||
const NO_DELAY = [0, 0, 0]
|
||||
@@ -111,6 +111,21 @@ describe('downloading a pinned runtime archive over a flaky network', () => {
|
||||
expect(signals[0]?.aborted).toBe(true)
|
||||
})
|
||||
|
||||
// QA: a refused proxy was blamed on the release instead of the network.
|
||||
it('blames the release only for a proven mismatch', async () => {
|
||||
const failure = (response: () => Promise<Response>) =>
|
||||
downloadVerifiedArchive(archive(), destination, vi.fn<typeof fetch>(response), undefined, [])
|
||||
.then(() => '', (error: unknown) => downloadFailureMessage('Node', error))
|
||||
expect(
|
||||
await failure(async () => {
|
||||
throw new Error('net::ERR_PROXY_CONNECTION_FAILED')
|
||||
})
|
||||
).toMatch(/^Couldn't download Node\./)
|
||||
expect(await failure(async () => ok(new Uint8Array([9, 9])))).toMatch(
|
||||
/didn't match the release/
|
||||
)
|
||||
})
|
||||
|
||||
it('stops waiting to retry once the caller cancels', async () => {
|
||||
const controller = new AbortController()
|
||||
const fetcher = vi.fn<typeof fetch>(async () => {
|
||||
|
||||
@@ -59,17 +59,31 @@ export async function extractRuntimeArchive(
|
||||
}
|
||||
}
|
||||
|
||||
/** The served file isn't the pinned release (missing, oversized or a different checksum), so no
|
||||
* retry or network fix helps; only a newer Orca does. */
|
||||
export class ReleaseMismatchError extends Error {}
|
||||
|
||||
/** What the person can do about a failed download. Only a proven mismatch blames the release;
|
||||
* every other failure (proxy, DNS, TLS, a dropped connection) is one the network can fix. */
|
||||
export function downloadFailureMessage(label: string, error: unknown): string {
|
||||
return error instanceof ReleaseMismatchError
|
||||
? `The ${label} download didn't match the release Orca expects, so nothing was installed. Updating Orca may fix this.`
|
||||
: `Couldn't download ${label}. Check your network connection or proxy, then try again.`
|
||||
}
|
||||
|
||||
/** Retries transient network failures; a checksum mismatch or a cancel is final. */
|
||||
export async function downloadVerifiedArchive(
|
||||
archive: PinnedRuntimeArchive,
|
||||
destination: string,
|
||||
fetcher: MainHttpClient['fetch'],
|
||||
signal?: AbortSignal,
|
||||
retryDelaysMs: readonly number[] = RUNTIME_ARCHIVE_RETRY_DELAYS_MS
|
||||
retryDelaysMs: readonly number[] = RUNTIME_ARCHIVE_RETRY_DELAYS_MS,
|
||||
/** The bytes this attempt has read so far; a retry counts from zero again. */
|
||||
onBytes?: (bytes: number) => void
|
||||
): Promise<void> {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
await downloadVerifiedArchiveOnce(archive, destination, fetcher, signal)
|
||||
await downloadVerifiedArchiveOnce(archive, destination, fetcher, signal, onBytes)
|
||||
return
|
||||
} catch (error) {
|
||||
const delay = retryDelaysMs[attempt]
|
||||
@@ -91,9 +105,11 @@ async function downloadVerifiedArchiveOnce(
|
||||
archive: PinnedRuntimeArchive,
|
||||
destination: string,
|
||||
fetcher: MainHttpClient['fetch'],
|
||||
signal?: AbortSignal
|
||||
signal?: AbortSignal,
|
||||
onBytes?: (bytes: number) => void
|
||||
): Promise<void> {
|
||||
const { label, url, archiveSha256: expectedSha256 } = archive
|
||||
const maxBytes = archive.maxArchiveBytes ?? MAX_RUNTIME_ARCHIVE_BYTES
|
||||
const stall = new AbortController()
|
||||
const downloadSignal = signal ? AbortSignal.any([signal, stall.signal]) : stall.signal
|
||||
const stalled: HttpStatusError = Object.assign(new Error(`${label} download stalled`), {
|
||||
@@ -101,18 +117,26 @@ async function downloadVerifiedArchiveOnce(
|
||||
})
|
||||
const stallTimer = setTimeout(() => stall.abort(stalled), 120_000)
|
||||
try {
|
||||
const response = await fetcher(url, { redirect: 'follow', signal: downloadSignal })
|
||||
// no-store: the verified file is the only copy, so Remove frees the space and a retry refetches.
|
||||
const response = await fetcher(url, {
|
||||
redirect: 'follow',
|
||||
cache: 'no-store',
|
||||
signal: downloadSignal
|
||||
})
|
||||
if (!response.ok || !response.body) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw Object.assign(
|
||||
new Error(`${label} download failed: ${response.status} ${response.statusText}`),
|
||||
{ httpStatusCode: response.status }
|
||||
)
|
||||
const text = `${label} download failed: ${response.status} ${response.statusText}`
|
||||
const missing = response.status === 404 || response.status === 410
|
||||
throw Object.assign(missing ? new ReleaseMismatchError(text) : new Error(text), {
|
||||
httpStatusCode: response.status
|
||||
})
|
||||
}
|
||||
// A compressed body declares its encoded size; the decoded bytes are still capped and hashed.
|
||||
const encoding = response.headers.get('content-encoding')?.trim().toLowerCase() || 'identity'
|
||||
const declaredLength = Number(response.headers.get('content-length'))
|
||||
if (Number.isFinite(declaredLength) && declaredLength > MAX_RUNTIME_ARCHIVE_BYTES) {
|
||||
if (encoding === 'identity' && Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||
await response.body.cancel().catch(() => undefined)
|
||||
throw new Error(`${label} download exceeded the archive size limit`)
|
||||
throw new ReleaseMismatchError(`${label} download exceeded the archive size limit`)
|
||||
}
|
||||
const handle = await open(destination, 'wx', 0o600).catch(async (error) => {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
@@ -132,8 +156,9 @@ async function downloadVerifiedArchiveOnce(
|
||||
stallTimer.refresh()
|
||||
}
|
||||
total += chunk.value.byteLength
|
||||
if (total > MAX_RUNTIME_ARCHIVE_BYTES) {
|
||||
throw new Error(`${label} download exceeded the archive size limit`)
|
||||
onBytes?.(total)
|
||||
if (total > maxBytes) {
|
||||
throw new ReleaseMismatchError(`${label} download exceeded the archive size limit`)
|
||||
}
|
||||
hash.update(chunk.value)
|
||||
await writeAll(handle, chunk.value)
|
||||
@@ -146,7 +171,7 @@ async function downloadVerifiedArchiveOnce(
|
||||
}
|
||||
const actual = hash.digest('hex')
|
||||
if (actual !== expectedSha256) {
|
||||
throw new Error(
|
||||
throw new ReleaseMismatchError(
|
||||
`${label} archive checksum mismatch: expected ${expectedSha256}, got ${actual}`
|
||||
)
|
||||
}
|
||||
@@ -160,7 +185,7 @@ async function downloadVerifiedArchiveOnce(
|
||||
}
|
||||
}
|
||||
|
||||
async function writeAll(
|
||||
export async function writeAll(
|
||||
handle: Awaited<ReturnType<typeof open>>,
|
||||
bytes: Uint8Array
|
||||
): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
// A value read by revision: a reader names the revision it shows and waits, bounded, for a newer
|
||||
// one. Its owner keeps its own lifecycle policy; this only publishes and waits.
|
||||
|
||||
import { delayUnlessAborted } from '../../shared/abort-aware-delay'
|
||||
|
||||
export class RevisionedState<S> {
|
||||
private revision = 0
|
||||
private readonly changed = new Set<() => void>()
|
||||
|
||||
constructor(private value: S) {}
|
||||
|
||||
get(): { value: S; revision: number } {
|
||||
return { value: this.value, revision: this.revision }
|
||||
}
|
||||
|
||||
set(value: S): void {
|
||||
this.value = value
|
||||
this.revision += 1
|
||||
this.changed.forEach((notify) => notify())
|
||||
}
|
||||
|
||||
/** Resolves once the revision differs from `shown`, after `waitMs`, or when `signal` aborts. */
|
||||
async waitForChange(shown: number | undefined, waitMs: number, signal?: AbortSignal) {
|
||||
if (shown !== this.revision) {
|
||||
return
|
||||
}
|
||||
const changed = new AbortController()
|
||||
const notify = (): void => changed.abort()
|
||||
this.changed.add(notify)
|
||||
try {
|
||||
await delayUnlessAborted(waitMs, AbortSignal.any([changed.signal, ...(signal ? [signal] : [])]))
|
||||
} finally {
|
||||
this.changed.delete(notify)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// A runtime Orca installs for an agent only when the person asks, as its host reports it.
|
||||
|
||||
export type ManagedRuntimeOperation = {
|
||||
id: string
|
||||
kind: 'install' | 'remove'
|
||||
/** A Cancel ends in `cancelled`, or `succeeded` when the install had already committed. */
|
||||
phase:
|
||||
| 'downloading'
|
||||
| 'extracting'
|
||||
| 'verifying'
|
||||
| 'removing'
|
||||
| 'succeeded'
|
||||
| 'failed'
|
||||
| 'cancelled'
|
||||
downloadedBytes: number
|
||||
totalBytes: number | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
export type ManagedRuntimeInstallState = {
|
||||
agent: string
|
||||
/** Rises with every change the host publishes, across operations. */
|
||||
revision: number
|
||||
/** The host has a pinned release; otherwise no Install is offered. */
|
||||
available: boolean
|
||||
version: string | null
|
||||
installedVersion: string | null
|
||||
/** False with an installed version: an update is available. */
|
||||
installedCurrent: boolean
|
||||
/** The pinned release's folder is there but not whole; Install repairs it. */
|
||||
damaged?: boolean
|
||||
/** What a chat start runs now: Orca's release, a copy found outside it, or nothing. */
|
||||
source: 'managed' | 'external' | null
|
||||
/** The latest operation; an ended one stays readable until the next starts. */
|
||||
operation: ManagedRuntimeOperation | null
|
||||
}
|
||||
Reference in New Issue
Block a user