mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
* fix(orcad): yield a lost instance lock and refuse non-durable terminals under a supervisor Fixes #37605 * fix(orcad): treat an unstattable lock as unproven; scope supervised help to orcad Fixes #37605 * fix(daemon): report fresh-terminal ownership from current spawn routing Fixes #37605 * fix(daemon): narrow fresh-spawn routing check to degraded providers Fixes #37605 * test(orcad): cover lock-loss yield and fix stale test paths after rebase Fixes #37605
311 lines
11 KiB
TypeScript
311 lines
11 KiB
TypeScript
import { SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from '../../shared/single-instance-exit-code'
|
|
import { ORCAD_EXIT_TERMINALS_NOT_DURABLE } from '../../shared/serve-supervised-terminal-durability'
|
|
import { orcadStaleLockReclaimCommand } from '../../shared/orcad-stale-lock-reclaim'
|
|
import {
|
|
brakeForBuild,
|
|
clearServeSupervisorBrake,
|
|
readServeSupervisorBrake,
|
|
recordSupervisedFailure,
|
|
SERVE_GAVE_UP_ADVICE,
|
|
standbyDelayMs,
|
|
writeServeSupervisorBrake,
|
|
type ServeSupervisorBrake
|
|
} from './serve-supervisor-brake'
|
|
import {
|
|
readServeInstallEvidence,
|
|
serveBuildIdentity,
|
|
SERVE_INSTALL_HOLD_REASON_TEXT,
|
|
type ServeInstallEvidence
|
|
} from './serve-install-hold'
|
|
import {
|
|
currentServeChild,
|
|
requestSupervisedServeStop,
|
|
resetSupervisedServeStop
|
|
} from './serve-supervised-stop'
|
|
import type { ServeBundlePark } from '../../shared/serve-bundle-park'
|
|
import { createSupervisedParkWatch } from './serve-supervised-park-watch'
|
|
import type { AppImageBuildState } from './serve-appimage-update'
|
|
|
|
export const SERVE_INSTALL_HOLD_POLL_MS = 2_000
|
|
/** A hold this long is reported as stuck; it still never times out into a spawn. */
|
|
export const SERVE_INSTALL_HOLD_STUCK_MS = 30 * 60_000
|
|
/** After this delay, exit so the service manager can find the updated launcher. */
|
|
export const SERVE_EXECUTABLE_MISSING_EXIT_MS = 2 * 60_000
|
|
|
|
type StopListener = () => void
|
|
|
|
export type SupervisedServeOptions = {
|
|
userDataPath: string
|
|
executable: string
|
|
/** Part of the build identity: the app archive changes with every release. */
|
|
appRoot?: string
|
|
force: boolean
|
|
/** One full serve attempt: preflight, spawn, and the existing update-aware wait. */
|
|
runOnce: () => Promise<number>
|
|
readInstallEvidence?: (executable: string) => Promise<ServeInstallEvidence>
|
|
buildIdentity?: (executable: string) => string
|
|
readBrake?: (userDataPath: string) => ServeSupervisorBrake | null
|
|
writeBrake?: (userDataPath: string, record: ServeSupervisorBrake) => void
|
|
clearBrake?: (userDataPath: string) => void
|
|
now?: () => number
|
|
random?: () => number
|
|
setTimer?: (callback: () => void, ms: number) => () => void
|
|
onStopSignal?: (listener: StopListener) => () => void
|
|
log?: (line: string) => void
|
|
/** The desktop-install park protocol; null when this supervisor does not take part. */
|
|
park?: ServeBundlePark | null
|
|
currentChild?: typeof currentServeChild
|
|
setRepeating?: (callback: () => void, ms: number) => () => void
|
|
/** The AppImage this build was unpacked from, checked before each start; null elsewhere. */
|
|
appImageUpdate?: SupervisedAppImageUpdate | null
|
|
}
|
|
|
|
export type SupervisedAppImageUpdate = {
|
|
inspect: () => AppImageBuildState
|
|
/** Re-execs into the new build and never returns; otherwise resolves with why it could not. */
|
|
restart: (isStopped: () => boolean) => Promise<string>
|
|
}
|
|
|
|
export function getServeSuperviseValidationError(input: {
|
|
supervise: boolean
|
|
recipeJson: boolean
|
|
platform?: NodeJS.Platform
|
|
}): string | null {
|
|
if (!input.supervise) {
|
|
return null
|
|
}
|
|
if ((input.platform ?? process.platform) === 'win32') {
|
|
return '--supervise is not supported on Windows yet. Run `orca serve` under your own service wrapper instead.'
|
|
}
|
|
if (input.recipeJson) {
|
|
return '--recipe-json starts a detached server and returns; it cannot be combined with --supervise.'
|
|
}
|
|
return null
|
|
}
|
|
|
|
function defaultOnStopSignal(listener: StopListener): () => void {
|
|
const signals: NodeJS.Signals[] =
|
|
process.platform === 'linux' ? ['SIGINT', 'SIGTERM', 'SIGHUP'] : ['SIGINT', 'SIGTERM']
|
|
for (const signal of signals) {
|
|
process.on(signal, listener)
|
|
}
|
|
return () => {
|
|
for (const signal of signals) {
|
|
process.off(signal, listener)
|
|
}
|
|
}
|
|
}
|
|
|
|
function defaultSetTimer(callback: () => void, ms: number): () => void {
|
|
const timer = setTimeout(callback, ms)
|
|
return () => clearTimeout(timer)
|
|
}
|
|
|
|
function defaultSetRepeating(callback: () => void, ms: number): () => void {
|
|
const timer = setInterval(callback, ms)
|
|
return () => clearInterval(timer)
|
|
}
|
|
|
|
async function settle(run: () => Promise<number>): Promise<PromiseSettledResult<number>> {
|
|
try {
|
|
return { status: 'fulfilled', value: await run() }
|
|
} catch (reason) {
|
|
return { status: 'rejected', reason }
|
|
}
|
|
}
|
|
|
|
/** Own server restarts; exit 0 after giving up so the service manager stops retrying. */
|
|
export async function superviseServeService(options: SupervisedServeOptions): Promise<number> {
|
|
const now = options.now ?? Date.now
|
|
const random = options.random ?? Math.random
|
|
const log = options.log ?? ((line: string) => process.stderr.write(`${line}\n`))
|
|
const setTimer = options.setTimer ?? defaultSetTimer
|
|
const readEvidence = options.readInstallEvidence ?? readServeInstallEvidence
|
|
const identify =
|
|
options.buildIdentity ??
|
|
((executable: string) => serveBuildIdentity(executable, options.appRoot))
|
|
const writeBrake = options.writeBrake ?? writeServeSupervisorBrake
|
|
|
|
// Mid-install missing executables must not reset the crash brake.
|
|
let record: ServeSupervisorBrake | null = (options.readBrake ?? readServeSupervisorBrake)(
|
|
options.userDataPath
|
|
)
|
|
if (options.force) {
|
|
;(options.clearBrake ?? clearServeSupervisorBrake)(options.userDataPath)
|
|
record = null
|
|
}
|
|
const refuseGaveUp = (gaveUp: { reason: string }): number => {
|
|
log(`[serve] Not starting: ${gaveUp.reason} ${SERVE_GAVE_UP_ADVICE}`)
|
|
return 0
|
|
}
|
|
// A new AppImage deserves a retry even after the previous build hit the brake.
|
|
if (
|
|
record?.gaveUp &&
|
|
record.build === identify(options.executable) &&
|
|
(options.appImageUpdate?.inspect() ?? 'current') === 'current'
|
|
) {
|
|
return refuseGaveUp(record.gaveUp)
|
|
}
|
|
|
|
const parkWatch = options.park
|
|
? createSupervisedParkWatch(options.park, {
|
|
currentChild: options.currentChild ?? currentServeChild,
|
|
setRepeating: options.setRepeating ?? defaultSetRepeating,
|
|
now
|
|
})
|
|
: null
|
|
let stopped = false
|
|
let wake: (() => void) | null = null
|
|
const removeStopListener = (options.onStopSignal ?? defaultOnStopSignal)(() => {
|
|
stopped = true
|
|
requestSupervisedServeStop()
|
|
wake?.()
|
|
})
|
|
const sleep = (ms: number): Promise<void> =>
|
|
new Promise((resolveSleep) => {
|
|
if (stopped) {
|
|
resolveSleep()
|
|
return
|
|
}
|
|
const cancel = setTimer(() => {
|
|
wake = null
|
|
resolveSleep()
|
|
}, ms)
|
|
wake = () => {
|
|
wake = null
|
|
cancel()
|
|
resolveSleep()
|
|
}
|
|
})
|
|
|
|
const awaitClearToSpawn = async (): Promise<boolean> => {
|
|
const since = now()
|
|
while (!stopped) {
|
|
const evidence: ServeInstallEvidence = parkWatch?.idle()
|
|
? { kind: 'hold', reason: 'install-park' }
|
|
: await readEvidence(options.executable)
|
|
if (evidence.kind === 'clear') {
|
|
return !stopped
|
|
}
|
|
if (
|
|
evidence.reason === 'bundle-missing' &&
|
|
now() - since >= SERVE_EXECUTABLE_MISSING_EXIT_MS
|
|
) {
|
|
return false
|
|
}
|
|
await sleep(SERVE_INSTALL_HOLD_POLL_MS)
|
|
}
|
|
return false
|
|
}
|
|
resetSupervisedServeStop(awaitClearToSpawn, () => options.park?.holds() ?? false)
|
|
|
|
let appImageUpdate = options.appImageUpdate ?? null
|
|
try {
|
|
let standbyAttempt = 0
|
|
const hold: { current: { reason: string; since: number; reportedStuck: boolean } | null } = {
|
|
current: null
|
|
}
|
|
while (!stopped) {
|
|
const evidence: ServeInstallEvidence = parkWatch?.idle()
|
|
? { kind: 'hold', reason: 'install-park' }
|
|
: await readEvidence(options.executable)
|
|
if (stopped) {
|
|
break
|
|
}
|
|
if (evidence.kind === 'hold') {
|
|
const held = hold.current
|
|
if (held?.reason !== evidence.reason) {
|
|
hold.current = { reason: evidence.reason, since: now(), reportedStuck: false }
|
|
log(
|
|
`[serve] Waiting before starting Orca serve: ${SERVE_INSTALL_HOLD_REASON_TEXT[evidence.reason]}.`
|
|
)
|
|
} else if (
|
|
evidence.reason === 'bundle-missing' &&
|
|
now() - held.since >= SERVE_EXECUTABLE_MISSING_EXIT_MS
|
|
) {
|
|
// Let the service manager find the updated app through its stable launcher.
|
|
log(
|
|
'[serve] The Orca executable this service started from is gone; exiting so the service manager starts the installed Orca.'
|
|
)
|
|
return 1
|
|
} else if (!held.reportedStuck && now() - held.since >= SERVE_INSTALL_HOLD_STUCK_MS) {
|
|
held.reportedStuck = true
|
|
log(
|
|
`[serve] Still waiting after ${SERVE_INSTALL_HOLD_STUCK_MS / 60_000} minutes: ${SERVE_INSTALL_HOLD_REASON_TEXT[evidence.reason]}. Stop the service and start it again once the install is done.`
|
|
)
|
|
}
|
|
await sleep(SERVE_INSTALL_HOLD_POLL_MS)
|
|
continue
|
|
}
|
|
hold.current = null
|
|
const appImage = appImageUpdate?.inspect() ?? 'current'
|
|
if (appImage === 'settling') {
|
|
// Wait until the new AppImage finishes copying before choosing a build.
|
|
await sleep(SERVE_INSTALL_HOLD_POLL_MS)
|
|
continue
|
|
}
|
|
if (appImage === 'superseded' && appImageUpdate) {
|
|
const reason = await appImageUpdate.restart(() => stopped)
|
|
// One attempt per supervisor: a failed restart keeps serving the running build.
|
|
appImageUpdate = null
|
|
log(`[serve] Kept the running build: ${reason}.`)
|
|
continue
|
|
}
|
|
record = brakeForBuild(record, identify(options.executable))
|
|
if (record.gaveUp) {
|
|
return refuseGaveUp(record.gaveUp)
|
|
}
|
|
|
|
const attempt = parkWatch
|
|
? await parkWatch.during(options.runOnce)
|
|
: { parked: false, settled: await settle(options.runOnce) }
|
|
if (stopped) {
|
|
break
|
|
}
|
|
if (attempt.parked) {
|
|
// Desktop update parks do not count against the crash brake.
|
|
continue
|
|
}
|
|
let failure: string
|
|
if (attempt.settled.status === 'fulfilled') {
|
|
const code = attempt.settled.value
|
|
if (code === SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE) {
|
|
// An occupied profile warrants standby, not a crash-brake failure.
|
|
const delayMs = standbyDelayMs(standbyAttempt++, random)
|
|
log(
|
|
`[serve] Another Orca serves this profile, or holds it and cannot be shown to have stopped; checking again in ${Math.round(delayMs / 1000)}s.${process.platform === 'linux' ? ` If its message says it may have stopped, \`${orcadStaleLockReclaimCommand(options.userDataPath, null)}\` checks it.` : ''}`
|
|
)
|
|
await sleep(delayMs)
|
|
continue
|
|
}
|
|
failure =
|
|
code === ORCAD_EXIT_TERMINALS_NOT_DURABLE
|
|
? `exited with code ${code}: the terminal daemon did not start, and terminals outside it would not survive a restart (orcad's log above has the cause)`
|
|
: `exited with code ${code}`
|
|
} else {
|
|
const error: unknown = attempt.settled.reason
|
|
failure = (error instanceof Error ? error.message : String(error))
|
|
.replace(/^Orca serve /, '')
|
|
.replace(/\.$/, '')
|
|
}
|
|
standbyAttempt = 0
|
|
const next = recordSupervisedFailure(record, { now: now(), reason: failure, random })
|
|
record = next.record
|
|
writeBrake(options.userDataPath, record)
|
|
if (next.action.kind === 'give-up') {
|
|
log(`[serve] ${next.action.reason}`)
|
|
return 0
|
|
}
|
|
log(
|
|
`[serve] Orca serve ${failure}; restarting in ${Math.max(1, Math.round(next.action.delayMs / 1000))}s.`
|
|
)
|
|
await sleep(next.action.delayMs)
|
|
}
|
|
return 0
|
|
} finally {
|
|
removeStopListener()
|
|
resetSupervisedServeStop()
|
|
parkWatch?.close()
|
|
}
|
|
}
|