mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix(preflight): preserve cancellation grace and bound hook admission
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
@@ -66,9 +66,11 @@ Affected tests follow Vitest's static import graph, so run suites explicitly for
|
||||
|
||||
On local macOS and Linux (including SSH hosts and WSL), `check:changed`, `lint`, the typecheck runners, `pnpm test` / `test:changed`, typed-code-quality audits, and the pre-commit hook share one per-user host validation lane across worktrees and clones. Queued commands print their wait immediately and every 15 seconds, then print admission time. Checks still run and failures still fail; queueing does not create a passing cache entry. Nested runner calls authenticate and reuse the active lane rather than waiting for themselves.
|
||||
|
||||
The pre-commit hook waits at most 30 seconds for admission, because application commits can buffer hook output and cannot offer terminal cancellation. If the lane stays busy, the hook fails before starting lint-staged; no checks are skipped and no commit is created. Retry the commit after active validation completes or the reported port conflict is resolved. This deadline applies only to admission, not to checks that have started. Other CLI validation keeps waiting until admitted or canceled.
|
||||
|
||||
Within that lane, preflight tasks, lint steps, and typecheck projects run serially. The project runner's compiler checkers, Bun compiler threads, and Vitest workers are capped at two (a requested single worker stays single). Rust/Go tools inherit `RAYON_NUM_THREADS` / `GOMAXPROCS` hints, which tools can override; admission is not an OS CPU or memory ceiling. This deliberately favors bounded aggregate load over one agent's fastest preflight. GitHub Actions and native Windows retain their existing scheduling; the lane depends on POSIX process groups and is not a Windows Job Object implementation. Setting `CI=1` for a noninteractive local command does not bypass admission.
|
||||
|
||||
Admission uses a loopback listening socket shared with the validation process group, not a PID file or an expiring heartbeat lock. The kernel releases it when its owners exit, so there is no stale lock to delete or PID-reuse recovery to guess. A supervisor and entry guard monitor each other: cancellation allows ten seconds for tool cleanup, then retires the remaining group; a crashed group supervisor makes the entry guard retire its own group. Workloads must not detach validation workers into independent sessions. Intentionally persistent dev servers and language servers are not managed by this lane.
|
||||
Admission uses a loopback listening socket shared with the validation process group, not a PID file or an expiring heartbeat lock. The kernel releases it when its owners exit, so there is no stale lock to delete or PID-reuse recovery to guess. A supervisor and entry guard monitor each other: cancellation retains the lane for the full ten-second cleanup grace even if an intermediate runner exits first, then retires the remaining group; a crashed group supervisor makes the entry guard retire its own group. Workloads must not detach validation workers into independent sessions. Intentionally persistent dev servers and language servers are not managed by this lane.
|
||||
|
||||
Use the normal command's cancellation control to leave the queue or stop your own run. Do not delete lock files, terminate another worktree's session, or bypass validation to make a commit proceed. If an unrelated service occupies the reported loopback port, choose one unused port with `ORCA_VALIDATION_PORT` **consistently for every Orca worktree on that host**. The default derives from the user's home directory. The lane is host-local, not a distributed limit between a laptop and a remote host, and contention is retried rather than strictly FIFO.
|
||||
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
#!/usr/bin/env sh
|
||||
node config/scripts/preflight/host-validation/run-host-validation.mjs -- pnpm exec lint-staged
|
||||
node config/scripts/preflight/host-validation/run-host-validation.mjs --wait-timeout=30 -- pnpm exec lint-staged
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { writeFileSync } from 'node:fs'
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
if (process.argv[2]) {
|
||||
setTimeout(() => {
|
||||
writeFileSync(process.argv[2], 'cleaned')
|
||||
process.exit(0)
|
||||
}, 1000)
|
||||
}
|
||||
})
|
||||
console.log('ready')
|
||||
setInterval(() => {}, 1000)
|
||||
@@ -1,18 +1,38 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { once } from 'node:events'
|
||||
import { join } from 'node:path'
|
||||
import { delegateHostValidation } from '../host-validation-budget.mjs'
|
||||
|
||||
if (!(await delegateHostValidation())) {
|
||||
if (process.argv.includes('--complete')) {
|
||||
console.log('nested-complete')
|
||||
const lateAdmission = process.argv.includes('--late-admission')
|
||||
if (lateAdmission) {
|
||||
let firstRead = true
|
||||
Object.defineProperty(performance, 'now', {
|
||||
value: () => {
|
||||
const time = firstRead ? 0 : 1000
|
||||
firstRead = false
|
||||
return time
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (!(await delegateHostValidation({ waitTimeoutMs: lateAdmission ? 1 : null }))) {
|
||||
if (process.argv.includes('--complete') || lateAdmission) {
|
||||
console.log(lateAdmission ? 'admitted-late' : 'nested-complete')
|
||||
} else {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
['-e', "process.on('SIGINT', () => {}); setInterval(() => {}, 1000)"],
|
||||
[
|
||||
join(import.meta.dirname, 'validation-descendant.mjs'),
|
||||
...(process.argv.includes('--cleanup-on-cancel')
|
||||
? [join(process.cwd(), 'cleanup-marker')]
|
||||
: [])
|
||||
],
|
||||
{
|
||||
stdio: 'ignore'
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
}
|
||||
)
|
||||
await once(child.stdout, 'data')
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'running',
|
||||
|
||||
@@ -54,10 +54,13 @@ function tryListen(port) {
|
||||
})
|
||||
}
|
||||
|
||||
export async function delegateHostValidation() {
|
||||
export async function delegateHostValidation({ waitTimeoutMs = null } = {}) {
|
||||
if (process.platform === 'win32' || process.env.GITHUB_ACTIONS === 'true') {
|
||||
return false
|
||||
}
|
||||
if (waitTimeoutMs !== null && (!Number.isSafeInteger(waitTimeoutMs) || waitTimeoutMs < 0)) {
|
||||
throw new Error('Validation admission timeout must be a nonnegative integer')
|
||||
}
|
||||
const defaultPort =
|
||||
43000 + (createHash('sha256').update(homedir()).digest().readUInt16BE() % 2000)
|
||||
const port = Number(process.env.ORCA_VALIDATION_PORT ?? defaultPort)
|
||||
@@ -90,9 +93,21 @@ export async function delegateHostValidation() {
|
||||
const startedAt = performance.now()
|
||||
let server
|
||||
let heartbeat
|
||||
let firstAttempt = true
|
||||
try {
|
||||
while (!controller.signal.aborted) {
|
||||
const immediateAttempt = firstAttempt && waitTimeoutMs === 0
|
||||
firstAttempt = false
|
||||
server = await tryListen(port)
|
||||
const remaining =
|
||||
waitTimeoutMs === null ? Infinity : waitTimeoutMs - (performance.now() - startedAt)
|
||||
if (remaining <= 0 && !(server && immediateAttempt)) {
|
||||
server?.close()
|
||||
server = null
|
||||
throw new Error(
|
||||
'Host validation lane is busy; validation did not start. Retry this command after the active validation completes, or resolve a loopback port conflict. Checks were not skipped.'
|
||||
)
|
||||
}
|
||||
if (server) {
|
||||
break
|
||||
}
|
||||
@@ -104,7 +119,9 @@ export async function delegateHostValidation() {
|
||||
announce()
|
||||
heartbeat = setInterval(announce, 15_000)
|
||||
}
|
||||
await delay(250 + Math.random() * 250, undefined, { signal: controller.signal })
|
||||
await delay(Math.min(250 + Math.random() * 250, remaining), undefined, {
|
||||
signal: controller.signal
|
||||
})
|
||||
}
|
||||
if (controller.signal.aborted) {
|
||||
return true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { once } from 'node:events'
|
||||
import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'
|
||||
import { mkdtempSync, mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs'
|
||||
import { createServer } from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -31,12 +31,12 @@ async function setup() {
|
||||
await new Promise((resolve) => reservation.close(resolve))
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-validation-'))
|
||||
directories.push(root)
|
||||
const start = (name, env = {}, args = []) => {
|
||||
const start = (name, env = {}, args = [], entry = fixture) => {
|
||||
const cwd = join(root, name)
|
||||
mkdirSync(cwd)
|
||||
const child = spawn(
|
||||
process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.execPath,
|
||||
[fixture, ...args],
|
||||
[entry, ...args],
|
||||
{
|
||||
cwd,
|
||||
env: {
|
||||
@@ -65,6 +65,7 @@ async function setup() {
|
||||
child.stderr.on('data', append)
|
||||
return {
|
||||
child,
|
||||
cwd,
|
||||
output: () => output,
|
||||
wait: (text) =>
|
||||
output.includes(text)
|
||||
@@ -161,6 +162,64 @@ it.skipIf(process.platform === 'win32')(
|
||||
20_000
|
||||
)
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'preserves descendant cleanup when a canceled intermediate exits first',
|
||||
async () => {
|
||||
const start = await setup()
|
||||
const first = start('owner', {}, ['--cleanup-on-cancel'])
|
||||
await first.wait('"running"')
|
||||
first.child.kill('SIGTERM')
|
||||
expect(await first.closed).toEqual({ code: 143, signal: null })
|
||||
const marker = join(first.cwd, 'cleanup-marker')
|
||||
expect(existsSync(marker) ? readFileSync(marker, 'utf8') : null).toBe('cleaned')
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'bounds hook admission without running or skipping its validation, then permits retry',
|
||||
async () => {
|
||||
const start = await setup()
|
||||
const first = start('owner')
|
||||
const owner = running(await first.wait('"running"'))
|
||||
const wrapper = fileURLToPath(new URL('./run-host-validation.mjs', import.meta.url))
|
||||
const args = [
|
||||
'--wait-timeout=0',
|
||||
'--',
|
||||
process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.execPath,
|
||||
'-e',
|
||||
'console.log("validated")'
|
||||
]
|
||||
const blocked = start('hook', {}, args, wrapper)
|
||||
expect(await blocked.closed).toEqual({ code: 1, signal: null })
|
||||
expect(blocked.output()).toContain('validation did not start')
|
||||
expect(blocked.output()).not.toContain('validated')
|
||||
expect(gone(owner.pid)).toBe(false)
|
||||
first.child.stdin.write('0\n')
|
||||
expect(await first.closed).toEqual({ code: 0, signal: null })
|
||||
const retry = start('retry', {}, args, wrapper)
|
||||
expect(await retry.closed).toEqual({ code: 0, signal: null })
|
||||
expect(retry.output()).toContain('validated')
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it.skipIf(process.platform === 'win32')(
|
||||
'rejects a late successful bind and releases it for the next command',
|
||||
async () => {
|
||||
const start = await setup()
|
||||
const late = start('late', {}, ['--late-admission'])
|
||||
expect(await late.closed).toEqual({ code: 1, signal: null })
|
||||
expect(late.output()).toContain('validation did not start')
|
||||
expect(late.output()).not.toContain('admitted-late')
|
||||
const retry = start('retry')
|
||||
await retry.wait('"running"')
|
||||
retry.child.stdin.write('0\n')
|
||||
expect(await retry.closed).toEqual({ code: 0, signal: null })
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it.skipIf(process.platform === 'win32').each(['parent', 'group', 'entry', 'cancel'])(
|
||||
'recovers from %s loss only after retiring its process tree',
|
||||
async (target) => {
|
||||
|
||||
@@ -3,21 +3,40 @@ import { constants } from 'node:os'
|
||||
import { delegateHostValidation } from './host-validation-budget.mjs'
|
||||
import { resolvePnpmCliInvocation } from '../../tool-invocation/pnpm-cli-invocation.mjs'
|
||||
|
||||
if (!(await delegateHostValidation())) {
|
||||
const [program, ...args] = process.argv.slice(process.argv[2] === '--' ? 3 : 2)
|
||||
if (!program) {
|
||||
throw new Error('Usage: run-host-validation.mjs -- <program> [args...]')
|
||||
try {
|
||||
const argv = process.argv.slice(2)
|
||||
let waitTimeoutMs = null
|
||||
if (argv[0]?.startsWith('--wait-timeout=')) {
|
||||
const seconds = Number(argv.shift().slice('--wait-timeout='.length))
|
||||
waitTimeoutMs = seconds * 1000
|
||||
if (!Number.isSafeInteger(waitTimeoutMs) || waitTimeoutMs < 0) {
|
||||
throw new Error('--wait-timeout must be a nonnegative number of seconds')
|
||||
}
|
||||
}
|
||||
const pnpm = program === 'pnpm' ? resolvePnpmCliInvocation() : null
|
||||
const child = spawnProcess({
|
||||
program: pnpm?.command ?? program,
|
||||
args: [...(pnpm?.prefixArgs ?? []), ...args],
|
||||
stdio: 'inherit'
|
||||
})
|
||||
process.exitCode = await new Promise((resolve, reject) => {
|
||||
child.on('error', reject)
|
||||
child.on('close', (code, signal) =>
|
||||
resolve(code ?? (signal ? 128 + (constants.signals[signal] ?? 0) : 1))
|
||||
if (argv[0] === '--') {
|
||||
argv.shift()
|
||||
}
|
||||
const [program, ...args] = argv
|
||||
if (!program) {
|
||||
throw new Error(
|
||||
'Usage: run-host-validation.mjs [--wait-timeout=<seconds>] -- <program> [args...]'
|
||||
)
|
||||
})
|
||||
}
|
||||
if (!(await delegateHostValidation({ waitTimeoutMs }))) {
|
||||
const pnpm = program === 'pnpm' ? resolvePnpmCliInvocation() : null
|
||||
const child = spawnProcess({
|
||||
program: pnpm?.command ?? program,
|
||||
args: [...(pnpm?.prefixArgs ?? []), ...args],
|
||||
stdio: 'inherit'
|
||||
})
|
||||
process.exitCode = await new Promise((resolve, reject) => {
|
||||
child.on('error', reject)
|
||||
child.on('close', (code, signal) =>
|
||||
resolve(code ?? (signal ? 128 + (constants.signals[signal] ?? 0) : 1))
|
||||
)
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`validation: ${error.message}`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
|
||||
@@ -45,6 +45,12 @@ process.on('message', (message, server) => {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
if (stopping) {
|
||||
if (process.connected) {
|
||||
process.send({ type: 'result', code }, () => {})
|
||||
}
|
||||
return
|
||||
}
|
||||
if (!process.connected) {
|
||||
stopGroup()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user