Isolate worktree checkout admission and bound default workers

Co-authored-by: Neil <4138956+nwparker@users.noreply.github.com>
This commit is contained in:
Brennan Benson
2026-11-13 00:32:09 -08:00
co-authored by Neil
parent 6734880408
commit a2aa4d449f
14 changed files with 271 additions and 25 deletions
@@ -75,6 +75,7 @@ async function gitExecFileAsyncUnlocked(
cwd: options.cwd,
wslDistro: options.wslDistro,
tier: options.admissionTier,
admissionClass: options.admissionClass,
signal: options.admissionSignal ?? options.signal
})
span?.setAttribute('git.queue_wait_ms', grant.queueWaitMs)
@@ -26,6 +26,8 @@ export type GitExecOptions = {
observeChildTermination?: (terminated: Promise<void>, writes: boolean) => void
/** Scheduler priority for this child; status is the safe default. */
admissionTier?: GitAdmissionTier
/** File materialization during worktree creation has a separate concurrency budget. */
admissionClass?: 'checkout'
/** Skips general admission; only for a caller that bounds its own concurrency (worktree deletes). */
admissionExempt?: true
/** Native macOS deletion spawns use the shared process worker. */
@@ -5,8 +5,8 @@ import {
type GitWorktreeExecOptions
} from '../worktree-operation-options'
// Git checks out with one worker by default; 0 means one per core (Git 2.32+; older Git ignores the key).
const PARALLEL_CHECKOUT_GIT_ARGS = ['-c', 'checkout.workers=0'] as const
// Bound default fan-out across simultaneous creates; older Git ignores the key (2.32+).
const PARALLEL_CHECKOUT_GIT_ARGS = ['-c', 'checkout.workers=4'] as const
/** Parallel checkout args unless checkout.workers is configured at any scope for this repo and host. */
export async function checkoutWorkersGitArgs(
@@ -271,6 +271,7 @@ async function performAddWorktree(
try {
await gitExecFileAsync([...(await checkoutWorkersArgs), ...args], {
...gitExecOptions(repoPath, options),
...(!noCheckout ? { admissionClass: 'checkout' as const } : {}),
// Resolve per call so later environment overrides take effect.
timeout: resolveWorktreeAddTimeoutMs(),
...(checkoutProgress ? { onStderr: checkoutProgress.read } : {})
@@ -13,6 +13,7 @@ import {
type LocalBaseBranchGit
} from '../../../../shared/worktree/local-base-branch-fast-forward'
import { gitExecFileAsync } from '../../command-runner/git-exec-file'
import { findGitSubcommandIndex } from '../../../../shared/git/commands/git-command-classification'
import { translateWslOutputPaths } from '../../command-runner/host-routing/wsl-path-translation'
import { runWithGitReadCacheInvalidation } from '../../source-control/git-read-cache-invalidation'
import type { AddWorktreeOptions, GitWorktreeExecOptions } from '../worktree-operation-options'
@@ -115,7 +116,13 @@ function localBaseBranchGit(options: GitWorktreeExecOptions): LocalBaseBranchGit
...(options.admissionTier ? { admissionTier: options.admissionTier } : {})
})
return {
exec: (args, cwd) => gitExecFileAsync(args, execOptions(cwd)),
exec: (args, cwd) =>
gitExecFileAsync(args, {
...execOptions(cwd),
...(args[findGitSubcommandIndex(args)] === 'merge'
? { admissionClass: 'checkout' as const }
: {})
}),
listWorktrees: async (repoPath) => {
const { stdout } = await withRepositoryGitCause(
repoPath,
@@ -0,0 +1,141 @@
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { GitAdmissionScheduler } from '../../../../shared/git/admission/git-admission-scheduler'
import { findGitSubcommandIndex } from '../../../../shared/git/commands/git-command-classification'
import { createWorktreePreparationLockReason } from '../../../../shared/worktree/create-preparation'
import { _resetGitAdmissionForTests } from '../../command-runner/admission/git-subprocess-admission'
import { gitExecFileAsync } from '../../command-runner/git-exec-file'
import { addWorktree } from './worktree-add'
import { addSparseWorktree } from './worktree-sparse-add'
import {
finalizePreparedWorktree,
prepareWorktreeCreateCheckout
} from './worktree-create-preparation'
import { refreshPreparedWorktreeTip } from './worktree-preparation-tip-refresh'
import { refreshLocalBaseRefForWorktreeCreate } from './worktree-base-refresh'
let root: string
let repo: string
let admission: GitAdmissionScheduler
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-create-admission-'))
repo = join(root, 'repo')
admission = new GitAdmissionScheduler()
_resetGitAdmissionForTests(admission)
await gitExecFileAsync(['init', '-q', repo], { cwd: root })
await gitExecFileAsync(['symbolic-ref', 'HEAD', 'refs/heads/main'], { cwd: repo })
await mkdir(join(repo, 'included'))
await writeFile(join(repo, 'included', 'file.txt'), 'one\n')
await gitExecFileAsync(['add', '.'], { cwd: repo })
await gitExecFileAsync(
['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture'],
{ cwd: repo }
)
})
afterEach(async () => {
vi.restoreAllMocks()
_resetGitAdmissionForTests()
await rm(root, { recursive: true, force: true })
})
async function behindCheckoutBudget(command: string, run: () => Promise<unknown>): Promise<void> {
const held = await Promise.all(
Array.from({ length: 3 }, () =>
admission.acquire({
args: ['worktree', 'add'],
cwd: repo,
admissionClass: 'checkout',
tier: 'interactive'
})
)
)
const requested = Promise.withResolvers<void>()
let admitted = false
const acquire = admission.acquire.bind(admission)
const spy = vi.spyOn(admission, 'acquire').mockImplementation((request) => {
const grant = acquire(request)
if (
request.args[findGitSubcommandIndex(request.args)] === command &&
(command !== 'sparse-checkout' || request.args.includes('set'))
) {
requested.resolve()
void grant.then(() => {
admitted = true
})
}
return grant
})
const operation = run()
try {
await requested.promise
const read = await gitExecFileAsync(['rev-parse', '--verify', 'HEAD'], { cwd: repo })
expect(read.stdout.trim()).toMatch(/^[0-9a-f]{40}$/)
expect(admitted).toBe(false)
} finally {
held.forEach((grant) => grant.release())
await operation
spy.mockRestore()
}
}
it('isolates ordinary and sparse creates while honoring the execution host checkout config', async () => {
await gitExecFileAsync(['config', 'checkout.workers', 'not-an-integer'], { cwd: repo })
await expect(addWorktree(repo, join(root, 'invalid'), 'invalid', 'main')).rejects.toThrow()
await gitExecFileAsync(['config', 'checkout.workers', '0'], { cwd: repo })
const ordinary = join(root, 'ordinary')
await behindCheckoutBudget('worktree', () => addWorktree(repo, ordinary, 'ordinary', 'main'))
expect(await readFile(join(ordinary, 'included', 'file.txt'), 'utf8')).toBe('one\n')
await gitExecFileAsync(['config', '--unset', 'checkout.workers'], { cwd: repo })
const sparse = join(root, 'sparse')
await behindCheckoutBudget('sparse-checkout', () =>
addSparseWorktree(repo, sparse, 'sparse', ['included'], 'main')
)
expect(await readFile(join(sparse, 'included', 'file.txt'), 'utf8')).toBe('one\n')
})
it('isolates preparation, fetched-tip refresh and changed-head finalization materializations', async () => {
const prepared = join(root, 'prepared')
const reason = createWorktreePreparationLockReason('admission')
await behindCheckoutBudget('reset', () =>
prepareWorktreeCreateCheckout(repo, prepared, 'main', reason)
)
await writeFile(join(repo, 'included', 'file.txt'), 'two\n')
await gitExecFileAsync(['commit', '-qam', 'next', '--author=Test <test@example.invalid>'], {
cwd: repo,
env: { ...process.env, GIT_COMMITTER_NAME: 'Test', GIT_COMMITTER_EMAIL: 'test@example.invalid' }
})
await behindCheckoutBudget('reset', () =>
refreshPreparedWorktreeTip(repo, prepared, 'main', reason)
)
expect(await readFile(join(prepared, 'included', 'file.txt'), 'utf8')).toBe('two\n')
await writeFile(join(repo, 'included', 'file.txt'), 'three\n')
await gitExecFileAsync(
['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qam', 'third'],
{ cwd: repo }
)
const target = join(root, 'finalized')
await behindCheckoutBudget('reset', () =>
finalizePreparedWorktree(repo, prepared, target, 'finalized', 'main', false, {}, reason)
)
expect(await readFile(join(target, 'included', 'file.txt'), 'utf8')).toBe('three\n')
})
it('isolates the production base fast-forward with its leading Git config arguments', async () => {
const newer = join(root, 'newer')
await addWorktree(repo, newer, 'newer', 'main')
await writeFile(join(newer, 'included', 'file.txt'), 'remote\n')
await gitExecFileAsync(
['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-qam', 'remote'],
{ cwd: newer }
)
const { stdout } = await gitExecFileAsync(['rev-parse', 'HEAD'], { cwd: newer })
await gitExecFileAsync(['update-ref', 'refs/remotes/origin/main', stdout.trim()], { cwd: repo })
await behindCheckoutBudget('merge', () =>
refreshLocalBaseRefForWorktreeCreate(repo, 'origin/main', 'refs/remotes/origin/main')
)
expect(await readFile(join(repo, 'included', 'file.txt'), 'utf8')).toBe('remote\n')
})
@@ -98,7 +98,11 @@ export async function prepareWorktreeCreateCheckout(
'--hard',
materializationBase
],
{ ...gitExecOptions(worktreePath, options), timeout: resolveWorktreeAddTimeoutMs() }
{
...gitExecOptions(worktreePath, options),
admissionClass: 'checkout',
timeout: resolveWorktreeAddTimeoutMs()
}
)
await verifyWorktreePreparationLockAtPath(lockPath, lockReason, options.signal)
})
@@ -93,7 +93,7 @@ export async function performPreparedWorktreeFinalization(
'--hard',
targetHead
],
gitExecOptions(preparedPath, finalizeGitOptions)
{ ...gitExecOptions(preparedPath, finalizeGitOptions), admissionClass: 'checkout' }
)
}
@@ -66,7 +66,7 @@ export async function refreshPreparedWorktreeTip(
'--hard',
targetHead
],
gitExecOptions(preparedPath, refreshOptions)
{ ...gitExecOptions(preparedPath, refreshOptions), admissionClass: 'checkout' }
)
)
await verifyWorktreePreparationLockAtPath(lockPath, lockReason, refreshOptions.signal)
@@ -38,14 +38,14 @@ export async function addSparseWorktree(
checkoutWorkersGitArgs(worktreePath, options),
gitExecFileAsync(['sparse-checkout', 'init', '--cone'], gitExecOptions(worktreePath, options))
])
await gitExecFileAsync(
[...longPathArgs, 'sparse-checkout', 'set', '--', ...directories],
gitExecOptions(worktreePath, options)
)
await gitExecFileAsync(
[...checkoutWorkersArgs, ...longPathArgs, 'checkout', branch],
gitExecOptions(worktreePath, options)
)
await gitExecFileAsync([...longPathArgs, 'sparse-checkout', 'set', '--', ...directories], {
...gitExecOptions(worktreePath, options),
admissionClass: 'checkout'
})
await gitExecFileAsync([...checkoutWorkersArgs, ...longPathArgs, 'checkout', branch], {
...gitExecOptions(worktreePath, options),
admissionClass: 'checkout'
})
return addResult
} catch (error) {
const wrapped: SparseWorktreeCreateError =
@@ -0,0 +1,68 @@
import { expect, it } from 'vitest'
import { GitAdmissionScheduler } from './git-admission-scheduler'
const checkout = {
args: ['worktree', 'add', '/checkout'],
cwd: '/repo',
admissionClass: 'checkout' as const
}
it('reserves interactive create capacity and leaves reads and other mutations independent', async () => {
const scheduler = new GitAdmissionScheduler()
const preparations = await Promise.all([
scheduler.acquire({ ...checkout, tier: 'status' }),
scheduler.acquire({ ...checkout, tier: 'status' })
])
const cancelPreparation = new AbortController()
const queuedPreparation = scheduler.acquire({
...checkout,
tier: 'status',
signal: cancelPreparation.signal
})
const canceled = expect(queuedPreparation).rejects.toMatchObject({ name: 'AbortError' })
const interactive = await scheduler.acquire({ ...checkout, tier: 'interactive' })
const cancelCreate = new AbortController()
const queuedCreate = scheduler.acquire({
...checkout,
tier: 'interactive',
signal: cancelCreate.signal
})
const canceledCreate = expect(queuedCreate).rejects.toMatchObject({ name: 'AbortError' })
const unrelated = await Promise.all(
[
['rev-parse', 'HEAD'],
['fetch', 'origin'],
['restore', 'file'],
['checkout', 'feature']
].map((args) => scheduler.acquire({ args, cwd: '/other', tier: 'interactive' }))
)
unrelated.forEach((grant) => grant.release())
cancelPreparation.abort()
cancelCreate.abort()
await Promise.all([canceled, canceledCreate])
preparations.forEach((grant) => grant.release())
interactive.release()
})
it('reuses checkout capacity after cancellation and idempotent release across WSL routes', async () => {
const scheduler = new GitAdmissionScheduler({ checkoutCap: 4, checkoutHeadroom: 0 })
const held = await Promise.all([
scheduler.acquire({ ...checkout, wslDistro: 'Ubuntu' }),
scheduler.acquire({ ...checkout, cwd: '\\\\wsl.localhost\\Ubuntu\\repo' })
])
const cancel = new AbortController()
const waiting = scheduler.acquire({ ...checkout, wslDistro: 'ubuntu', signal: cancel.signal })
const rejected = expect(waiting).rejects.toMatchObject({ name: 'AbortError' })
const otherRoute = await scheduler.acquire({ ...checkout, wslDistro: 'Debian' })
cancel.abort()
await rejected
const replacement = scheduler.acquire({ ...checkout, wslDistro: 'UBUNTU' })
held[0].release()
held[0].release()
const granted = await replacement
granted.release()
held[1].release()
otherRoute.release()
const next = await scheduler.acquire({ ...checkout, wslDistro: 'Ubuntu' })
next.release()
})
@@ -91,7 +91,9 @@ export class GitAdmissionScheduler {
route: string | null
budgetKeys: readonly string[]
} {
const admissionClass = classifyGitCommand(request.args) === 'network' ? 'network' : 'general'
const admissionClass =
request.admissionClass ??
(classifyGitCommand(request.args) === 'network' ? 'network' : 'general')
const route = routeKey(request)
const keys: string[] = [admissionClass]
if (route) {
@@ -110,17 +112,22 @@ export class GitAdmissionScheduler {
}
const isRoute = key.startsWith('route:')
const isNetwork = key === 'network'
const isCheckout = key === 'checkout'
budget = {
baseCapacity: isRoute
? this.config.routeCap
: isNetwork
? this.config.networkCap
: this.config.generalCap,
: isCheckout
? this.config.checkoutCap
: this.config.generalCap,
headroomCapacity: isRoute
? this.config.routeHeadroom
: isNetwork
? this.config.networkHeadroom
: this.config.generalHeadroom,
: isCheckout
? this.config.checkoutHeadroom
: this.config.generalHeadroom,
baseUsed: 0,
headroomUsed: 0
}
@@ -5,16 +5,18 @@ import type { GitAdmissionTier } from '../../rpc-contract/git-admission-tier-par
const parallelism = typeof os.availableParallelism === 'function' ? os.availableParallelism() : 1
export const GENERAL_CAP = Math.max(2, Math.min(4, parallelism - 4))
export const NETWORK_CAP = 3
export const CHECKOUT_CAP = 2
export const GENERAL_HEADROOM = 2
export const NETWORK_HEADROOM = 1
export const CHECKOUT_HEADROOM = 1
export const ROUTE_CAP = 2
export const ROUTE_HEADROOM = 1
export const GIT_ADMISSION_AGING_MS = 15_000
// General 4+2 and network 3+1 are disjoint, so at most ten admitted git children run globally;
// General 4+2, network 3+1 and checkout 2+1 admit at most thirteen Git roots per runtime;
// worktree deletes bypass admission under their own limit (WORKTREE_DELETE_CONCURRENCY).
export const MAX_GIT_CHILDREN = 10
export const MAX_GIT_CHILDREN = 13
export type AdmissionClass = 'general' | 'network'
export type AdmissionClass = 'general' | 'network' | 'checkout'
export type AdmissionSlotKind = 'base' | 'headroom'
export type GitAdmissionRequest = {
@@ -22,6 +24,7 @@ export type GitAdmissionRequest = {
cwd: string
wslDistro?: string
tier?: GitAdmissionTier
admissionClass?: 'checkout'
signal?: AbortSignal
}
@@ -55,8 +58,10 @@ export type GitAdmissionEvent = {
export type AdmissionSchedulerConfig = {
generalCap: number
networkCap: number
checkoutCap: number
generalHeadroom: number
networkHeadroom: number
checkoutHeadroom: number
routeCap: number
routeHeadroom: number
agingMs: number
@@ -67,8 +72,10 @@ export type AdmissionSchedulerConfig = {
export const DEFAULT_ADMISSION_SCHEDULER_CONFIG: AdmissionSchedulerConfig = {
generalCap: GENERAL_CAP,
networkCap: NETWORK_CAP,
checkoutCap: CHECKOUT_CAP,
generalHeadroom: GENERAL_HEADROOM,
networkHeadroom: NETWORK_HEADROOM,
checkoutHeadroom: CHECKOUT_HEADROOM,
routeCap: ROUTE_CAP,
routeHeadroom: ROUTE_HEADROOM,
agingMs: GIT_ADMISSION_AGING_MS,
@@ -25,7 +25,8 @@ const createTierLanes = (): TierLanes => ({
export class GitAdmissionWaiterQueue {
private readonly lanes: Record<AdmissionClass, TierLanes> = {
general: createTierLanes(),
network: createTierLanes()
network: createTierLanes(),
checkout: createTierLanes()
}
private readonly countsByBudget = new Map<string, number>()
private readonly baseCandidates: Record<AdmissionClass, Record<GitAdmissionTier, CandidateHeap>> =
@@ -39,11 +40,17 @@ export class GitAdmissionWaiterQueue {
interactive: new CandidateHeap(),
status: new CandidateHeap(),
background: new CandidateHeap()
},
checkout: {
interactive: new CandidateHeap(),
status: new CandidateHeap(),
background: new CandidateHeap()
}
}
private readonly headroomCandidates: Record<AdmissionClass, CandidateHeap> = {
general: new CandidateHeap(),
network: new CandidateHeap()
network: new CandidateHeap(),
checkout: new CandidateHeap()
}
private totalCount = 0
@@ -53,8 +60,9 @@ export class GitAdmissionWaiterQueue {
/** @internal - exposed for bounded-storage regression tests only. */
get candidateCountForTests(): number {
let count = this.headroomCandidates.general.size + this.headroomCandidates.network.size
for (const admissionClass of ['general', 'network'] as const) {
let count = 0
for (const admissionClass of ['general', 'network', 'checkout'] as const) {
count += this.headroomCandidates[admissionClass].size
for (const tier of TIERS) {
count += this.baseCandidates[admissionClass][tier].size
}
@@ -136,7 +144,7 @@ export class GitAdmissionWaiterQueue {
}
snapshot(): AdmissionWaiter[] {
return (['general', 'network'] as const)
return (['general', 'network', 'checkout'] as const)
.flatMap((admissionClass) =>
TIERS.flatMap((tier) => {
return [...this.lanes[admissionClass][tier].values()].flatMap((lane) =>