fix(git): retry fetches after remote-ref update races

This commit is contained in:
Jinjing
2026-11-12 23:52:58 -08:00
parent 66db0a6a50
commit 9d638032f9
3 changed files with 228 additions and 0 deletions
@@ -3,6 +3,7 @@ import {
resolveGitFetchHeadCommand,
runWithGitFetchHeadLock
} from '../../../shared/git/fetch/git-fetch-head-lock'
import { retryGitFetchRefRace } from '../../../shared/git/fetch/git-fetch-ref-race'
import { pendingWslDriveCheckoutGitRouting } from './host-routing/wsl-drive-checkout-git-routing'
import type { ResolvedCommand } from './host-routing/wsl-command-resolution'
import { annotateWslHostFailure, isTransientWslFailure } from './host-routing/wsl-host-failure'
@@ -163,6 +164,14 @@ async function gitExecFileAsyncUnlocked(
throw createAbortError()
}
return await runCapturedCommand()
} catch (error) {
return await retryGitFetchRefRace(
args,
error,
runCapturedCommand,
() => terminationState.current,
options.signal
)
} finally {
const termination = terminationState.current
if (termination) {
@@ -0,0 +1,143 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const { capture } = vi.hoisted(() => ({ capture: vi.fn() }))
vi.mock('./process-capture/exec-file-capture', () => ({
execFileCapture: capture,
execFileCaptureToTermination: capture
}))
vi.mock('../../../shared/git/fetch/git-fetch-head-lock', async (importOriginal) => ({
...(await importOriginal<Record<string, unknown>>()),
runWithGitFetchHeadLock: (_cwd: string, _signal: AbortSignal | undefined, run: () => unknown) =>
run()
}))
import { gitExecFileAsync } from './git-exec-file'
const CAS_DIAGNOSTIC =
'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided'
const options = { cwd: '/repo', env: { GIT_SSH_COMMAND: 'ssh' } }
function failure(stderr: string, extra: Record<string, unknown> = {}): Error {
return Object.assign(new Error(stderr), { code: 1, stderr, ...extra })
}
function answerWith(...outcomes: (Error | string)[]): void {
capture.mockImplementation((_command, _args, captureOptions) => {
captureOptions.onChildTerminated()
const outcome = outcomes.length > 1 ? outcomes.shift()! : outcomes[0]
return outcome instanceof Error
? Promise.reject(outcome)
: Promise.resolve({ stdout: outcome, stderr: '' })
})
}
beforeEach(() => {
vi.useFakeTimers()
capture.mockReset()
})
afterEach(() => vi.useRealTimers())
describe('standalone fetch ref-update recovery', () => {
it.each([
CAS_DIAGNOSTIC,
"error: cannot lock ref 'refs/remotes/upstream/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
])('recovers a failed fetch after %s', async (diagnostic) => {
answerWith(failure(diagnostic), 'fresh refs')
const result = gitExecFileAsync(['-c', 'gc.auto=0', 'fetch', 'origin'], options)
await vi.runAllTimersAsync()
await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' })
})
it('waits for the failed foreground child to exit before replaying fetch', async () => {
let exited = false
capture.mockImplementationOnce((_command, _args, captureOptions) => {
setTimeout(() => {
exited = true
captureOptions.onChildTerminated()
}, 1_000)
return Promise.reject(failure(CAS_DIAGNOSTIC))
})
capture.mockImplementation((_command, _args, captureOptions) => {
captureOptions.onChildTerminated()
return exited
? Promise.resolve({ stdout: 'fresh refs', stderr: '' })
: Promise.reject(new Error('The previous writer is still running'))
})
const result = gitExecFileAsync(['fetch', 'origin'], options)
await vi.runAllTimersAsync()
await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' })
})
it('retains the original failure when ref contention persists', async () => {
const original = failure(CAS_DIAGNOSTIC)
answerWith(original)
const result = gitExecFileAsync(['fetch', 'origin'], options)
const rejection = expect(result).rejects.toMatchObject({
cause: original,
message: expect.stringContaining(
'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided'
)
})
await vi.runAllTimersAsync()
await rejection
})
it.each([
failure('fatal: Authentication failed'),
failure(`${CAS_DIAGNOSTIC}\nfatal: Could not resolve host: example.invalid`),
failure(`${CAS_DIAGNOSTIC}\nremote: error: access denied`),
failure(`${CAS_DIAGNOSTIC}\nremote: ERROR access denied`),
failure(`${CAS_DIAGNOSTIC}\nfatal: bad object HEAD`),
failure(
"error: cannot lock ref 'refs/heads/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
),
failure("fatal: Unable to create '.git/refs/remotes/origin/main.lock': File exists"),
failure(CAS_DIAGNOSTIC, { killed: true, signal: 'SIGTERM' }),
failure(CAS_DIAGNOSTIC, { name: 'GitCommandTimeoutError' }),
failure(CAS_DIAGNOSTIC, { outputTruncated: true }),
Object.assign(new Error(CAS_DIAGNOSTIC), { stdout: CAS_DIAGNOSTIC })
])('preserves a non-retriable failure: %s', async (original) => {
answerWith(original, 'failure was hidden')
const result = gitExecFileAsync(['fetch', 'origin'], options)
const rejection = expect(result).rejects.toBe(original)
await vi.runAllTimersAsync()
await rejection
})
it('cancels the backoff instead of turning a cancelled fetch into success', async () => {
const controller = new AbortController()
answerWith(failure(CAS_DIAGNOSTIC), 'failure was hidden')
const result = gitExecFileAsync(['fetch', 'origin'], {
...options,
signal: controller.signal,
observeChildTermination: (terminated) => {
void terminated.then(() => setTimeout(() => controller.abort(), 10))
}
})
const rejection = expect(result).rejects.toMatchObject({ name: 'AbortError' })
await vi.runAllTimersAsync()
await rejection
})
it('surfaces a different failure from a later fetch attempt unchanged', async () => {
const terminal = failure('fatal: Authentication failed')
answerWith(failure(CAS_DIAGNOSTIC), terminal, 'failure was hidden')
const result = gitExecFileAsync(['fetch', 'origin'], options)
const rejection = expect(result).rejects.toBe(terminal)
await vi.runAllTimersAsync()
await rejection
})
it('preserves pull failures even when stderr resembles a failed fetch', async () => {
const original = failure(CAS_DIAGNOSTIC)
capture.mockImplementationOnce((_command, _args, captureOptions) => {
captureOptions.onChildTerminated()
return Promise.reject(original)
})
answerWith('Already up to date')
const result = gitExecFileAsync(['pull', '--ff-only'], options)
const rejection = expect(result).rejects.toBe(original)
await vi.runAllTimersAsync()
await rejection
})
})
@@ -0,0 +1,76 @@
import { setTimeout as delay } from 'node:timers/promises'
import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../abort-signal-reason'
import { readGitCommandFailureStderr } from '../commands/git-command-failure-text'
import { findGitSubcommandIndex } from '../commands/git-command-classification'
import { isExecKilledError, stripCredentialsFromMessage } from '../remotes/git-remote-error'
const REMOTE_REF_CAS_ERROR =
/^(?:error|fatal): (?:fetching ref refs\/remotes\/[^\s]+ failed: incorrect old value provided|cannot lock ref 'refs\/remotes\/[^'\s]+': is at (?:[0-9a-f]{40}|[0-9a-f]{64}) but expected (?:[0-9a-f]{40}|[0-9a-f]{64}))$/i
const OTHER_GIT_ERROR = /\b(?:error|fatal):|^remote:.*\b(?:error|fatal)\b/i
function remoteRefRaceStderr(error: unknown): string | null {
if (!(error instanceof Error) || isExecKilledError(error)) {
return null
}
if (
('timedOut' in error && error.timedOut === true) ||
('outputTruncated' in error && error.outputTruncated === true) ||
error.name === 'AbortError' ||
error.name === 'TimeoutError' ||
error.name === 'GitCommandTimeoutError' ||
('code' in error &&
(error.code === 'ABORT_ERR' ||
error.code === 'ETIMEDOUT' ||
error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER'))
) {
return null
}
const stderr = readGitCommandFailureStderr(error)
if (!stderr) {
return null
}
let matched = false
for (const line of stderr.split(/[\r\n]+/)) {
if (REMOTE_REF_CAS_ERROR.test(line.trim())) {
matched = true
} else if (OTHER_GIT_ERROR.test(line.trim())) {
return null
}
}
return matched ? stderr : null
}
/** Only standalone fetches may be replayed; pull can fail after changing the worktree. */
export async function retryGitFetchRefRace<T>(
args: readonly string[],
initialError: unknown,
attempt: () => Promise<T>,
terminated: () => Promise<void> | null,
signal?: AbortSignal
): Promise<T> {
if (args[findGitSubcommandIndex(args)] !== 'fetch') {
throw initialError
}
let error = initialError
for (let retry = 0; ; retry += 1) {
const stderr = remoteRefRaceStderr(error)
if (stderr === null) {
throw error
}
throwIfSignalAborted(signal)
if (retry === 2) {
const detail = stripCredentialsFromMessage(stderr.trim())
throw new Error(`${detail}\nAnother Git process updated remote references. Try again.`, {
cause: initialError
})
}
await waitForPromiseWithSignal(terminated() ?? Promise.resolve(), signal)
await delay(50 + retry * 100 + Math.floor(Math.random() * 50), undefined, { signal })
throwIfSignalAborted(signal)
try {
return await attempt()
} catch (nextError) {
error = nextError
}
}
}