mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(git): retry fetches after remote-ref update races
This commit is contained in:
@@ -3,6 +3,7 @@ import {
|
||||
resolveGitFetchHeadCommand,
|
||||
runWithGitFetchHeadLock
|
||||
} from '../../../shared/git/fetch/git-fetch-head-lock'
|
||||
import { retryGitFetchRefRace } from '../../../shared/git/fetch/git-fetch-ref-race'
|
||||
import { pendingWslDriveCheckoutGitRouting } from './host-routing/wsl-drive-checkout-git-routing'
|
||||
import type { ResolvedCommand } from './host-routing/wsl-command-resolution'
|
||||
import { annotateWslHostFailure, isTransientWslFailure } from './host-routing/wsl-host-failure'
|
||||
@@ -163,6 +164,14 @@ async function gitExecFileAsyncUnlocked(
|
||||
throw createAbortError()
|
||||
}
|
||||
return await runCapturedCommand()
|
||||
} catch (error) {
|
||||
return await retryGitFetchRefRace(
|
||||
args,
|
||||
error,
|
||||
runCapturedCommand,
|
||||
() => terminationState.current,
|
||||
options.signal
|
||||
)
|
||||
} finally {
|
||||
const termination = terminationState.current
|
||||
if (termination) {
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { capture } = vi.hoisted(() => ({ capture: vi.fn() }))
|
||||
vi.mock('./process-capture/exec-file-capture', () => ({
|
||||
execFileCapture: capture,
|
||||
execFileCaptureToTermination: capture
|
||||
}))
|
||||
vi.mock('../../../shared/git/fetch/git-fetch-head-lock', async (importOriginal) => ({
|
||||
...(await importOriginal<Record<string, unknown>>()),
|
||||
runWithGitFetchHeadLock: (_cwd: string, _signal: AbortSignal | undefined, run: () => unknown) =>
|
||||
run()
|
||||
}))
|
||||
|
||||
import { gitExecFileAsync } from './git-exec-file'
|
||||
|
||||
const CAS_DIAGNOSTIC =
|
||||
'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided'
|
||||
const options = { cwd: '/repo', env: { GIT_SSH_COMMAND: 'ssh' } }
|
||||
|
||||
function failure(stderr: string, extra: Record<string, unknown> = {}): Error {
|
||||
return Object.assign(new Error(stderr), { code: 1, stderr, ...extra })
|
||||
}
|
||||
|
||||
function answerWith(...outcomes: (Error | string)[]): void {
|
||||
capture.mockImplementation((_command, _args, captureOptions) => {
|
||||
captureOptions.onChildTerminated()
|
||||
const outcome = outcomes.length > 1 ? outcomes.shift()! : outcomes[0]
|
||||
return outcome instanceof Error
|
||||
? Promise.reject(outcome)
|
||||
: Promise.resolve({ stdout: outcome, stderr: '' })
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
capture.mockReset()
|
||||
})
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
describe('standalone fetch ref-update recovery', () => {
|
||||
it.each([
|
||||
CAS_DIAGNOSTIC,
|
||||
"error: cannot lock ref 'refs/remotes/upstream/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
])('recovers a failed fetch after %s', async (diagnostic) => {
|
||||
answerWith(failure(diagnostic), 'fresh refs')
|
||||
const result = gitExecFileAsync(['-c', 'gc.auto=0', 'fetch', 'origin'], options)
|
||||
await vi.runAllTimersAsync()
|
||||
await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' })
|
||||
})
|
||||
|
||||
it('waits for the failed foreground child to exit before replaying fetch', async () => {
|
||||
let exited = false
|
||||
capture.mockImplementationOnce((_command, _args, captureOptions) => {
|
||||
setTimeout(() => {
|
||||
exited = true
|
||||
captureOptions.onChildTerminated()
|
||||
}, 1_000)
|
||||
return Promise.reject(failure(CAS_DIAGNOSTIC))
|
||||
})
|
||||
capture.mockImplementation((_command, _args, captureOptions) => {
|
||||
captureOptions.onChildTerminated()
|
||||
return exited
|
||||
? Promise.resolve({ stdout: 'fresh refs', stderr: '' })
|
||||
: Promise.reject(new Error('The previous writer is still running'))
|
||||
})
|
||||
const result = gitExecFileAsync(['fetch', 'origin'], options)
|
||||
await vi.runAllTimersAsync()
|
||||
await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' })
|
||||
})
|
||||
|
||||
it('retains the original failure when ref contention persists', async () => {
|
||||
const original = failure(CAS_DIAGNOSTIC)
|
||||
answerWith(original)
|
||||
const result = gitExecFileAsync(['fetch', 'origin'], options)
|
||||
const rejection = expect(result).rejects.toMatchObject({
|
||||
cause: original,
|
||||
message: expect.stringContaining(
|
||||
'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided'
|
||||
)
|
||||
})
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
})
|
||||
|
||||
it.each([
|
||||
failure('fatal: Authentication failed'),
|
||||
failure(`${CAS_DIAGNOSTIC}\nfatal: Could not resolve host: example.invalid`),
|
||||
failure(`${CAS_DIAGNOSTIC}\nremote: error: access denied`),
|
||||
failure(`${CAS_DIAGNOSTIC}\nremote: ERROR access denied`),
|
||||
failure(`${CAS_DIAGNOSTIC}\nfatal: bad object HEAD`),
|
||||
failure(
|
||||
"error: cannot lock ref 'refs/heads/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
),
|
||||
failure("fatal: Unable to create '.git/refs/remotes/origin/main.lock': File exists"),
|
||||
failure(CAS_DIAGNOSTIC, { killed: true, signal: 'SIGTERM' }),
|
||||
failure(CAS_DIAGNOSTIC, { name: 'GitCommandTimeoutError' }),
|
||||
failure(CAS_DIAGNOSTIC, { outputTruncated: true }),
|
||||
Object.assign(new Error(CAS_DIAGNOSTIC), { stdout: CAS_DIAGNOSTIC })
|
||||
])('preserves a non-retriable failure: %s', async (original) => {
|
||||
answerWith(original, 'failure was hidden')
|
||||
const result = gitExecFileAsync(['fetch', 'origin'], options)
|
||||
const rejection = expect(result).rejects.toBe(original)
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
})
|
||||
|
||||
it('cancels the backoff instead of turning a cancelled fetch into success', async () => {
|
||||
const controller = new AbortController()
|
||||
answerWith(failure(CAS_DIAGNOSTIC), 'failure was hidden')
|
||||
const result = gitExecFileAsync(['fetch', 'origin'], {
|
||||
...options,
|
||||
signal: controller.signal,
|
||||
observeChildTermination: (terminated) => {
|
||||
void terminated.then(() => setTimeout(() => controller.abort(), 10))
|
||||
}
|
||||
})
|
||||
const rejection = expect(result).rejects.toMatchObject({ name: 'AbortError' })
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
})
|
||||
|
||||
it('surfaces a different failure from a later fetch attempt unchanged', async () => {
|
||||
const terminal = failure('fatal: Authentication failed')
|
||||
answerWith(failure(CAS_DIAGNOSTIC), terminal, 'failure was hidden')
|
||||
const result = gitExecFileAsync(['fetch', 'origin'], options)
|
||||
const rejection = expect(result).rejects.toBe(terminal)
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
})
|
||||
|
||||
it('preserves pull failures even when stderr resembles a failed fetch', async () => {
|
||||
const original = failure(CAS_DIAGNOSTIC)
|
||||
capture.mockImplementationOnce((_command, _args, captureOptions) => {
|
||||
captureOptions.onChildTerminated()
|
||||
return Promise.reject(original)
|
||||
})
|
||||
answerWith('Already up to date')
|
||||
const result = gitExecFileAsync(['pull', '--ff-only'], options)
|
||||
const rejection = expect(result).rejects.toBe(original)
|
||||
await vi.runAllTimersAsync()
|
||||
await rejection
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,76 @@
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../abort-signal-reason'
|
||||
import { readGitCommandFailureStderr } from '../commands/git-command-failure-text'
|
||||
import { findGitSubcommandIndex } from '../commands/git-command-classification'
|
||||
import { isExecKilledError, stripCredentialsFromMessage } from '../remotes/git-remote-error'
|
||||
|
||||
const REMOTE_REF_CAS_ERROR =
|
||||
/^(?:error|fatal): (?:fetching ref refs\/remotes\/[^\s]+ failed: incorrect old value provided|cannot lock ref 'refs\/remotes\/[^'\s]+': is at (?:[0-9a-f]{40}|[0-9a-f]{64}) but expected (?:[0-9a-f]{40}|[0-9a-f]{64}))$/i
|
||||
const OTHER_GIT_ERROR = /\b(?:error|fatal):|^remote:.*\b(?:error|fatal)\b/i
|
||||
|
||||
function remoteRefRaceStderr(error: unknown): string | null {
|
||||
if (!(error instanceof Error) || isExecKilledError(error)) {
|
||||
return null
|
||||
}
|
||||
if (
|
||||
('timedOut' in error && error.timedOut === true) ||
|
||||
('outputTruncated' in error && error.outputTruncated === true) ||
|
||||
error.name === 'AbortError' ||
|
||||
error.name === 'TimeoutError' ||
|
||||
error.name === 'GitCommandTimeoutError' ||
|
||||
('code' in error &&
|
||||
(error.code === 'ABORT_ERR' ||
|
||||
error.code === 'ETIMEDOUT' ||
|
||||
error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER'))
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const stderr = readGitCommandFailureStderr(error)
|
||||
if (!stderr) {
|
||||
return null
|
||||
}
|
||||
let matched = false
|
||||
for (const line of stderr.split(/[\r\n]+/)) {
|
||||
if (REMOTE_REF_CAS_ERROR.test(line.trim())) {
|
||||
matched = true
|
||||
} else if (OTHER_GIT_ERROR.test(line.trim())) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
return matched ? stderr : null
|
||||
}
|
||||
|
||||
/** Only standalone fetches may be replayed; pull can fail after changing the worktree. */
|
||||
export async function retryGitFetchRefRace<T>(
|
||||
args: readonly string[],
|
||||
initialError: unknown,
|
||||
attempt: () => Promise<T>,
|
||||
terminated: () => Promise<void> | null,
|
||||
signal?: AbortSignal
|
||||
): Promise<T> {
|
||||
if (args[findGitSubcommandIndex(args)] !== 'fetch') {
|
||||
throw initialError
|
||||
}
|
||||
let error = initialError
|
||||
for (let retry = 0; ; retry += 1) {
|
||||
const stderr = remoteRefRaceStderr(error)
|
||||
if (stderr === null) {
|
||||
throw error
|
||||
}
|
||||
throwIfSignalAborted(signal)
|
||||
if (retry === 2) {
|
||||
const detail = stripCredentialsFromMessage(stderr.trim())
|
||||
throw new Error(`${detail}\nAnother Git process updated remote references. Try again.`, {
|
||||
cause: initialError
|
||||
})
|
||||
}
|
||||
await waitForPromiseWithSignal(terminated() ?? Promise.resolve(), signal)
|
||||
await delay(50 + retry * 100 + Math.floor(Math.random() * 50), undefined, { signal })
|
||||
throwIfSignalAborted(signal)
|
||||
try {
|
||||
return await attempt()
|
||||
} catch (nextError) {
|
||||
error = nextError
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user