diff --git a/src/main/git/command-runner/git-exec-file.ts b/src/main/git/command-runner/git-exec-file.ts index eb7865fac23..0436137f58e 100644 --- a/src/main/git/command-runner/git-exec-file.ts +++ b/src/main/git/command-runner/git-exec-file.ts @@ -3,6 +3,7 @@ import { resolveGitFetchHeadCommand, runWithGitFetchHeadLock } from '../../../shared/git/fetch/git-fetch-head-lock' +import { retryGitFetchRefRace } from '../../../shared/git/fetch/git-fetch-ref-race' import { pendingWslDriveCheckoutGitRouting } from './host-routing/wsl-drive-checkout-git-routing' import type { ResolvedCommand } from './host-routing/wsl-command-resolution' import { annotateWslHostFailure, isTransientWslFailure } from './host-routing/wsl-host-failure' @@ -163,6 +164,14 @@ async function gitExecFileAsyncUnlocked( throw createAbortError() } return await runCapturedCommand() + } catch (error) { + return await retryGitFetchRefRace( + args, + error, + runCapturedCommand, + () => terminationState.current, + options.signal + ) } finally { const termination = terminationState.current if (termination) { diff --git a/src/main/git/command-runner/git-fetch-ref-race.test.ts b/src/main/git/command-runner/git-fetch-ref-race.test.ts new file mode 100644 index 00000000000..c8499519f9e --- /dev/null +++ b/src/main/git/command-runner/git-fetch-ref-race.test.ts @@ -0,0 +1,143 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { capture } = vi.hoisted(() => ({ capture: vi.fn() })) +vi.mock('./process-capture/exec-file-capture', () => ({ + execFileCapture: capture, + execFileCaptureToTermination: capture +})) +vi.mock('../../../shared/git/fetch/git-fetch-head-lock', async (importOriginal) => ({ + ...(await importOriginal>()), + runWithGitFetchHeadLock: (_cwd: string, _signal: AbortSignal | undefined, run: () => unknown) => + run() +})) + +import { gitExecFileAsync } from './git-exec-file' + +const CAS_DIAGNOSTIC = + 'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided' +const options = { cwd: '/repo', env: { GIT_SSH_COMMAND: 'ssh' } } + +function failure(stderr: string, extra: Record = {}): Error { + return Object.assign(new Error(stderr), { code: 1, stderr, ...extra }) +} + +function answerWith(...outcomes: (Error | string)[]): void { + capture.mockImplementation((_command, _args, captureOptions) => { + captureOptions.onChildTerminated() + const outcome = outcomes.length > 1 ? outcomes.shift()! : outcomes[0] + return outcome instanceof Error + ? Promise.reject(outcome) + : Promise.resolve({ stdout: outcome, stderr: '' }) + }) +} + +beforeEach(() => { + vi.useFakeTimers() + capture.mockReset() +}) +afterEach(() => vi.useRealTimers()) + +describe('standalone fetch ref-update recovery', () => { + it.each([ + CAS_DIAGNOSTIC, + "error: cannot lock ref 'refs/remotes/upstream/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ])('recovers a failed fetch after %s', async (diagnostic) => { + answerWith(failure(diagnostic), 'fresh refs') + const result = gitExecFileAsync(['-c', 'gc.auto=0', 'fetch', 'origin'], options) + await vi.runAllTimersAsync() + await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' }) + }) + + it('waits for the failed foreground child to exit before replaying fetch', async () => { + let exited = false + capture.mockImplementationOnce((_command, _args, captureOptions) => { + setTimeout(() => { + exited = true + captureOptions.onChildTerminated() + }, 1_000) + return Promise.reject(failure(CAS_DIAGNOSTIC)) + }) + capture.mockImplementation((_command, _args, captureOptions) => { + captureOptions.onChildTerminated() + return exited + ? Promise.resolve({ stdout: 'fresh refs', stderr: '' }) + : Promise.reject(new Error('The previous writer is still running')) + }) + const result = gitExecFileAsync(['fetch', 'origin'], options) + await vi.runAllTimersAsync() + await expect(result).resolves.toEqual({ stdout: 'fresh refs', stderr: '' }) + }) + + it('retains the original failure when ref contention persists', async () => { + const original = failure(CAS_DIAGNOSTIC) + answerWith(original) + const result = gitExecFileAsync(['fetch', 'origin'], options) + const rejection = expect(result).rejects.toMatchObject({ + cause: original, + message: expect.stringContaining( + 'error: fetching ref refs/remotes/origin/main failed: incorrect old value provided' + ) + }) + await vi.runAllTimersAsync() + await rejection + }) + + it.each([ + failure('fatal: Authentication failed'), + failure(`${CAS_DIAGNOSTIC}\nfatal: Could not resolve host: example.invalid`), + failure(`${CAS_DIAGNOSTIC}\nremote: error: access denied`), + failure(`${CAS_DIAGNOSTIC}\nremote: ERROR access denied`), + failure(`${CAS_DIAGNOSTIC}\nfatal: bad object HEAD`), + failure( + "error: cannot lock ref 'refs/heads/main': is at aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa but expected bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ), + failure("fatal: Unable to create '.git/refs/remotes/origin/main.lock': File exists"), + failure(CAS_DIAGNOSTIC, { killed: true, signal: 'SIGTERM' }), + failure(CAS_DIAGNOSTIC, { name: 'GitCommandTimeoutError' }), + failure(CAS_DIAGNOSTIC, { outputTruncated: true }), + Object.assign(new Error(CAS_DIAGNOSTIC), { stdout: CAS_DIAGNOSTIC }) + ])('preserves a non-retriable failure: %s', async (original) => { + answerWith(original, 'failure was hidden') + const result = gitExecFileAsync(['fetch', 'origin'], options) + const rejection = expect(result).rejects.toBe(original) + await vi.runAllTimersAsync() + await rejection + }) + + it('cancels the backoff instead of turning a cancelled fetch into success', async () => { + const controller = new AbortController() + answerWith(failure(CAS_DIAGNOSTIC), 'failure was hidden') + const result = gitExecFileAsync(['fetch', 'origin'], { + ...options, + signal: controller.signal, + observeChildTermination: (terminated) => { + void terminated.then(() => setTimeout(() => controller.abort(), 10)) + } + }) + const rejection = expect(result).rejects.toMatchObject({ name: 'AbortError' }) + await vi.runAllTimersAsync() + await rejection + }) + + it('surfaces a different failure from a later fetch attempt unchanged', async () => { + const terminal = failure('fatal: Authentication failed') + answerWith(failure(CAS_DIAGNOSTIC), terminal, 'failure was hidden') + const result = gitExecFileAsync(['fetch', 'origin'], options) + const rejection = expect(result).rejects.toBe(terminal) + await vi.runAllTimersAsync() + await rejection + }) + + it('preserves pull failures even when stderr resembles a failed fetch', async () => { + const original = failure(CAS_DIAGNOSTIC) + capture.mockImplementationOnce((_command, _args, captureOptions) => { + captureOptions.onChildTerminated() + return Promise.reject(original) + }) + answerWith('Already up to date') + const result = gitExecFileAsync(['pull', '--ff-only'], options) + const rejection = expect(result).rejects.toBe(original) + await vi.runAllTimersAsync() + await rejection + }) +}) diff --git a/src/shared/git/fetch/git-fetch-ref-race.ts b/src/shared/git/fetch/git-fetch-ref-race.ts new file mode 100644 index 00000000000..d71f30f6cac --- /dev/null +++ b/src/shared/git/fetch/git-fetch-ref-race.ts @@ -0,0 +1,76 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { throwIfSignalAborted, waitForPromiseWithSignal } from '../../abort-signal-reason' +import { readGitCommandFailureStderr } from '../commands/git-command-failure-text' +import { findGitSubcommandIndex } from '../commands/git-command-classification' +import { isExecKilledError, stripCredentialsFromMessage } from '../remotes/git-remote-error' + +const REMOTE_REF_CAS_ERROR = + /^(?:error|fatal): (?:fetching ref refs\/remotes\/[^\s]+ failed: incorrect old value provided|cannot lock ref 'refs\/remotes\/[^'\s]+': is at (?:[0-9a-f]{40}|[0-9a-f]{64}) but expected (?:[0-9a-f]{40}|[0-9a-f]{64}))$/i +const OTHER_GIT_ERROR = /\b(?:error|fatal):|^remote:.*\b(?:error|fatal)\b/i + +function remoteRefRaceStderr(error: unknown): string | null { + if (!(error instanceof Error) || isExecKilledError(error)) { + return null + } + if ( + ('timedOut' in error && error.timedOut === true) || + ('outputTruncated' in error && error.outputTruncated === true) || + error.name === 'AbortError' || + error.name === 'TimeoutError' || + error.name === 'GitCommandTimeoutError' || + ('code' in error && + (error.code === 'ABORT_ERR' || + error.code === 'ETIMEDOUT' || + error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER')) + ) { + return null + } + const stderr = readGitCommandFailureStderr(error) + if (!stderr) { + return null + } + let matched = false + for (const line of stderr.split(/[\r\n]+/)) { + if (REMOTE_REF_CAS_ERROR.test(line.trim())) { + matched = true + } else if (OTHER_GIT_ERROR.test(line.trim())) { + return null + } + } + return matched ? stderr : null +} + +/** Only standalone fetches may be replayed; pull can fail after changing the worktree. */ +export async function retryGitFetchRefRace( + args: readonly string[], + initialError: unknown, + attempt: () => Promise, + terminated: () => Promise | null, + signal?: AbortSignal +): Promise { + if (args[findGitSubcommandIndex(args)] !== 'fetch') { + throw initialError + } + let error = initialError + for (let retry = 0; ; retry += 1) { + const stderr = remoteRefRaceStderr(error) + if (stderr === null) { + throw error + } + throwIfSignalAborted(signal) + if (retry === 2) { + const detail = stripCredentialsFromMessage(stderr.trim()) + throw new Error(`${detail}\nAnother Git process updated remote references. Try again.`, { + cause: initialError + }) + } + await waitForPromiseWithSignal(terminated() ?? Promise.resolve(), signal) + await delay(50 + retry * 100 + Math.floor(Math.random() * 50), undefined, { signal }) + throwIfSignalAborted(signal) + try { + return await attempt() + } catch (nextError) { + error = nextError + } + } +}