Tier GitHub PR lookup polling to prevent quota exhaustion (#12013)

* Tier GitHub PR lookup polling to prevent quota exhaustion

The selected worktree (O(1)) checks per-minute; card list (O(N)) per-15-minutes.
Introduce process-wide cache to collapse concurrent polling and gate lookups on
available rate-limit budget with exponential backoff on failure.

- Preserve last-known review during backoff
- Invalidate cache when Orca opens a PR
- Stop coordinator from double-charging

* Tier GitHub PR lookup polling to prevent quota exhaustion

- Return the latest reset time when both GitHub API buckets are rate-limited, preventing premature retries against still-blocked buckets.
- Serve the last known review on transient lookup failures, preventing reviews from blinking out on temporary errors.
- Discard in-flight lookups that predate an invalidation so stale answers cannot overwrite newly opened reviews.

* fix: give rate-limit reset tests unique titles

oxlint vitest/no-identical-title was failing static analysis because two
cases shared the same describe title.
This commit is contained in:
Jinjing
2026-08-01 15:50:54 -07:00
committed by GitHub
parent a7d769e13d
commit 2f104d8713
24 changed files with 1070 additions and 53 deletions
+68
View File
@@ -170,6 +170,7 @@ import {
getPRComments,
getPRForBranch,
getPRForBranchOutcome,
getGitHubPRLookupRateLimitBlock,
getRepoSlug,
getRepoUpstream,
getWorkItem,
@@ -4609,3 +4610,70 @@ describe('GitHub GraphQL rate-limit guard', () => {
expect(noteRateLimitSpendMock).not.toHaveBeenCalled()
})
})
describe('getGitHubPRLookupRateLimitBlock', () => {
beforeEach(() => {
execFileAsyncMock.mockReset()
ghExecFileAsyncMock.mockReset()
getOwnerRepoMock.mockReset()
getOwnerRepoMock.mockResolvedValue({ owner: 'acme', repo: 'widgets' })
getRemoteUrlForRepoMock.mockReset()
gitExecFileAsyncMock.mockReset()
getRateLimitMock.mockReset()
getRateLimitMock.mockResolvedValue(undefined)
rateLimitGuardMock.mockReset()
rateLimitGuardMock.mockReturnValue({ blocked: false })
noteRateLimitSpendMock.mockReset()
_resetOwnerRepoCache()
})
it('reports no block while every lookup bucket has budget', async () => {
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toBeNull()
expect(getRateLimitMock).toHaveBeenCalled()
})
it('reports a block when either lookup bucket is exhausted', async () => {
rateLimitGuardMock.mockImplementation(((bucket: string) =>
bucket === 'graphql'
? { blocked: true, remaining: 4, limit: 5000, resetAt: 1_800_000_000 }
: { blocked: false }) as () => RateLimitGuardResult)
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
resetAt: 1_800_000_000
})
})
it('reports the latest reset when both lookup buckets are exhausted', async () => {
rateLimitGuardMock.mockImplementation(((bucket: string) => ({
blocked: true,
remaining: 4,
limit: 5000,
// Why: core resets first, so returning it would retry into graphql's block.
resetAt: bucket === 'core' ? 1_800_000_000 : 1_800_003_600
})) as () => RateLimitGuardResult)
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
resetAt: 1_800_003_600
})
})
it('reports the later reset when graphql outlasts core', async () => {
// Retrying at the earlier reset would fail again on the bucket still blocked.
rateLimitGuardMock.mockImplementation(((bucket: string) => ({
blocked: true,
remaining: 0,
limit: 5000,
resetAt: bucket === 'graphql' ? 1_800_000_600 : 1_800_000_000
})) as () => RateLimitGuardResult)
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toEqual({
resetAt: 1_800_000_600
})
})
it('fails open when the exempt rate-limit probe itself fails', async () => {
getRateLimitMock.mockRejectedValue(new Error('probe offline'))
await expect(getGitHubPRLookupRateLimitBlock('/repo-root')).resolves.toBeNull()
})
})
+53
View File
@@ -195,6 +195,52 @@ async function assertRateLimitBudget(
}
}
// Why: a branch lookup prefers REST but can fall back to `gh pr list` and
// `gh pr view`, so both buckets are guarded and charged. Mirrors the PR refresh
// coordinator's own estimate.
const PR_BRANCH_LOOKUP_BUCKETS = ['core', 'graphql'] as const
/**
* Rate-limit floor for GitHub PR lookups that do not run through the PR refresh
* coordinator's queue (#11532).
*
* The coordinator guards and paces its own background refreshes, but
* `hostedReview:forBranch` polls the same lookup straight from the renderer.
* Ungated, the two paths together could spend the user's entire hourly quota —
* which is per user and shared with their own `gh` and CLI agents.
* Returns the reset time when the caller must not spend, else `null`.
*/
export async function getGitHubPRLookupRateLimitBlock(
repoPath: string,
connectionId?: string | null,
localGitOptions: LocalGitExecOptions = {}
): Promise<{ resetAt: number } | null> {
const executionOptions = ghRepoExecOptions(
githubRepoContext(repoPath, connectionId, localGitOptions)
)
// Why: identity resolution runs local git, which can fail for reasons that
// have nothing to do with the budget; let the lookup itself classify those.
const repository = await getOriginGitHubApiRepository(
repoPath,
connectionId,
executionOptions
).catch(() => null)
if (repository === null) {
return null
}
if (spendsSharedGitHubComQuota(repository, executionOptions)) {
// Why: the probe only warms the snapshot and is exempt from limits, so a
// failure must fail open rather than block the lookup (#7553).
await getRateLimit().catch(() => undefined)
}
// Why: retrying at the earlier reset would fail again on the bucket that has
// not reset yet, so the latest blocked reset is the only honest retry time.
const resets = PR_BRANCH_LOOKUP_BUCKETS.map((bucket) =>
repositoryRateLimitGuard(repository, bucket, executionOptions)
).flatMap((guard) => (guard.blocked ? [guard.resetAt] : []))
return resets.length > 0 ? { resetAt: Math.max(...resets) } : null
}
function prRefreshUpstreamError(
err: unknown
): Extract<PRRefreshOutcome, { kind: 'upstream-error' }> {
@@ -2956,6 +3002,13 @@ export async function getPRForBranchOutcome(
if (connectionId && candidates.length === 0) {
return { kind: 'no-pr', fetchedAt: Date.now() }
}
// Why (#11532): account every lookup, not just the coordinator's queue —
// `hostedReview:forBranch` reaches this directly from renderer polling and
// was spending the shared quota invisibly. headRepo is `origin`, the same
// identity the coordinator guards on.
for (const bucket of PR_BRANCH_LOOKUP_BUCKETS) {
noteRepositoryRateLimitSpend(headRepo ?? candidates[0], bucket, 1, ghOptions)
}
let data: PullRequestLookupData | null = null
let dataRepo: OwnerRepo | null = null
let dataHeadRepo: OwnerRepo | null = headRepo
@@ -32,6 +32,7 @@ const {
getRateLimitMock,
rateLimitGuardMock,
noteRateLimitSpendMock,
noteRepositoryRateLimitSpendMock,
ghRepoExecOptionsMock,
githubRepoContextMock,
getSshGitProviderMock,
@@ -52,6 +53,7 @@ const {
blocked: false
})),
noteRateLimitSpendMock: vi.fn(),
noteRepositoryRateLimitSpendMock: vi.fn(),
ghRepoExecOptionsMock: vi.fn((context) =>
context.connectionId
? {}
@@ -116,7 +118,8 @@ vi.mock('./local-git-config-signature', () => ({
vi.mock('./rate-limit', () => ({
getRateLimit: getRateLimitMock,
rateLimitGuard: rateLimitGuardMock,
noteRateLimitSpend: noteRateLimitSpendMock
noteRateLimitSpend: noteRateLimitSpendMock,
noteRepositoryRateLimitSpend: noteRepositoryRateLimitSpendMock
}))
import {
+3 -12
View File
@@ -649,18 +649,9 @@ describe('pr-refresh-coordinator', () => {
'graphql',
executionOptions
)
expect(noteRepositoryRateLimitSpendMock).toHaveBeenCalledWith(
testCase.repository,
'core',
1,
executionOptions
)
expect(noteRepositoryRateLimitSpendMock).toHaveBeenCalledWith(
testCase.repository,
'graphql',
1,
executionOptions
)
// Why (#11532): the lookup itself debits the snapshot now, so every caller
// is accounted for; the coordinator must not double-charge on top.
expect(noteRepositoryRateLimitSpendMock).not.toHaveBeenCalled()
expect(getPRForBranchOutcomeMock).toHaveBeenCalledTimes(1)
})
+8 -13
View File
@@ -11,12 +11,11 @@ import type {
import { getPRForBranchOutcome, type GitHubPRBranchLookupOptions } from './client'
import { getOriginGitHubApiRepository } from './github-api-repository'
import { ghRepoExecOptions, githubRepoContext } from './gh-utils'
import { getRateLimit, repositoryRateLimitGuard, spendsSharedGitHubComQuota } from './rate-limit'
import {
getRateLimit,
noteRepositoryRateLimitSpend,
repositoryRateLimitGuard,
spendsSharedGitHubComQuota
} from './rate-limit'
lookupBackoffDelayMs,
NO_REVIEW_REFRESH_INTERVAL_MS
} from '../source-control/hosted-review-refresh-pacing'
import { recordCoalescedCrashBreadcrumb } from '../crash-reporting/crash-breadcrumb-store'
import { sendToTrustedUIRenderer } from '../ipc/ui'
@@ -74,8 +73,6 @@ const BACKGROUND_BUDGET_WINDOW_MS = 5 * 60_000
const MIN_BACKGROUND_SPACING_MS = 10_000
const BACKGROUND_BUDGET_MAX = 20
const POST_PUSH_DELAY_MS = 2_500
const BACKOFF_BASE_MS = 60_000
const BACKOFF_MAX_MS = 15 * 60_000
const DIAGNOSTIC_BREADCRUMB_MIN_INTERVAL_MS = 30_000
const ACTIVE_BURST_WINDOW_MS = 30_000
const ACTIVE_BURST_MAX = 3
@@ -424,8 +421,7 @@ function removeQueuedAliasForInvalidCandidate(key: string, alias: GitHubPRRefres
*/
function nextVisibleErrorRetryAt(key: string): number {
const failures = (errorBackoff.get(key)?.failures ?? 0) + 1
const retryAt =
Date.now() + Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 4))
const retryAt = Date.now() + lookupBackoffDelayMs(failures)
errorBackoff.set(key, { failures, retryAt })
return retryAt
}
@@ -509,7 +505,7 @@ function refreshIntervalForCandidate(candidate: GitHubPRRefreshCandidate): numbe
return 30 * 60_000
}
if (candidate.cachedHasPR === false) {
return 15 * 60_000
return NO_REVIEW_REFRESH_INTERVAL_MS
}
if (
candidate.cachedHasPR === true &&
@@ -776,9 +772,8 @@ async function drainQueue(): Promise<void> {
// Why: tab/worktree churn can enqueue many distinct active refreshes that each probe local Git.
noteActiveStart(next)
}
for (const bucket of buckets) {
noteRepositoryRateLimitSpend(repository, bucket, 1, executionOptions)
}
// Why (#11532): the lookup itself now debits the snapshot, so every
// caller is accounted for; debiting here too would double-count.
}
const outcome = await getPRForBranchOutcome(