mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 03:15:01 +00:00
perf(runtime): cache OMP and Grok model discovery (#38977)
* perf(runtime): cache OMP and Grok model discovery * perf(runtime): share a slow model probe for its whole timeout * refactor(runtime): key and probe model discovery from one input
This commit is contained in:
@@ -15,7 +15,6 @@ import { prepareLocalCommitMessageAgentEnv } from '../../../text-generation/comm
|
||||
import {
|
||||
cancelGenerateCommitMessageLocal,
|
||||
cancelGeneratePullRequestFieldsLocal,
|
||||
discoverCommitMessageModelsLocal,
|
||||
generateCommitMessageFromContext,
|
||||
generatePullRequestFieldsFromContext,
|
||||
resolveCommitMessageSettings
|
||||
@@ -40,6 +39,7 @@ import {
|
||||
pullRequestDraftGitExec,
|
||||
type RuntimeCommitMessageSettingsOverride
|
||||
} from './runtime-git-generation-context'
|
||||
import { discoverModelsWithHostCache } from './runtime-model-discovery-cache'
|
||||
|
||||
export class RuntimeGitGenerationCommands {
|
||||
constructor(private readonly host: RuntimeGitCommandHost) {}
|
||||
@@ -257,12 +257,12 @@ export class RuntimeGitGenerationCommands {
|
||||
if (!localEnv.ok) {
|
||||
return { success: false, error: localEnv.error }
|
||||
}
|
||||
const localOptions = localGitOptionsForTarget(target)
|
||||
return localOptions.wslDistro
|
||||
? discoverCommitMessageModelsLocal(typedAgentId, localEnv.env, agentCommandOverride, {
|
||||
cwd: target.worktree.path,
|
||||
wslDistro: localOptions.wslDistro
|
||||
})
|
||||
: discoverCommitMessageModelsLocal(typedAgentId, localEnv.env, agentCommandOverride)
|
||||
const { wslDistro } = localGitOptionsForTarget(target)
|
||||
return discoverModelsWithHostCache({
|
||||
agentId: typedAgentId,
|
||||
agentCommandOverride,
|
||||
env: localEnv.env,
|
||||
wsl: wslDistro ? { cwd: target.worktree.path, wslDistro } : null
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import type { TuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent'
|
||||
import {
|
||||
InFlightPromiseDedupe,
|
||||
stableInFlightKey
|
||||
} from '../../../../shared/in-flight-promise-dedupe'
|
||||
import { setBoundedScopeCacheEntry } from '../../../../shared/native-chat/native-chat-scope-cache'
|
||||
import { PRIMARY_AGENT_DIR_ENV_BY_KIND } from '../../../../shared/pi-agent-kind'
|
||||
import { onAgentsRefreshedOnHost } from '../../../preflight/workspace-agent-detection'
|
||||
import { discoverCommitMessageModelsLocal } from '../../../text-generation/commit-message-text-generation'
|
||||
import { SOURCE_CONTROL_GENERATION_TIMEOUT_MS } from '../../../text-generation/source-control-generation-limits'
|
||||
import type { DiscoverCommitMessageModelsResult } from '../../../text-generation/source-control-text-generation-types'
|
||||
|
||||
// Phones list these on every chat open; other agents' lists have their own catalog owners.
|
||||
// Each maps to the config-dir var that picks its login (mirrors prepareShellConfigDirEnv).
|
||||
const ACCOUNT_DIR_VAR: Partial<Record<TuiAgent, string>> = {
|
||||
omp: PRIMARY_AGENT_DIR_ENV_BY_KIND.omp,
|
||||
grok: 'GROK_HOME'
|
||||
}
|
||||
const TTL_MS = 10 * 60_000
|
||||
const MAX_ENTRIES = 32
|
||||
|
||||
const cache = new Map<string, { result: DiscoverCommitMessageModelsResult; expiresAt: number }>()
|
||||
// Share a probe for its whole timeout, not the dedupe's 30 s default.
|
||||
const inFlight = new InFlightPromiseDedupe<DiscoverCommitMessageModelsResult>(
|
||||
SOURCE_CONTROL_GENERATION_TIMEOUT_MS + 5_000
|
||||
)
|
||||
let generation = 0
|
||||
|
||||
// An agent refresh is how a user says the CLI or its login changed.
|
||||
onAgentsRefreshedOnHost(() => {
|
||||
generation += 1
|
||||
cache.clear()
|
||||
inFlight.clear()
|
||||
})
|
||||
|
||||
export type ModelDiscoveryInput = {
|
||||
agentId: TuiAgent
|
||||
agentCommandOverride: string | undefined
|
||||
env: NodeJS.ProcessEnv | undefined
|
||||
wsl: { cwd: string; wslDistro: string } | null
|
||||
}
|
||||
|
||||
/** Probes the agent CLI; a probe-confirmed OMP or Grok list is reused for 10 min. */
|
||||
export function discoverModelsWithHostCache(
|
||||
input: ModelDiscoveryInput
|
||||
): Promise<DiscoverCommitMessageModelsResult> {
|
||||
const probe = (): Promise<DiscoverCommitMessageModelsResult> =>
|
||||
discoverCommitMessageModelsLocal(
|
||||
input.agentId,
|
||||
input.env,
|
||||
input.agentCommandOverride,
|
||||
input.wsl ?? undefined
|
||||
)
|
||||
const accountDirVar = ACCOUNT_DIR_VAR[input.agentId]
|
||||
if (!accountDirVar) {
|
||||
return probe()
|
||||
}
|
||||
// Not the whole env: the login-shell capture differs on every read (e.g. STARSHIP_SESSION_KEY).
|
||||
const key = stableInFlightKey([
|
||||
input.agentId,
|
||||
input.agentCommandOverride ?? null,
|
||||
input.wsl,
|
||||
input.env?.[accountDirVar] ?? null
|
||||
])
|
||||
const cached = cache.get(key)
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
return Promise.resolve(cached.result)
|
||||
}
|
||||
return inFlight.run(key, async () => {
|
||||
const startedIn = generation
|
||||
const result = await probe()
|
||||
// A refresh during the probe makes its answer stale.
|
||||
if (result.success && result.catalogOrigin === 'probe' && startedIn === generation) {
|
||||
setBoundedScopeCacheEntry(cache, key, { result, expiresAt: Date.now() + TTL_MS }, MAX_ENTRIES)
|
||||
}
|
||||
return result
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user