Settle resumed session reads and cancel pending discovery

Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
Brennan Benson
2026-11-10 12:52:49 -08:00
co-authored by Jinjing
parent f93c13f431
commit 3fc8adbc0f
13 changed files with 233 additions and 28 deletions
@@ -35,6 +35,19 @@ function groupKey(candidate: SessionFileCandidate, root: string): string {
/** Ordinary paths own one record; only competing aliases allocate a group and selected set. */
export class SessionSearchCandidateInventory extends SessionSearchCandidatePublications {
private readonly sourceOrders = new Map<string, ReadonlyMap<string, number>>()
rootFor(candidate: SessionFileCandidate): string | undefined {
const owner = this.byPath.get(candidate.file.path)
for (const key of typeof owner === 'string' ? [owner] : (owner ?? [])) {
const row = rows(this.groups.get(key)).find(
(row) =>
row.candidate.agent === candidate.agent && row.candidate.file.path === candidate.file.path
)
if (row) {
return row.root
}
}
return undefined
}
has(path: string, agent?: string | null): boolean {
const owner = this.byPath.get(path)
if (!agent) {
@@ -51,13 +64,14 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli
removed: ReadonlySet<string>,
options: AiVaultScanOptions,
invalidated: ReadonlySet<string> = new Set(),
removedCandidates: ReadonlyMap<string, ReadonlySet<string>> = new Map()
removedCandidates: ReadonlyMap<string, ReadonlySet<string>> = new Map(),
partial = false
): Promise<{ candidates: SessionFileCandidate[]; suppressed: Set<string>; forced: Set<string> }> {
const scopes = new Set(
discoveries.map((discovery) => `${discovery.agent}:${discovery.rootDir}`)
)
for (const scope of this.sourceOrders.keys()) {
if (!scopes.has(scope)) {
if (!partial && !scopes.has(scope)) {
this.sourceOrders.delete(scope)
}
}
@@ -90,6 +90,21 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
return map
}
async observeForRead(agent: string, rootDir: string, path: string) {
const root = this.roots.get(`${agent}:${rootDir}`)
// Container rows retain their source-owned admission and are never filesystem paths.
if (!root) {
return null
}
const change = await root.inventory.observeForRead(path, root.args, this.recentLimit)
if (change) {
for (const observer of root.dependencies.values()) {
observer.addFile(path)
}
}
return change
}
async files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery> {
const key = `${args.agent}:${args.rootDir}`
this.seen.add(key)
@@ -14,7 +14,7 @@ import {
import { fileIdentity } from './session-search-file-cursor'
import { sessionSearchReadDecision } from './session-search-read-decision'
import type { SessionSearchIndexStore } from './session-search-store'
import type { SessionSearchWorkQueue } from './session-search-work-queue'
import type { SessionSearchWork, SessionSearchWorkQueue } from './session-search-work-queue'
export async function runSessionSearchIndexPass(
store: SessionSearchIndexStore,
@@ -23,6 +23,7 @@ export async function runSessionSearchIndexPass(
signal?: AbortSignal
overdue?: () => boolean
publications?: SessionSearchCandidateInventory
admit?: (work: SessionSearchWork) => Promise<SessionSearchWork | undefined>
}
): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> {
const stats = createSessionParseStats()
@@ -36,10 +37,17 @@ export async function runSessionSearchIndexPass(
outOfTime = true
break
}
const work = queue.next()
let work = queue.next()
if (!work) {
break
}
if (options.admit) {
work = await options.admit(work)
if (!work) {
consumed++
continue
}
}
const { candidate } = work
const path = candidate.file.path
let row = store.fileRow(path)
@@ -120,6 +128,7 @@ export async function runSessionSearchIndexPass(
queue.acknowledge(work, false)
continue
}
const failuresBeforeRead = row?.failCount ?? 0
try {
await parseAgentSessionFileCached(
candidate,
@@ -132,12 +141,19 @@ export async function runSessionSearchIndexPass(
await store.flush?.()
} catch (error) {
throwIfAiVaultScanCancelled(options.signal)
if (error instanceof Error && error.name === 'AbortError') {
throw error
}
console.warn(
'[ai-vault-search] indexing skipped',
candidate.agent,
error instanceof Error ? error.name : 'ParseError'
)
store.setFileState(path, 'failed', candidate.file.mtimeMs)
// The attached transcript consumer may already have settled this failed attempt.
await store.flush?.()
if ((store.fileRow(path)?.failCount ?? 0) <= failuresBeforeRead) {
store.setFileState(path, 'failed', candidate.file.mtimeMs)
}
await store.flush?.()
}
const committed = store.fileRow(path)
@@ -127,6 +127,7 @@ export async function renameReplaceTranscript(
await rename(replacement, path)
const later = new Date(before.mtimeMs + 5_000)
await utimes(path, later, later)
notifySessionSearchFileChange(path)
}
export async function writeMessageGraphTranscript(
@@ -1,5 +1,8 @@
import './session-search-worker.test-fixture'
import { notifySessionSearchRootChanges } from './session-search-watcher.test-fixture'
import {
notifySessionSearchFileChange,
notifySessionSearchRootChanges
} from './session-search-watcher.test-fixture'
import { mkdirSync, rmSync } from 'node:fs'
import { appendFile, chmod, mkdir, rm, stat, utimes } from 'node:fs/promises'
import { join } from 'node:path'
@@ -152,6 +155,7 @@ it('reflects a grown transcript within one reconcile interval', async () => {
path,
`${claudeLines(['quarantine the leaking pty'], SESSION_ID, 10).join('\n')}\n`
)
notifySessionSearchFileChange(path)
await nextCycle()
expect(sessionsMatching('quarantine')).toEqual([SESSION_ID])
@@ -282,6 +286,7 @@ it.skipIf(!CAN_DENY_READ)('stops re-reading a transcript it cannot read', async
await chmod(path, 0o644)
const later = new Date(Date.now() + 60_000)
await utimes(path, later, later)
notifySessionSearchFileChange(path)
await nextCycle()
expect(sessionsMatching('mode')).toEqual([SESSION_ID])
@@ -1,4 +1,5 @@
import { systemSessionSearchClock, type SessionSearchClock } from './session-search-clock'
import { waitForAiVaultScan } from '../ai-vault/ai-vault-scan-cancellation'
import {
DEFAULT_SESSION_SEARCH_FULL_SWEEP_EVERY_CYCLES,
DEFAULT_SESSION_SEARCH_PASS_DEADLINE_FRACTION,
@@ -246,7 +247,7 @@ export class SessionSearchIndexer {
this.sweepInProgress ||= full
try {
if (maintenance && this.options.resolveRoots) {
const roots = await this.options.resolveRoots(signal)
const roots = await waitForAiVaultScan(this.options.resolveRoots(signal), signal)
if (signal.aborted) {
return
}
@@ -147,7 +147,46 @@ export async function runSessionSearchPass(
const read = await runSessionSearchIndexPass(store, state.queue, {
signal,
overdue: readOverdue,
publications: state.candidates
publications: state.candidates,
admit: async (work) => {
const { candidate } = work
const path = candidate.file.path
const rootDir = state.candidates.rootFor(candidate)
if (rootDir === undefined) {
return undefined
}
const observation = await state.discovery.observeForRead(candidate.agent, rootDir, path)
if (observation === null) {
return work
}
if (!observation) {
// Observation debt owns retry/holdout; accepted facts cannot certify a refused input.
state.queue.acknowledge(work, false)
return undefined
}
if (!observation.changed) {
return work
}
const refreshed = await state.candidates.update(
[{ agent: candidate.agent, rootDir, files: [observation.file] }],
new Set(),
{ ...args.roots, signal },
observation.force ? new Set([path]) : new Set(),
new Map(),
true
)
for (const suppressed of refreshed.suppressed) {
state.queue.remove(suppressed)
}
for (const selected of refreshed.candidates) {
const force = observation.force || refreshed.forced.has(selected.file.path)
if (force) {
state.candidates.requirePublication(selected.file.path)
}
state.queue.enqueue(selected, store, force, !args.full)
}
return state.queue.get(path)
}
})
await state.candidates.reconcilePublications(store, readOverdue, signal, (path) =>
state.queue.hasPublicationDebt(path)
@@ -26,36 +26,33 @@ afterEach(async () => {
await harness.cleanup()
})
it('refreshes roots on scheduled full sweeps and reuses them on recent cycles', async () => {
it('finds a changed session location on maintenance after ordinary cycles reuse the old location', async () => {
const clock = new FakeSessionSearchClock()
let roots = harness.roots
const resolveRoots = vi.fn(async () => roots)
const resolveRoots = async () => roots
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots,
resolveRoots,
historyDays: null,
clock,
fullSweepEveryCycles: 1
fullSweepEveryCycles: 2
})
await indexer.start()
expect(resolveRoots).toHaveBeenCalledTimes(1)
const newRoot = join(harness.root, 'late')
const id = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
await writeClaudeTranscript(join(newRoot, 'project', `${id}.jsonl`), ['a late conversation'], id)
roots = { ...roots, claudeProjectsDir: newRoot }
clock.advance(20_000)
await indexer.settled()
expect(resolveRoots).toHaveBeenCalledTimes(1)
expect(indexer.status().filesIndexed).toBe(0)
clock.advance(20_000)
await indexer.settled()
expect(resolveRoots).toHaveBeenCalledTimes(2)
expect(indexer.status().filesIndexed).toBe(1)
clock.advance(20_000)
await indexer.settled()
expect(resolveRoots).toHaveBeenCalledTimes(2)
expect(indexer.status().filesIndexed).toBe(1)
expect(
harness.read((db) =>
db.prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'late'").get()
)
).toEqual({ n: 2 })
})
it('does not access a closed store when pending discovery completes', async () => {
@@ -80,10 +77,15 @@ it('does not access a closed store when pending discovery completes', async () =
it('retries discovery after failure without silently sweeping stale roots', async () => {
const errors = vi.fn()
const resolveRoots = vi
.fn()
.mockRejectedValueOnce(new Error('unavailable'))
.mockResolvedValue(harness.roots)
let unavailable = true
const path = join(harness.claudeProjectDir, 'recovery.jsonl')
await writeClaudeTranscript(path, ['recovered conversation'], 'recovery')
const resolveRoots = async () => {
if (unavailable) {
throw new Error('unavailable')
}
return harness.roots
}
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
@@ -94,7 +96,14 @@ it('retries discovery after failure without silently sweeping stale roots', asyn
await indexer.start()
expect(errors).toHaveBeenCalledTimes(1)
expect(indexer.status().lastSweepCompletedAt).toBeNull()
expect(indexer.status().filesIndexed).toBe(0)
unavailable = false
await indexer.reconcile()
expect(resolveRoots).toHaveBeenCalledTimes(2)
expect(indexer.status().lastSweepCompletedAt).not.toBeNull()
expect(
harness.read((db) =>
db
.prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'recovered'")
.get()
)
).toEqual({ n: 2 })
})
@@ -4,6 +4,7 @@ import { WatcherProcessSupervisor } from '../ipc/filesystem/watcher-process/parc
import type { WatcherProcessSubscription } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription'
import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol'
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
import { RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS } from '../../shared/file-watch/runtime-file-watch-limits'
export type SessionSearchRootWatch = {
identity: string
subscription?: WatcherProcessSubscription
@@ -127,6 +128,7 @@ export class SessionSearchRootWatchers {
{ mode },
{
signal: this.owner.signal,
subscribeTimeoutMs: RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS,
onInterruption: () => {
watch.reconcile = true
},
@@ -0,0 +1,85 @@
import './session-search-worker.test-fixture'
import { appendFile } from 'node:fs/promises'
import { join } from 'node:path'
import { expect, it, vi } from 'vitest'
import { FakeWatcherChild } from '../ipc/filesystem/watcher-process/parcel-watcher-process-test-child'
import { SessionSearchIndexer } from './session-search-indexer'
import {
claudeLines,
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
writeClaudeTranscript
} from './session-search-indexer-test-fixture'
const { fork } = vi.hoisted(() => ({ fork: vi.fn() }))
vi.mock('node:child_process', async (original) => ({ ...(await original<object>()), fork }))
vi.mock('./session-search-filesystem', () => ({
sessionSearchRequiresPolling: async () => false
}))
it('keeps indexing after a native crawl never becomes ready and retires its child', async () => {
const harness = await openSessionSearchIndexerHarness('ss-pending-watch')
const path = join(harness.claudeProjectDir, 'pending.jsonl')
const clock = new FakeSessionSearchClock()
let crawling = false
const children: FakeWatcherChild[] = []
fork.mockImplementation(() => {
const child = new FakeWatcherChild()
children.push(child)
child.send.mockImplementation((message) => {
child.sent.push(message)
queueMicrotask(() => {
if (message.op === 'subscribe') {
crawling = true
child.emit('message', { op: 'subscribe-started', id: message.id })
} else if (message.op === 'cancel-subscribe') {
child.emit('message', { op: 'cancel-requires-restart', id: message.id })
}
})
return true
})
child.kill.mockImplementation(() => {
child.connected = false
queueMicrotask(() => child.emit('exit', 0, null))
return true
})
return child
})
vi.stubEnv('VITEST', '')
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
watcherEntryPath: import.meta.filename
})
try {
await writeClaudeTranscript(path, ['beforepressure'], 'pending')
const start = subject.start()
await vi.waitFor(() => expect(crawling).toBe(true))
await vi.advanceTimersByTimeAsync(60_000)
await start
expect(subject.status().filesIndexed).toBe(1)
expect(children.every((child) => !child.connected)).toBe(true)
await appendFile(path, `${claudeLines(['afterpressure'], 'pending', 10).join('\n')}\n`)
clock.advance(20_000)
await subject.settled()
expect(
harness.read((db) =>
db
.prepare(
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'afterpressure'"
)
.get()
)
).toEqual({ n: 2 })
} finally {
await subject.close()
vi.useRealTimers()
vi.unstubAllEnvs()
fork.mockReset()
await harness.cleanup()
}
})
@@ -73,6 +73,10 @@ export class SessionSearchWorkQueue {
return preferred.first ?? (preferred === this.changed ? this.backlog : this.changed).first
}
get(path: string): SessionSearchWork | undefined {
return this.changed.get(path) ?? this.backlog.get(path)
}
acknowledge(work: SessionSearchWork, retry: boolean): void {
const path = work.candidate.file.path
const queue = this.changed.has(path) ? this.changed : this.backlog
@@ -23,7 +23,13 @@ export class SessionInventoryFiles {
private readonly scheduleParent: (path: string) => void,
private readonly forgetDirty: (path: string) => void
) {}
async observe(path: string, args: DiscoverSessionFilesArgs, force: boolean, limit: number) {
async observe(
path: string,
args: DiscoverSessionFilesArgs,
force: boolean,
limit: number,
readOwed = false
) {
if (!this.failures.mayAttempt(path)) {
return
}
@@ -46,7 +52,7 @@ export class SessionInventoryFiles {
}
this.failures.fail(
path,
force,
force || readOwed,
observation.missing ? undefined : { agent: args.agent, ...observation.issue }
)
// Only apparent absence needs a parent membership proof.
@@ -65,7 +71,7 @@ export class SessionInventoryFiles {
codexHome: null
})
if (!metadataOnly && observation.file.sidecar === 'unknown') {
this.failures.fail(path, force)
this.failures.fail(path, force || readOwed)
file.sidecar = previous?.sidecar ?? 'unknown'
} else {
this.failures.delete(path)
@@ -117,6 +123,7 @@ export class SessionInventoryFiles {
})
}
retainRecentSessionFile(this.recent, file, limit)
return !metadataOnly && observation.file.sidecar === 'unknown' ? undefined : file
}
forget(path: string): void {
@@ -127,6 +127,13 @@ export class SessionFileInventory {
}
}
async observeForRead(path: string, args: DiscoverSessionFilesArgs, limit: number) {
const file = await this.observations.observe(path, args, false, limit, true)
const change = this.observations.changes.get(path)
this.observations.changes.delete(path)
return file ? { file, force: change?.force ?? false, changed: change !== undefined } : undefined
}
async drain(
args: DiscoverSessionFilesArgs,
overdue: () => boolean,