From 3fc8adbc0f2099e4d5f469a17e52fa0e9adef3cd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:52:49 -0800 Subject: [PATCH] Settle resumed session reads and cancel pending discovery Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com> --- .../session-search-candidate-inventory.ts | 18 +++- .../session-search-discovery-inventory.ts | 15 ++++ .../session-search-index-pass.ts | 22 ++++- .../session-search-indexer-test-fixture.ts | 1 + .../session-search-indexer.test.ts | 7 +- .../ai-vault-search/session-search-indexer.ts | 3 +- .../ai-vault-search/session-search-pass.ts | 41 ++++++++- .../session-search-root-refresh.test.ts | 43 ++++++---- .../session-search-root-watchers.ts | 2 + .../session-search-watch-attachment.test.ts | 85 +++++++++++++++++++ .../session-search-work-queue.ts | 4 + .../ai-vault/session-file-inventory-files.ts | 13 ++- src/main/ai-vault/session-file-inventory.ts | 7 ++ 13 files changed, 233 insertions(+), 28 deletions(-) create mode 100644 src/main/ai-vault-search/session-search-watch-attachment.test.ts diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index fec0eb7c0a0..52650ee78a1 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -35,6 +35,19 @@ function groupKey(candidate: SessionFileCandidate, root: string): string { /** Ordinary paths own one record; only competing aliases allocate a group and selected set. */ export class SessionSearchCandidateInventory extends SessionSearchCandidatePublications { private readonly sourceOrders = new Map>() + rootFor(candidate: SessionFileCandidate): string | undefined { + const owner = this.byPath.get(candidate.file.path) + for (const key of typeof owner === 'string' ? [owner] : (owner ?? [])) { + const row = rows(this.groups.get(key)).find( + (row) => + row.candidate.agent === candidate.agent && row.candidate.file.path === candidate.file.path + ) + if (row) { + return row.root + } + } + return undefined + } has(path: string, agent?: string | null): boolean { const owner = this.byPath.get(path) if (!agent) { @@ -51,13 +64,14 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli removed: ReadonlySet, options: AiVaultScanOptions, invalidated: ReadonlySet = new Set(), - removedCandidates: ReadonlyMap> = new Map() + removedCandidates: ReadonlyMap> = new Map(), + partial = false ): Promise<{ candidates: SessionFileCandidate[]; suppressed: Set; forced: Set }> { const scopes = new Set( discoveries.map((discovery) => `${discovery.agent}:${discovery.rootDir}`) ) for (const scope of this.sourceOrders.keys()) { - if (!scopes.has(scope)) { + if (!partial && !scopes.has(scope)) { this.sourceOrders.delete(scope) } } diff --git a/src/main/ai-vault-search/session-search-discovery-inventory.ts b/src/main/ai-vault-search/session-search-discovery-inventory.ts index 0451d1baf8a..15a63fdac52 100644 --- a/src/main/ai-vault-search/session-search-discovery-inventory.ts +++ b/src/main/ai-vault-search/session-search-discovery-inventory.ts @@ -90,6 +90,21 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext return map } + async observeForRead(agent: string, rootDir: string, path: string) { + const root = this.roots.get(`${agent}:${rootDir}`) + // Container rows retain their source-owned admission and are never filesystem paths. + if (!root) { + return null + } + const change = await root.inventory.observeForRead(path, root.args, this.recentLimit) + if (change) { + for (const observer of root.dependencies.values()) { + observer.addFile(path) + } + } + return change + } + async files(args: DiscoverSessionFilesArgs): Promise { const key = `${args.agent}:${args.rootDir}` this.seen.add(key) diff --git a/src/main/ai-vault-search/session-search-index-pass.ts b/src/main/ai-vault-search/session-search-index-pass.ts index d5c6465daa5..444920a7e68 100644 --- a/src/main/ai-vault-search/session-search-index-pass.ts +++ b/src/main/ai-vault-search/session-search-index-pass.ts @@ -14,7 +14,7 @@ import { import { fileIdentity } from './session-search-file-cursor' import { sessionSearchReadDecision } from './session-search-read-decision' import type { SessionSearchIndexStore } from './session-search-store' -import type { SessionSearchWorkQueue } from './session-search-work-queue' +import type { SessionSearchWork, SessionSearchWorkQueue } from './session-search-work-queue' export async function runSessionSearchIndexPass( store: SessionSearchIndexStore, @@ -23,6 +23,7 @@ export async function runSessionSearchIndexPass( signal?: AbortSignal overdue?: () => boolean publications?: SessionSearchCandidateInventory + admit?: (work: SessionSearchWork) => Promise } ): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> { const stats = createSessionParseStats() @@ -36,10 +37,17 @@ export async function runSessionSearchIndexPass( outOfTime = true break } - const work = queue.next() + let work = queue.next() if (!work) { break } + if (options.admit) { + work = await options.admit(work) + if (!work) { + consumed++ + continue + } + } const { candidate } = work const path = candidate.file.path let row = store.fileRow(path) @@ -120,6 +128,7 @@ export async function runSessionSearchIndexPass( queue.acknowledge(work, false) continue } + const failuresBeforeRead = row?.failCount ?? 0 try { await parseAgentSessionFileCached( candidate, @@ -132,12 +141,19 @@ export async function runSessionSearchIndexPass( await store.flush?.() } catch (error) { throwIfAiVaultScanCancelled(options.signal) + if (error instanceof Error && error.name === 'AbortError') { + throw error + } console.warn( '[ai-vault-search] indexing skipped', candidate.agent, error instanceof Error ? error.name : 'ParseError' ) - store.setFileState(path, 'failed', candidate.file.mtimeMs) + // The attached transcript consumer may already have settled this failed attempt. + await store.flush?.() + if ((store.fileRow(path)?.failCount ?? 0) <= failuresBeforeRead) { + store.setFileState(path, 'failed', candidate.file.mtimeMs) + } await store.flush?.() } const committed = store.fileRow(path) diff --git a/src/main/ai-vault-search/session-search-indexer-test-fixture.ts b/src/main/ai-vault-search/session-search-indexer-test-fixture.ts index 74b27a50706..cefc447b631 100644 --- a/src/main/ai-vault-search/session-search-indexer-test-fixture.ts +++ b/src/main/ai-vault-search/session-search-indexer-test-fixture.ts @@ -127,6 +127,7 @@ export async function renameReplaceTranscript( await rename(replacement, path) const later = new Date(before.mtimeMs + 5_000) await utimes(path, later, later) + notifySessionSearchFileChange(path) } export async function writeMessageGraphTranscript( diff --git a/src/main/ai-vault-search/session-search-indexer.test.ts b/src/main/ai-vault-search/session-search-indexer.test.ts index d26ee05ec71..27394930d60 100644 --- a/src/main/ai-vault-search/session-search-indexer.test.ts +++ b/src/main/ai-vault-search/session-search-indexer.test.ts @@ -1,5 +1,8 @@ import './session-search-worker.test-fixture' -import { notifySessionSearchRootChanges } from './session-search-watcher.test-fixture' +import { + notifySessionSearchFileChange, + notifySessionSearchRootChanges +} from './session-search-watcher.test-fixture' import { mkdirSync, rmSync } from 'node:fs' import { appendFile, chmod, mkdir, rm, stat, utimes } from 'node:fs/promises' import { join } from 'node:path' @@ -152,6 +155,7 @@ it('reflects a grown transcript within one reconcile interval', async () => { path, `${claudeLines(['quarantine the leaking pty'], SESSION_ID, 10).join('\n')}\n` ) + notifySessionSearchFileChange(path) await nextCycle() expect(sessionsMatching('quarantine')).toEqual([SESSION_ID]) @@ -282,6 +286,7 @@ it.skipIf(!CAN_DENY_READ)('stops re-reading a transcript it cannot read', async await chmod(path, 0o644) const later = new Date(Date.now() + 60_000) await utimes(path, later, later) + notifySessionSearchFileChange(path) await nextCycle() expect(sessionsMatching('mode')).toEqual([SESSION_ID]) diff --git a/src/main/ai-vault-search/session-search-indexer.ts b/src/main/ai-vault-search/session-search-indexer.ts index edafdb8338a..aeb4c77611f 100644 --- a/src/main/ai-vault-search/session-search-indexer.ts +++ b/src/main/ai-vault-search/session-search-indexer.ts @@ -1,4 +1,5 @@ import { systemSessionSearchClock, type SessionSearchClock } from './session-search-clock' +import { waitForAiVaultScan } from '../ai-vault/ai-vault-scan-cancellation' import { DEFAULT_SESSION_SEARCH_FULL_SWEEP_EVERY_CYCLES, DEFAULT_SESSION_SEARCH_PASS_DEADLINE_FRACTION, @@ -246,7 +247,7 @@ export class SessionSearchIndexer { this.sweepInProgress ||= full try { if (maintenance && this.options.resolveRoots) { - const roots = await this.options.resolveRoots(signal) + const roots = await waitForAiVaultScan(this.options.resolveRoots(signal), signal) if (signal.aborted) { return } diff --git a/src/main/ai-vault-search/session-search-pass.ts b/src/main/ai-vault-search/session-search-pass.ts index 082dc5abfcd..820dbe78f7e 100644 --- a/src/main/ai-vault-search/session-search-pass.ts +++ b/src/main/ai-vault-search/session-search-pass.ts @@ -147,7 +147,46 @@ export async function runSessionSearchPass( const read = await runSessionSearchIndexPass(store, state.queue, { signal, overdue: readOverdue, - publications: state.candidates + publications: state.candidates, + admit: async (work) => { + const { candidate } = work + const path = candidate.file.path + const rootDir = state.candidates.rootFor(candidate) + if (rootDir === undefined) { + return undefined + } + const observation = await state.discovery.observeForRead(candidate.agent, rootDir, path) + if (observation === null) { + return work + } + if (!observation) { + // Observation debt owns retry/holdout; accepted facts cannot certify a refused input. + state.queue.acknowledge(work, false) + return undefined + } + if (!observation.changed) { + return work + } + const refreshed = await state.candidates.update( + [{ agent: candidate.agent, rootDir, files: [observation.file] }], + new Set(), + { ...args.roots, signal }, + observation.force ? new Set([path]) : new Set(), + new Map(), + true + ) + for (const suppressed of refreshed.suppressed) { + state.queue.remove(suppressed) + } + for (const selected of refreshed.candidates) { + const force = observation.force || refreshed.forced.has(selected.file.path) + if (force) { + state.candidates.requirePublication(selected.file.path) + } + state.queue.enqueue(selected, store, force, !args.full) + } + return state.queue.get(path) + } }) await state.candidates.reconcilePublications(store, readOverdue, signal, (path) => state.queue.hasPublicationDebt(path) diff --git a/src/main/ai-vault-search/session-search-root-refresh.test.ts b/src/main/ai-vault-search/session-search-root-refresh.test.ts index 4d77eca8317..b9cee4e033f 100644 --- a/src/main/ai-vault-search/session-search-root-refresh.test.ts +++ b/src/main/ai-vault-search/session-search-root-refresh.test.ts @@ -26,36 +26,33 @@ afterEach(async () => { await harness.cleanup() }) -it('refreshes roots on scheduled full sweeps and reuses them on recent cycles', async () => { +it('finds a changed session location on maintenance after ordinary cycles reuse the old location', async () => { const clock = new FakeSessionSearchClock() let roots = harness.roots - const resolveRoots = vi.fn(async () => roots) + const resolveRoots = async () => roots indexer = new SessionSearchIndexer({ databasePath: harness.databasePath, roots, resolveRoots, historyDays: null, clock, - fullSweepEveryCycles: 1 + fullSweepEveryCycles: 2 }) await indexer.start() - expect(resolveRoots).toHaveBeenCalledTimes(1) const newRoot = join(harness.root, 'late') const id = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' await writeClaudeTranscript(join(newRoot, 'project', `${id}.jsonl`), ['a late conversation'], id) roots = { ...roots, claudeProjectsDir: newRoot } clock.advance(20_000) await indexer.settled() - expect(resolveRoots).toHaveBeenCalledTimes(1) expect(indexer.status().filesIndexed).toBe(0) clock.advance(20_000) await indexer.settled() - expect(resolveRoots).toHaveBeenCalledTimes(2) - expect(indexer.status().filesIndexed).toBe(1) - clock.advance(20_000) - await indexer.settled() - expect(resolveRoots).toHaveBeenCalledTimes(2) - expect(indexer.status().filesIndexed).toBe(1) + expect( + harness.read((db) => + db.prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'late'").get() + ) + ).toEqual({ n: 2 }) }) it('does not access a closed store when pending discovery completes', async () => { @@ -80,10 +77,15 @@ it('does not access a closed store when pending discovery completes', async () = it('retries discovery after failure without silently sweeping stale roots', async () => { const errors = vi.fn() - const resolveRoots = vi - .fn() - .mockRejectedValueOnce(new Error('unavailable')) - .mockResolvedValue(harness.roots) + let unavailable = true + const path = join(harness.claudeProjectDir, 'recovery.jsonl') + await writeClaudeTranscript(path, ['recovered conversation'], 'recovery') + const resolveRoots = async () => { + if (unavailable) { + throw new Error('unavailable') + } + return harness.roots + } indexer = new SessionSearchIndexer({ databasePath: harness.databasePath, roots: harness.roots, @@ -94,7 +96,14 @@ it('retries discovery after failure without silently sweeping stale roots', asyn await indexer.start() expect(errors).toHaveBeenCalledTimes(1) expect(indexer.status().lastSweepCompletedAt).toBeNull() + expect(indexer.status().filesIndexed).toBe(0) + unavailable = false await indexer.reconcile() - expect(resolveRoots).toHaveBeenCalledTimes(2) - expect(indexer.status().lastSweepCompletedAt).not.toBeNull() + expect( + harness.read((db) => + db + .prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'recovered'") + .get() + ) + ).toEqual({ n: 2 }) }) diff --git a/src/main/ai-vault-search/session-search-root-watchers.ts b/src/main/ai-vault-search/session-search-root-watchers.ts index 4867b605990..33da4f2b222 100644 --- a/src/main/ai-vault-search/session-search-root-watchers.ts +++ b/src/main/ai-vault-search/session-search-root-watchers.ts @@ -4,6 +4,7 @@ import { WatcherProcessSupervisor } from '../ipc/filesystem/watcher-process/parc import type { WatcherProcessSubscription } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription' import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol' import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation' +import { RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS } from '../../shared/file-watch/runtime-file-watch-limits' export type SessionSearchRootWatch = { identity: string subscription?: WatcherProcessSubscription @@ -127,6 +128,7 @@ export class SessionSearchRootWatchers { { mode }, { signal: this.owner.signal, + subscribeTimeoutMs: RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS, onInterruption: () => { watch.reconcile = true }, diff --git a/src/main/ai-vault-search/session-search-watch-attachment.test.ts b/src/main/ai-vault-search/session-search-watch-attachment.test.ts new file mode 100644 index 00000000000..a68c700aabf --- /dev/null +++ b/src/main/ai-vault-search/session-search-watch-attachment.test.ts @@ -0,0 +1,85 @@ +import './session-search-worker.test-fixture' +import { appendFile } from 'node:fs/promises' +import { join } from 'node:path' +import { expect, it, vi } from 'vitest' +import { FakeWatcherChild } from '../ipc/filesystem/watcher-process/parcel-watcher-process-test-child' +import { SessionSearchIndexer } from './session-search-indexer' +import { + claudeLines, + FakeSessionSearchClock, + openSessionSearchIndexerHarness, + writeClaudeTranscript +} from './session-search-indexer-test-fixture' + +const { fork } = vi.hoisted(() => ({ fork: vi.fn() })) +vi.mock('node:child_process', async (original) => ({ ...(await original()), fork })) +vi.mock('./session-search-filesystem', () => ({ + sessionSearchRequiresPolling: async () => false +})) + +it('keeps indexing after a native crawl never becomes ready and retires its child', async () => { + const harness = await openSessionSearchIndexerHarness('ss-pending-watch') + const path = join(harness.claudeProjectDir, 'pending.jsonl') + const clock = new FakeSessionSearchClock() + let crawling = false + const children: FakeWatcherChild[] = [] + fork.mockImplementation(() => { + const child = new FakeWatcherChild() + children.push(child) + child.send.mockImplementation((message) => { + child.sent.push(message) + queueMicrotask(() => { + if (message.op === 'subscribe') { + crawling = true + child.emit('message', { op: 'subscribe-started', id: message.id }) + } else if (message.op === 'cancel-subscribe') { + child.emit('message', { op: 'cancel-requires-restart', id: message.id }) + } + }) + return true + }) + child.kill.mockImplementation(() => { + child.connected = false + queueMicrotask(() => child.emit('exit', 0, null)) + return true + }) + return child + }) + vi.stubEnv('VITEST', '') + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock, + watcherEntryPath: import.meta.filename + }) + try { + await writeClaudeTranscript(path, ['beforepressure'], 'pending') + const start = subject.start() + await vi.waitFor(() => expect(crawling).toBe(true)) + await vi.advanceTimersByTimeAsync(60_000) + await start + expect(subject.status().filesIndexed).toBe(1) + expect(children.every((child) => !child.connected)).toBe(true) + + await appendFile(path, `${claudeLines(['afterpressure'], 'pending', 10).join('\n')}\n`) + clock.advance(20_000) + await subject.settled() + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'afterpressure'" + ) + .get() + ) + ).toEqual({ n: 2 }) + } finally { + await subject.close() + vi.useRealTimers() + vi.unstubAllEnvs() + fork.mockReset() + await harness.cleanup() + } +}) diff --git a/src/main/ai-vault-search/session-search-work-queue.ts b/src/main/ai-vault-search/session-search-work-queue.ts index 314937cc4b1..33488380af9 100644 --- a/src/main/ai-vault-search/session-search-work-queue.ts +++ b/src/main/ai-vault-search/session-search-work-queue.ts @@ -73,6 +73,10 @@ export class SessionSearchWorkQueue { return preferred.first ?? (preferred === this.changed ? this.backlog : this.changed).first } + get(path: string): SessionSearchWork | undefined { + return this.changed.get(path) ?? this.backlog.get(path) + } + acknowledge(work: SessionSearchWork, retry: boolean): void { const path = work.candidate.file.path const queue = this.changed.has(path) ? this.changed : this.backlog diff --git a/src/main/ai-vault/session-file-inventory-files.ts b/src/main/ai-vault/session-file-inventory-files.ts index 37ef356ce35..ca31a48ace4 100644 --- a/src/main/ai-vault/session-file-inventory-files.ts +++ b/src/main/ai-vault/session-file-inventory-files.ts @@ -23,7 +23,13 @@ export class SessionInventoryFiles { private readonly scheduleParent: (path: string) => void, private readonly forgetDirty: (path: string) => void ) {} - async observe(path: string, args: DiscoverSessionFilesArgs, force: boolean, limit: number) { + async observe( + path: string, + args: DiscoverSessionFilesArgs, + force: boolean, + limit: number, + readOwed = false + ) { if (!this.failures.mayAttempt(path)) { return } @@ -46,7 +52,7 @@ export class SessionInventoryFiles { } this.failures.fail( path, - force, + force || readOwed, observation.missing ? undefined : { agent: args.agent, ...observation.issue } ) // Only apparent absence needs a parent membership proof. @@ -65,7 +71,7 @@ export class SessionInventoryFiles { codexHome: null }) if (!metadataOnly && observation.file.sidecar === 'unknown') { - this.failures.fail(path, force) + this.failures.fail(path, force || readOwed) file.sidecar = previous?.sidecar ?? 'unknown' } else { this.failures.delete(path) @@ -117,6 +123,7 @@ export class SessionInventoryFiles { }) } retainRecentSessionFile(this.recent, file, limit) + return !metadataOnly && observation.file.sidecar === 'unknown' ? undefined : file } forget(path: string): void { diff --git a/src/main/ai-vault/session-file-inventory.ts b/src/main/ai-vault/session-file-inventory.ts index 991499115f9..73312a468dd 100644 --- a/src/main/ai-vault/session-file-inventory.ts +++ b/src/main/ai-vault/session-file-inventory.ts @@ -127,6 +127,13 @@ export class SessionFileInventory { } } + async observeForRead(path: string, args: DiscoverSessionFilesArgs, limit: number) { + const file = await this.observations.observe(path, args, false, limit, true) + const change = this.observations.changes.get(path) + this.observations.changes.delete(path) + return file ? { file, force: change?.force ?? false, changed: change !== undefined } : undefined + } + async drain( args: DiscoverSessionFilesArgs, overdue: () => boolean,