diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index 838a337b593..f5b8940edd4 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -11,55 +11,99 @@ import type { SessionFileDiscovery } from '../ai-vault/session-scanner-types' -/** Keep suppressed aliases so deleting a preferred path can promote the surviving source. */ +type Row = { candidate: SessionFileCandidate; root: string } +type Group = Row | Map +function rows(group: Group | undefined): Row[] { + return group ? (group instanceof Map ? [...group.values()] : [group]) : [] +} +function rowId(row: Row): string { + return `${row.candidate.agent}:${row.candidate.file.path}` +} +function groupKey(candidate: SessionFileCandidate, root: string): string { + const path = candidate.file.path + return candidate.agent === 'codex' + ? `codex:${sessionFileExecutionNamespace(path)}:${basename(path)}` + : candidate.agent === 'antigravity' + ? `antigravity:${dirname(path)}` + : candidate.agent === 'opencode' || + candidate.agent === 'opencode2' || + candidate.agent === 'zcode' + ? `${candidate.agent}:${root}:${splitOpenCodeSqliteCandidate(path, candidate.agent)?.sessionId ?? basename(path, extname(path))}` + : path +} + +/** Ordinary paths own one record; only competing aliases allocate a group and selected set. */ export class SessionSearchCandidateInventory { - private readonly groups = new Map< - string, - Map - >() - private readonly byPath = new Map>() + private readonly groups = new Map() + private readonly byPath = new Map>() private readonly selected = new Map>() - - has(path: string): boolean { - return this.byPath.has(path) + has(path: string, agent?: string | null): boolean { + const owner = this.byPath.get(path) + if (!agent) { + return owner !== undefined + } + return (typeof owner === 'string' ? [owner] : [...(owner ?? [])]).some((key) => + rows(this.groups.get(key)).some( + (row) => row.candidate.file.path === path && row.candidate.agent === agent + ) + ) } - async update( discoveries: readonly SessionFileDiscovery[], removed: ReadonlySet, options: AiVaultScanOptions, - invalidated: ReadonlySet = new Set() + invalidated: ReadonlySet = new Set(), + removedCandidates: ReadonlyMap> = new Map() ): Promise<{ candidates: SessionFileCandidate[]; suppressed: Set; forced: Set }> { - const changed = new Set() + const previous = new Map>() const forced = new Set() - for (const path of removed) { - for (const key of this.byPath.get(path) ?? []) { - const group = this.groups.get(key) - if (group) { - for (const [id, row] of group) { - if (row.candidate.file.path === path) { - group.delete(id) - } - } + const touch = (key: string) => { + if (previous.has(key)) { + return + } + const group = this.groups.get(key) + previous.set( + key, + this.selected.get(key) ?? + new Set(group && !(group instanceof Map) ? [group.candidate.file.path] : []) + ) + } + for (const path of new Set([...removed, ...removedCandidates.keys()])) { + const owner = this.byPath.get(path) + for (const key of typeof owner === 'string' ? [owner] : (owner ?? [])) { + touch(key) + const remaining = rows(this.groups.get(key)).filter( + (row) => + row.candidate.file.path !== path || + (!removed.has(path) && !removedCandidates.get(path)?.has(row.candidate.agent)) + ) + if (remaining.length === 0) { + this.groups.delete(key) + } else if (remaining.length === 1) { + this.groups.set(key, remaining[0]) + } else { + this.groups.set(key, new Map(remaining.map((row) => [rowId(row), row]))) } - changed.add(key) + } + const retained = new Set( + (typeof owner === 'string' ? [owner] : [...(owner ?? [])]).filter((key) => + rows(this.groups.get(key)).some((row) => row.candidate.file.path === path) + ) + ) + if (retained.size === 0) { + this.byPath.delete(path) + } else { + this.byPath.set(path, retained.size === 1 ? retained.values().next().value! : retained) } - this.byPath.delete(path) } for (const discovery of discoveries) { for (const candidate of discoveryFileCandidates(discovery, discovery.files, options)) { - const path = candidate.file.path - const key = - candidate.agent === 'codex' - ? `codex:${sessionFileExecutionNamespace(path)}:${basename(path)}` - : candidate.agent === 'antigravity' - ? `antigravity:${dirname(path)}` - : candidate.agent === 'opencode' - ? `opencode:${sessionFileExecutionNamespace(path)}:${splitOpenCodeSqliteCandidate(path)?.sessionId ?? basename(path, extname(path))}` - : path - const group = this.groups.get(key) ?? new Map() + const path = candidate.file.path, + key = groupKey(candidate, discovery.rootDir) + touch(key) + const group = this.groups.get(key) if (candidate.agent === 'codex' && candidate.file.nlink && candidate.file.nlink > 1) { - for (const row of group.values()) { + for (const row of rows(group)) { const file = row.candidate.file if (file.dev === candidate.file.dev && file.ino === candidate.file.ino) { row.candidate = { ...row.candidate, file: { ...candidate.file, path: file.path } } @@ -69,27 +113,53 @@ export class SessionSearchCandidateInventory { } } } - group.set(`${candidate.agent}:${path}`, { candidate, root: discovery.rootDir }) - this.groups.set(key, group) - const keys = this.byPath.get(path) ?? new Set() - keys.add(key) - this.byPath.set(path, keys) - changed.add(key) + const row = { candidate, root: discovery.rootDir } + if (!group) { + this.groups.set(key, row) + } else if (group instanceof Map) { + group.set(rowId(row), row) + } else if (rowId(group) === rowId(row)) { + this.groups.set(key, row) + } else { + this.groups.set( + key, + new Map([ + [rowId(group), group], + [rowId(row), row] + ]) + ) + } + const owner = this.byPath.get(path) + if (!owner) { + this.byPath.set(path, key) + } else if (typeof owner === 'string') { + if (owner !== key) { + this.byPath.set(path, new Set([owner, key])) + } + } else { + owner.add(key) + } } } - const candidates: SessionFileCandidate[] = [] - const suppressed = new Set() - for (const key of changed) { - const group = this.groups.get(key)! - const rows = [...group.values()] - const eligible = rows.some( - (row) => - row.candidate.agent === 'opencode' && - splitOpenCodeSqliteCandidate(row.candidate.file.path) + const candidates: SessionFileCandidate[] = [], + suppressed = new Set() + for (const [key, oldPaths] of previous) { + const group = this.groups.get(key) + const members = rows(group) + const eligible = members.some((row) => + splitOpenCodeSqliteCandidate( + row.candidate.file.path, + row.candidate.agent === 'zcode' ? 'zcode' : 'opencode' + ) ) - ? rows.filter((row) => splitOpenCodeSqliteCandidate(row.candidate.file.path)) - : rows - const selected = await sessionCandidatesFromDiscoveries( + ? members.filter((row) => + splitOpenCodeSqliteCandidate( + row.candidate.file.path, + row.candidate.agent === 'zcode' ? 'zcode' : 'opencode' + ) + ) + : members + const ranked = await sessionCandidatesFromDiscoveries( eligible.map(({ candidate, root }) => ({ agent: candidate.agent, rootDir: root, @@ -97,23 +167,42 @@ export class SessionSearchCandidateInventory { })), options ) + // Each database source originally chose greatest mtime, with source order breaking ties. + const selected = eligible.some((row) => + splitOpenCodeSqliteCandidate( + row.candidate.file.path, + row.candidate.agent === 'zcode' ? 'zcode' : 'opencode' + ) + ) + ? ranked.slice(0, 1) + : ranked const paths = new Set(selected.map((candidate) => candidate.file.path)) - for (const path of this.selected.get(key) ?? []) { + for (const path of oldPaths) { if (!paths.has(path)) { suppressed.add(path) } } - this.selected.set(key, paths) - candidates.push(...selected) - if (group.size === 0) { - this.groups.delete(key) + if (group instanceof Map) { + this.selected.set(key, paths) + } else { this.selected.delete(key) } + candidates.push(...selected) } - return { - candidates: candidates.sort((a, b) => b.file.mtimeMs - a.file.mtimeMs), - suppressed, - forced + for (const path of suppressed) { + const owner = this.byPath.get(path) + if ( + (typeof owner === 'string' ? [owner] : [...(owner ?? [])]).some((key) => { + const group = this.groups.get(key) + return group instanceof Map + ? this.selected.get(key)?.has(path) + : group?.candidate.file.path === path + }) + ) { + suppressed.delete(path) + } } + candidates.sort((left, right) => right.file.mtimeMs - left.file.mtimeMs) + return { candidates, suppressed, forced } } } diff --git a/src/main/ai-vault-search/session-search-container-inventory.ts b/src/main/ai-vault-search/session-search-container-inventory.ts new file mode 100644 index 00000000000..076037f7b7b --- /dev/null +++ b/src/main/ai-vault-search/session-search-container-inventory.ts @@ -0,0 +1,201 @@ +import { dirname } from 'node:path' +import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types' +import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation' +import type { FileWithMtime, SessionFileDiscovery } from '../ai-vault/session-scanner-types' +import { isMissingTranscriptPathError } from '../ai-vault/session-scanner-discovery' +import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access' +import { splitSyntheticSessionSource } from './session-search-synthetic-sources' + +type Physical = { + files: Map + signature?: string + identity?: string + fullDue: boolean +} +type Family = { discovery?: SessionFileDiscovery; physical: Map } + +/** Recent observations add facts; only a successful exhaustive observation can withdraw them. */ +export class SessionSearchContainerInventory { + private readonly families = new Map() + private seen = new Set() + private recentLimit = 12 + readonly removed = new Map>() + readonly proofs = new Map>() + + begin(full: boolean, recentLimit: number): void { + this.seen = new Set() + this.recentLimit = recentLimit + this.removed.clear() + if (full) { + for (const family of this.families.values()) { + for (const physical of family.physical.values()) { + physical.fullDue = true + } + } + } + } + get pending(): boolean { + for (const family of this.families.values()) { + for (const physical of family.physical.values()) { + if (physical.fullDue) { + return true + } + } + } + return false + } + async discover( + key: string, + paths: readonly string[], + read: (paths: readonly string[], limit: number) => Promise, + issues: readonly AiVaultScanIssue[], + signal?: AbortSignal + ): Promise { + this.seen.add(key) + let family = this.families.get(key) + if (!family) { + family = { physical: new Map() } + this.families.set(key, family) + } + const changes: FileWithMtime[] = [] + for (const path of paths) { + let physical = family.physical.get(path) + if (!physical) { + physical = { files: new Map(), fullDue: true } + family.physical.set(path, physical) + } + const before = await this.signature(path, signal) + const proofKey = `${key.split(':', 1)[0]}:${path}` + if (!before.known || physical.signature !== before.signature) { + this.proofs.delete(proofKey) + } + if (physical.identity !== before.identity && before.known) { + physical.fullDue = true + } + if (before.identity === '' && before.known && family.discovery) { + this.release(key, path, family, physical) + continue + } + const full = physical.fullDue + const discovery = await read([path], full ? Infinity : this.recentLimit) + family.discovery = discovery + throwIfAiVaultScanCancelled(signal) + const current = new Map() + for (const file of discovery.files) { + current.set(file.path, file) + const source = splitSyntheticSessionSource(file.path) + const proof = this.proofs.get(proofKey) + if (source && proof && !proof.has(source.id)) { + this.proofs.delete(proofKey) + } + const previous = physical.files.get(file.path) + if ( + !previous || + previous.mtimeMs !== file.mtimeMs || + previous.sizeBytes !== file.sizeBytes + ) { + changes.push(file) + } + physical.files.set(file.path, file) + } + const after = await this.signature(path, signal) + const complete = discovery.complete !== false && before.known && after.known + const stable = complete && before.signature === after.signature + if (full && complete) { + physical.fullDue = !stable + // Empty also means an unknown schema under the existing reader contract. + if (stable && current.size) { + for (const old of physical.files.keys()) { + if (!current.has(old)) { + this.remove(old, discovery.agent) + physical.files.delete(old) + } + } + const ids = new Set() + for (const file of current.values()) { + const source = splitSyntheticSessionSource(file.path) + if (source) { + ids.add(source.id) + } + } + this.proofs.set(proofKey, ids) + } + } + if (!stable) { + this.proofs.delete(proofKey) + } + physical.signature = after.signature + physical.identity = after.known ? after.identity : physical.identity + } + // A refused family listing cannot certify that a physical source disappeared. + const root = key.slice(key.indexOf(':') + 1) + if ( + !issues.some( + (issue) => + issue.kind !== 'notice' && + (issue.path === root || issue.path === dirname(root) || family.physical.has(issue.path)) + ) + ) { + for (const [path, physical] of family.physical) { + if (!paths.includes(path)) { + this.release(key, path, family, physical) + } + } + } + family.discovery ??= await read([], this.recentLimit) + return { ...family.discovery, files: changes } + } + end(): void { + for (const [key, family] of this.families) { + if (!this.seen.has(key)) { + for (const [path, physical] of family.physical) { + this.release(key, path, family, physical) + } + this.families.delete(key) + } + } + } + clear(): void { + this.families.clear() + this.proofs.clear() + this.removed.clear() + } + private remove(path: string, agent: string): void { + const agents = this.removed.get(path) ?? new Set() + agents.add(agent) + this.removed.set(path, agents) + } + private release(key: string, path: string, family: Family, physical: Physical): void { + for (const old of physical.files.keys()) { + if (family.discovery) { + this.remove(old, family.discovery.agent) + } + } + this.proofs.delete(`${key.split(':', 1)[0]}:${path}`) + family.physical.delete(path) + } + private async signature(path: string, signal?: AbortSignal) { + let known = true + let identity = '' + const facts = await Promise.all( + [path, `${path}-wal`].map(async (input) => { + try { + const stat = await wslGatedStat(input, 'scan', signal) + const id = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}` + if (input === path) { + identity = id + } + return `${id}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}` + } catch (error) { + throwIfAiVaultScanCancelled(signal) + if (isMissingTranscriptPathError(error)) { + return 'missing' + } + known = false + return 'unknown' + } + }) + ) + return { known, identity, signature: facts.join('|') } + } +} diff --git a/src/main/ai-vault-search/session-search-deleted-sources.ts b/src/main/ai-vault-search/session-search-deleted-sources.ts index c62916c6bf6..4bda107270a 100644 --- a/src/main/ai-vault-search/session-search-deleted-sources.ts +++ b/src/main/ai-vault-search/session-search-deleted-sources.ts @@ -79,7 +79,12 @@ export async function retireDeletedSessionSearchSources( signal }) const proof = synthetic - ? proveSyntheticSource(synthetic, containerProof, args.enumeratedContainers) + ? proveSyntheticSource( + synthetic, + containerProof, + args.enumeratedContainers, + store.fileRow(path)?.agent + ) : containerProof if (proof.verdict === 'gone') { store.removeFile(path) @@ -146,12 +151,14 @@ async function proveSource( function proveSyntheticSource( synthetic: { container: string; id: string }, containerProof: SessionSearchSourceVerdict, - enumerated?: ReadonlyMap> + enumerated?: ReadonlyMap>, + agent?: string | null ): SessionSearchSourceVerdict { if (containerProof.verdict !== 'present') { return containerProof } - const ids = enumerated?.get(synthetic.container) + const ids = + enumerated?.get(`${agent}:${synthetic.container}`) ?? enumerated?.get(synthetic.container) // Unknown schemas can enumerate empty; require nonempty evidence before retiring container entries. if (!ids || ids.size === 0) { return { diff --git a/src/main/ai-vault-search/session-search-dependency-inventory.ts b/src/main/ai-vault-search/session-search-dependency-inventory.ts new file mode 100644 index 00000000000..2d5bf4d60d5 --- /dev/null +++ b/src/main/ai-vault-search/session-search-dependency-inventory.ts @@ -0,0 +1,117 @@ +import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation' +import { recordSessionScanIssue, scanIssueTextFromError } from '../ai-vault/session-scan-issues' +import { SessionFileInventory } from '../ai-vault/session-file-inventory' +import type { DiscoverSessionFilesArgs } from '../ai-vault/session-scanner-discovery' +import type { SessionDiscoveryDependency } from '../ai-vault/session-discovery-context' +import type { SessionSearchRootWatch } from './session-search-root-watchers' + +/** Losing a subscription invalidates the input observation, not every transcript using that input. */ +export class SessionSearchDependencyInventory { + private readonly inventory: SessionFileInventory + private readonly observe: ((force: boolean) => Promise>) | undefined + private refused = false + private readonly keys = new Map>() + constructor( + readonly dependency: SessionDiscoveryDependency, + public polling = false + ) { + this.inventory = new SessionFileInventory(dependency.path) + this.observe = dependency.observe?.() + } + get pending(): boolean { + return this.observe ? this.refused : this.inventory.pending + } + owners(key: string): Iterable { + const owners = this.keys.get(key) + return typeof owners === 'string' ? [owners] : (owners ?? []) + } + addFile(path: string): void { + const key = this.dependency.fileKey?.(path) + if (!key) { + return + } + const paths = this.keys.get(key) + if (!paths) { + this.keys.set(key, path) + } else if (typeof paths === 'string') { + if (paths !== path) { + this.keys.set(key, new Set([paths, path])) + } + } else { + paths.add(path) + } + } + removeFile(path: string): void { + const key = this.dependency.fileKey?.(path) + if (!key) { + return + } + const paths = this.keys.get(key) + if (typeof paths === 'string') { + if (paths === path) { + this.keys.delete(key) + } + } else if (paths) { + paths.delete(path) + if (!paths.size) { + this.keys.delete(key) + } else if (paths.size === 1) { + this.keys.set(key, paths.values().next().value!) + } + } + } + async changedKeys( + watch: SessionSearchRootWatch | undefined, + args: DiscoverSessionFilesArgs, + overdue: () => boolean + ): Promise> { + if (this.observe) { + try { + const delivered = [...(watch?.events.values() ?? [])].some( + (event) => !!this.dependency.eventKey?.(event.path) + ) + const keys = new Set(await this.observe(delivered)) + this.refused = false + return keys + } catch (error) { + throwIfAiVaultScanCancelled(args.signal) + recordSessionScanIssue(args.issues, { + agent: args.agent, + path: this.dependency.path, + ...scanIssueTextFromError(error) + }) + this.refused = true + return new Set() + } + } + const keys = new Set() + if ((!watch || watch.failed || watch.reconcile) && !this.inventory.walking) { + this.inventory.reconcile() + } + for (const event of watch?.events.values() ?? []) { + this.inventory.invalidate(event.path) + } + this.inventory.retryFailures() + const delta = await this.inventory.drain( + { + ...args, + rootDir: this.dependency.path, + extensions: ['*'], + filePredicate: (path) => !!this.dependency.eventKey?.(path), + directoryPredicate: this.dependency.mode === 'shallow' ? () => false : undefined, + contentDependencyPath: undefined, + searchDependencies: undefined, + onDirectoryEntries: undefined + }, + overdue, + 0 + ) + for (const path of [...delta.changes.map((change) => change.file.path), ...delta.removed]) { + const key = this.dependency.eventKey?.(path) + if (key) { + keys.add(key) + } + } + return keys + } +} diff --git a/src/main/ai-vault-search/session-search-dependency-work.test.ts b/src/main/ai-vault-search/session-search-dependency-work.test.ts new file mode 100644 index 00000000000..c510e9f3fed --- /dev/null +++ b/src/main/ai-vault-search/session-search-dependency-work.test.ts @@ -0,0 +1,103 @@ +import './session-search-worker.test-fixture' +import { + notifySessionSearchFileChange, + notifySessionSearchOverflow +} from './session-search-watcher.test-fixture' +import { appendFile, mkdir, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { expect, it, vi } from 'vitest' +import * as filesystem from '../native-chat/wsl-transcript-fs-access' +import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache' +import { + codexFixture, + CODEX_FIXTURE_SESSION_ID +} from '../ai-vault/scanners/codex/session-scanner-codex-fixtures' +import { SessionSearchIndexer } from './session-search-indexer' +import { + FakeSessionSearchClock, + openSessionSearchIndexerHarness +} from './session-search-indexer-test-fixture' + +it.each([40, 400])( + 'bounds one title append independently of %s index records and transcript owners', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-title-input-work') + const clock = new FakeSessionSearchClock() + const root = harness.roots.codexSessionsDir! + const index = join(dirname(root), 'session_index.jsonl') + const fixture = codexFixture() + const ids = Array.from( + { length: population }, + (_, id) => `019f0000-1111-7222-8333-${String(id).padStart(12, '0')}` + ) + const records = ids.map((id) => `${JSON.stringify({ id, thread_name: 'Original title' })}\n`) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + const streams = vi.spyOn(filesystem, 'openTranscriptReadStream') + const slices = vi.spyOn(filesystem, 'readTranscriptSlice') + const reads = vi.spyOn(parser, 'parseAgentSessionFileCached') + try { + await mkdir(root, { recursive: true }) + for (const id of ids) { + await writeFile( + join(root, fixture.fileName.replace(CODEX_FIXTURE_SESSION_ID, id)), + `${fixture.seedLines.join('\n').replaceAll(CODEX_FIXTURE_SESSION_ID, id)}\n` + ) + } + await writeFile(index, records.join('')) + await subject.start() + streams.mockClear() + slices.mockClear() + reads.mockClear() + // Lost delivery must re-observe the input, not reread it or force every transcript. + notifySessionSearchOverflow(dirname(root)) + clock.advance(20000) + await subject.settled() + expect(streams.mock.calls.filter(([path]) => path === index)).toHaveLength(0) + expect(reads).not.toHaveBeenCalled() + const append = `${JSON.stringify({ id: ids[0], thread_name: 'Changed title' })}\n` + const size = Buffer.byteLength(records.join('') + append) + await appendFile(index, append) + notifySessionSearchFileChange(index) + clock.advance(20000) + await subject.settled() + const indexBytes = + streams.mock.calls.reduce( + (bytes, [path, options]) => + path !== index + ? bytes + : bytes + Math.max(0, (options?.end ?? size - 1) - (options?.start ?? 0) + 1), + 0 + ) + + slices.mock.calls.reduce( + (bytes, [path, , length]) => (path === index ? bytes + length : bytes), + 0 + ) + // Two independent consumers need the appended record plus one fixed continuity record each. + expect(indexBytes).toBeLessThanOrEqual( + 2 * (Buffer.byteLength(append) + Buffer.byteLength(records[0]) + 1) + ) + expect(reads).toHaveBeenCalledTimes(1) + expect( + harness.read((db) => + db.prepare("SELECT count(*) AS n FROM sessions WHERE title='Changed title'").get() + ) + ).toEqual({ n: 1 }) + expect( + harness.read((db) => + db.prepare("SELECT count(*) AS n FROM sessions WHERE title='Original title'").get() + ) + ).toEqual({ n: population - 1 }) + } finally { + streams.mockRestore() + slices.mockRestore() + reads.mockRestore() + await subject.close() + await harness.cleanup() + } + } +) diff --git a/src/main/ai-vault-search/session-search-discovery-inventory.ts b/src/main/ai-vault-search/session-search-discovery-inventory.ts index c75c5003e75..12bd6c3173d 100644 --- a/src/main/ai-vault-search/session-search-discovery-inventory.ts +++ b/src/main/ai-vault-search/session-search-discovery-inventory.ts @@ -1,62 +1,73 @@ +import { SessionSearchDependencyInventory } from './session-search-dependency-inventory' import { SessionSearchRootWatchers } from './session-search-root-watchers' -import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access' +import { SessionSearchContainerInventory } from './session-search-container-inventory' import { SessionFileInventory } from '../ai-vault/session-file-inventory' -import { - discoverSessionFiles, - isMissingTranscriptPathError, - type DiscoverSessionFilesArgs -} from '../ai-vault/session-scanner-discovery' +import type { DiscoverSessionFilesArgs } from '../ai-vault/session-scanner-discovery' import type { SessionFileDiscovery } from '../ai-vault/session-scanner-types' import type { SessionDiscoveryContext } from '../ai-vault/session-discovery-context' -import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation' -import { splitSyntheticSessionSource } from './session-search-synthetic-sources' +import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types' type Root = { inventory: SessionFileInventory args: DiscoverSessionFilesArgs identity?: string fallback: boolean - fallbackCount?: number - dependencyKeys: Map>> - dependencySnapshots: Map> + dependencies: Map } -type Container = { signature: string; discovery: SessionFileDiscovery; membership: Set } /** Search owns its notifications and raw membership; ordinary History discovery stays stateless. */ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext { private readonly roots = new Map() private readonly artifactMaps = new Map>() - private readonly containers = new Map() + readonly containers = new SessionSearchContainerInventory() private readonly owner = new AbortController() private readonly watchers: SessionSearchRootWatchers private force = false + private maintenance = false private overdue = () => false private recentLimit = 12 private seen = new Set() readonly forced = new Set() readonly metadata = new Set() readonly removed = new Set() - readonly enumeratedContainers = new Map>() + readonly removedCandidates = this.containers.removed + readonly enumeratedContainers = this.containers.proofs constructor(entryPath?: string) { this.watchers = new SessionSearchRootWatchers(entryPath) } - begin(options: { full: boolean; overdue: () => boolean; recentLimit: number }): void { + begin(options: { + full: boolean + maintenance?: boolean + overdue: () => boolean + recentLimit: number + }): void { this.force = options.full + this.maintenance = options.maintenance === true this.overdue = options.overdue this.recentLimit = options.recentLimit this.watchers.begin() this.seen = new Set() + this.containers.begin(options.full || this.maintenance, this.recentLimit) this.forced.clear() this.metadata.clear() this.removed.clear() - this.enumeratedContainers.clear() + } + + get walking(): boolean { + return [...this.roots.values()].some((root) => root.inventory.walking) } get pending(): boolean { + if (this.containers.pending) { + return true + } for (const root of this.roots.values()) { - if (!root.fallback && root.inventory.pending) { + if ( + root.inventory.pending || + [...root.dependencies.values()].some((dependency) => dependency.pending) + ) { return true } } @@ -66,7 +77,7 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext listings(): { root: string; files: number }[] { return [...this.roots.values()].map((root) => ({ root: root.inventory.root, - files: root.fallback ? (root.fallbackCount ?? 0) : root.inventory.files.size + files: root.inventory.files.size })) } @@ -79,10 +90,6 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext return map } - membership(key: string): ReadonlySet | undefined { - return this.containers.get(key)?.membership - } - async files(args: DiscoverSessionFilesArgs): Promise { const key = `${args.agent}:${args.rootDir}` this.seen.add(key) @@ -93,92 +100,69 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext inventory: new SessionFileInventory(args.rootDir), args, fallback, - dependencyKeys: new Map(), - dependencySnapshots: new Map() + dependencies: new Map() } this.roots.set(key, root) } root.args = args - if (root.fallback) { - const discovery = await discoverSessionFiles(args) - root.fallbackCount = discovery.files.length - return discovery + if (this.maintenance) { + root.fallback = await this.watchers.requiresPolling(args.rootDir) } - const watch = await this.watchers.watch(args.rootDir, args.signal) - if (!watch) { - // Missing roots remain obligations; never replace their retained membership with an empty delta. - root.inventory.reconcile() - } else { - if (this.force || watch.reconcile || root.identity !== watch.identity) { + if (root.fallback) { + if (!root.inventory.walking) { root.inventory.reconcile() } - if (root.identity !== watch.identity && (await this.watchers.requiresPolling(args.rootDir))) { - root.fallback = true - const discovery = await discoverSessionFiles(args) - root.fallbackCount = discovery.files.length - return discovery - } - root.identity = watch.identity - for (const event of watch.events.values()) { - root.inventory.invalidate(event.path) + } else { + const watch = await this.watchers.watch(args.rootDir, args.signal) + if (!watch) { + if (!root.inventory.walking) { + root.inventory.reconcile() + } + } else { + if ( + (this.force || watch.reconcile || root.identity !== watch.identity) && + (root.identity !== undefined || !root.inventory.walking) + ) { + root.inventory.reconcile() + } + if ( + root.identity !== watch.identity && + (await this.watchers.requiresPolling(args.rootDir)) + ) { + root.fallback = true + } + if (watch.failed && !root.inventory.walking) { + root.inventory.reconcile() + } + root.identity = watch.identity + for (const event of watch.events.values()) { + root.inventory.invalidate(event.path) + } } } const auxiliaries = await Promise.all( - (args.searchDependencies ?? []).map(async (dependency) => ({ - dependency, - watch: await this.watchers.watch(dependency.path, args.signal) - })) + (args.searchDependencies ?? []).map(async (dependency) => { + let observer = root.dependencies.get(dependency.path) + if (!observer) { + const polling = root.fallback || (await this.watchers.requiresPolling(dependency.path)) + observer = new SessionSearchDependencyInventory(dependency, polling) + root.dependencies.set(dependency.path, observer) + } else if (this.maintenance) { + observer.polling = root.fallback || (await this.watchers.requiresPolling(dependency.path)) + } + return { + dependency, + observer, + watch: observer.polling + ? undefined + : await this.watchers.watch(dependency.path, args.signal, dependency.mode) + } + }) ) - for (const { dependency, watch: auxiliary } of auxiliaries) { - let keys = root.dependencyKeys.get(dependency.path) - if (!keys) { - keys = new Map() - root.dependencyKeys.set(dependency.path, keys) - } - if (!auxiliary) { - continue - } - if (auxiliary.reconcile || auxiliary.failed) { - for (const file of root.inventory.files.values()) { - root.inventory.invalidate(file.path) - if (dependency.metadata) { - this.metadata.add(file.path) - } - } - } - const eventKeys = new Set() - for (const event of auxiliary.events.values()) { - for (const path of root.inventory.dependencies.get(event.path) ?? []) { - root.inventory.invalidate(path) - if (dependency.metadata) { - this.metadata.add(path) - } - } - const key = dependency.eventKey?.(event.path) - if (key) { - eventKeys.add(key) - } - } - if (dependency.snapshot && (auxiliary.reconcile || eventKeys.size)) { - const snapshot = await dependency.snapshot() - const previous = root.dependencySnapshots.get(dependency.path) - root.dependencySnapshots.set(dependency.path, snapshot) - eventKeys.clear() - for (const [key, value] of snapshot) { - if (previous?.get(key) !== value) { - eventKeys.add(key) - } - } - for (const key of previous?.keys() ?? []) { - if (!snapshot.has(key)) { - eventKeys.add(key) - } - } - // Nonstandard filenames cannot identify their thread until parsed. - eventKeys.add('*') - } + for (const { dependency, observer, watch: auxiliary } of auxiliaries) { + const eventKeys = await observer.changedKeys(auxiliary, args, this.overdue) for (const key of eventKeys) { - for (const path of keys.get(key) ?? []) { + for (const path of observer.owners(key)) { root.inventory.invalidate(path) if (dependency.metadata) { this.metadata.add(path) @@ -193,28 +177,16 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext if (change.force) { this.forced.add(change.file.path) } - for (const { dependency } of auxiliaries) { - const key = dependency.fileKey?.(change.file.path) - if (!key) { - continue - } - const keys = root.dependencyKeys.get(dependency.path)! - const paths = keys.get(key) ?? new Set() - paths.add(change.file.path) - keys.set(key, paths) + for (const observer of root.dependencies.values()) { + observer.addFile(change.file.path) } } + for (const path of delta.removedDirectories) { + this.artifactMaps.get(args.rootDir)?.delete(path) + } for (const path of delta.removed) { - for (const dependency of args.searchDependencies ?? []) { - const key = dependency.fileKey?.(path) - if (key) { - const keys = root.dependencyKeys.get(dependency.path) - const paths = keys?.get(key) - paths?.delete(path) - if (!paths?.size) { - keys?.delete(key) - } - } + for (const observer of root.dependencies.values()) { + observer.removeFile(path) } this.removed.add(path) } @@ -225,65 +197,26 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext } } - async container( + container( key: string, paths: readonly string[], - read: () => Promise + read: (paths: readonly string[], limit: number) => Promise, + _limit: number, + issues: readonly AiVaultScanIssue[] ): Promise { - const signature = ( - await Promise.all( - paths - .flatMap((path) => [path, `${path}-wal`]) - .map(async (path) => { - try { - const stat = await wslGatedStat(path, 'scan', this.owner.signal) - return `${path}:${stat.dev}:${stat.ino}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}` - } catch (error) { - throwIfAiVaultScanCancelled(this.owner.signal) - return `${path}:${isMissingTranscriptPathError(error) ? 'missing' : 'unknown'}` - } - }) - ) - ).join('|') - const old = this.containers.get(key) - if (!this.force && old?.signature === signature && !signature.includes(':unknown')) { - return { agent: old.discovery.agent, rootDir: old.discovery.rootDir, files: [] } - } - const discovery = await read() - const membership = new Set() - const idsByContainer = new Map>() - for (const file of discovery.files) { - const synthetic = splitSyntheticSessionSource(file.path) - if (!synthetic) { - continue - } - membership.add(synthetic.id) - const ids = idsByContainer.get(synthetic.container) ?? new Set() - ids.add(synthetic.id) - idsByContainer.set(synthetic.container, ids) - } - if (discovery.complete === false) { - return discovery - } - for (const [path, ids] of idsByContainer) { - this.enumeratedContainers.set(path, ids) - } - if (old) { - const current = new Set(discovery.files.map((file) => file.path)) - for (const file of old.discovery.files) { - if (!current.has(file.path)) { - this.removed.add(file.path) - } - } - } - this.containers.set(key, { signature, discovery, membership }) - return discovery + return this.containers.discover(key, paths, read, issues, this.owner.signal) } async end(): Promise { await this.watchers.end() + this.containers.end() for (const key of this.roots.keys()) { if (!this.seen.has(key)) { + const root = this.roots.get(key)! + for (const path of root.inventory.files.keys()) { + this.removed.add(path) + } + this.artifactMaps.delete(root.inventory.root) this.roots.delete(key) } } @@ -293,6 +226,7 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext this.owner.abort() this.roots.clear() this.containers.clear() + this.artifactMaps.clear() return this.watchers.close() } } diff --git a/src/main/ai-vault-search/session-search-discovery-recovery.test.ts b/src/main/ai-vault-search/session-search-discovery-recovery.test.ts new file mode 100644 index 00000000000..e2e73687eec --- /dev/null +++ b/src/main/ai-vault-search/session-search-discovery-recovery.test.ts @@ -0,0 +1,287 @@ +import './session-search-worker.test-fixture' +import { + notifySessionSearchFileChange, + notifySessionSearchOverflow +} from './session-search-watcher.test-fixture' +import { appendFile, mkdir, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { expect, it, vi } from 'vitest' +import * as filesystem from '../native-chat/wsl-transcript-fs-access' +import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache' +import * as policy from './session-search-filesystem' +import { SessionSearchIndexer } from './session-search-indexer' +import { + FakeSessionSearchClock, + openSessionSearchIndexerHarness, + writeClaudeTranscript, + claudeLines +} from './session-search-indexer-test-fixture' + +it.each([100, 1000])( + 'reads appended bytes after a delivered update to a recent %s-turn transcript', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-recent-append') + const clock = new FakeSessionSearchClock() + const path = join(harness.claudeProjectDir, 'active.jsonl') + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + const reads = vi.spyOn(parser, 'parseAgentSessionFileCached') + try { + await writeClaudeTranscript( + path, + Array.from({ length: population }, () => 'archivedbody'), + 'active' + ) + await subject.start() + for (let cycle = 0; cycle < 3; cycle++) { + reads.mockClear() + const append = `${claudeLines(['appendedneedle'], 'active', population * 2 + cycle * 2).join('\n')}\n` + await appendFile(path, append) + notifySessionSearchFileChange(path) + clock.advance(20000) + await subject.settled() + const call = reads.mock.calls.find(([candidate]) => candidate.file.path === path) + expect(call).toBeDefined() + // The parser stream needs only appended records; its fixed continuity probe does not grow with history. + expect(call?.[2]?.bytesRead).toBeLessThanOrEqual(Buffer.byteLength(append) + 65536) + expect(call?.[2]?.fullParses).toBe(0) + } + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'appendedneedle'" + ) + .get() + ) + ).toEqual({ n: 6 }) + } finally { + reads.mockRestore() + await subject.close() + await harness.cleanup() + } + } +) + +it('retries a refused first observation after settlement without another filesystem event', async () => { + const harness = await openSessionSearchIndexerHarness('ss-stat-retry') + const clock = new FakeSessionSearchClock(), + path = join(harness.claudeProjectDir, 'recovered.jsonl') + const actual = filesystem.wslGatedStat + let refused = false + const probe = vi.spyOn(filesystem, 'wslGatedStat').mockImplementation((observed, ...rest) => { + if (observed === path && !refused) { + refused = true + return Promise.reject( + Object.assign(new Error('fixture temporary stat refusal'), { code: 'EIO' }) + ) + } + return actual(observed, ...rest) + }) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + try { + await writeClaudeTranscript(path, ['recoveredneedle'], 'recovered') + await subject.start() + expect(refused).toBe(true) + clock.advance(20000) + await subject.settled() + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'recoveredneedle'" + ) + .get() + ) + ).toEqual({ n: 2 }) + } finally { + probe.mockRestore() + await subject.close() + await harness.cleanup() + } +}) + +it('advances the discovery suffix during continuous updates and repeated overflow', async () => { + const harness = await openSessionSearchIndexerHarness('ss-crawl-fairness') + const clock = new FakeSessionSearchClock(), + root = harness.roots.claudeProjectsDir! + const active = join(harness.claudeProjectDir, 'active.jsonl') + const listings = vi.spyOn(filesystem, 'wslGatedReaddir') + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock, + discoveryDeadlineMs: 0 + }) + try { + await writeClaudeTranscript(active, ['activebody'], 'active') + for (let index = 0; index < 16; index++) { + await writeClaudeTranscript( + join(root, `bucket${index}`, 'history.jsonl'), + ['frontierneedle'], + `history${index}` + ) + } + await subject.start() + listings.mockClear() + for (let cycle = 0; cycle < 64; cycle++) { + await appendFile( + active, + `${claudeLines(['activeappend'], 'active', cycle * 2 + 10).join('\n')}\n` + ) + notifySessionSearchFileChange(active) + notifySessionSearchOverflow(root) + clock.advance(20000) + await subject.settled() + } + expect( + harness.read((db) => + db + .prepare( + "SELECT count(DISTINCT session_row_id) AS n FROM messages JOIN messages_fts ON messages.id=messages_fts.rowid WHERE messages_fts MATCH 'frontierneedle'" + ) + .get() + ) + ).toEqual({ n: 16 }) + // An epoch covers sixteen child directories; sixty-four overflow signals cannot restart its root prefix. + expect(listings.mock.calls.filter(([path]) => path === root).length).toBeLessThanOrEqual(4) + } finally { + listings.mockRestore() + await subject.close() + await harness.cleanup() + } +}) + +it.each([40, 400])( + 'polls %s retained files without rereading unchanged transcripts and retires a deletion', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-polling-delta') + const clock = new FakeSessionSearchClock() + const path = join(harness.claudeProjectDir, 'changed.jsonl') + const deleted = join(harness.claudeProjectDir, 'deleted.jsonl') + const polling = vi.spyOn(policy, 'sessionSearchRequiresPolling').mockResolvedValue(true) + const reads = vi.spyOn(parser, 'parseAgentSessionFileCached') + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + try { + await writeClaudeTranscript(path, ['originalbody'], 'changed') + await writeClaudeTranscript(deleted, ['deletedneedle'], 'deleted') + for (let index = 0; index < population; index++) { + await writeClaudeTranscript( + join(harness.claudeProjectDir, `${index}.jsonl`), + ['archivebody'], + `archive${index}` + ) + } + await subject.start() + reads.mockClear() + clock.advance(20000) + await subject.settled() + // Polling must observe membership, but unchanged observations create no transcript read work. + expect(reads).not.toHaveBeenCalled() + await appendFile(path, `${claudeLines(['pollchangedneedle'], 'changed', 10).join('\n')}\n`) + await rm(deleted) + clock.advance(20000) + await subject.settled() + expect(reads).toHaveBeenCalledTimes(1) + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'pollchangedneedle'" + ) + .get() + ) + ).toEqual({ n: 2 }) + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'deletedneedle'" + ) + .get() + ) + ).toEqual({ n: 0 }) + } finally { + reads.mockRestore() + polling.mockRestore() + await subject.close() + await harness.cleanup() + } + } +) + +it('observes remote message-only edits through polling without a native notification', async () => { + const harness = await openSessionSearchIndexerHarness('ss-polling-content') + const clock = new FakeSessionSearchClock() + const storage = harness.roots.opencodeStorageDir! + const primary = join(storage, 'session', 'project', 'ses_remote.json') + const messages = join(storage, 'message', 'ses_remote') + const polling = vi.spyOn(policy, 'sessionSearchRequiresPolling').mockResolvedValue(true) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + try { + await mkdir(join(storage, 'session', 'project'), { recursive: true }) + await mkdir(messages, { recursive: true }) + await writeFile(primary, JSON.stringify({ id: 'ses_remote', directory: '/repo/remote' })) + await writeFile( + join(messages, 'msg_1.json'), + JSON.stringify({ + role: 'user', + content: [{ type: 'text', text: 'originalremote' }], + time: { created: 1777634000000 } + }) + ) + await subject.start() + await writeFile( + join(messages, 'msg_2.json'), + JSON.stringify({ + role: 'assistant', + content: [{ type: 'text', text: 'remoteinputneedle' }], + time: { created: 1777634001000 } + }) + ) + clock.advance(20000) + await subject.settled() + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'remoteinputneedle'" + ) + .get() + ) + ).toEqual({ n: 1 }) + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'originalremote'" + ) + .get() + ) + ).toEqual({ n: 1 }) + } finally { + polling.mockRestore() + await subject.close() + await harness.cleanup() + } +}) diff --git a/src/main/ai-vault-search/session-search-file-rows.ts b/src/main/ai-vault-search/session-search-file-rows.ts index dedf54d59aa..d5d0ad89dbf 100644 --- a/src/main/ai-vault-search/session-search-file-rows.ts +++ b/src/main/ai-vault-search/session-search-file-rows.ts @@ -2,20 +2,22 @@ import type SyncDatabase from '../sqlite/sync-database' import type { SessionSearchFileRow } from './session-search-store-types' export function readSessionSearchFileRows(db: SyncDatabase, path?: string): SessionSearchFileRow[] { - return ( - db - .prepare( - // Numeric stat IDs may exceed SQLite's safe INTEGER-to-number read range. - `SELECT f.path, f.byte_offset AS byteOffset, f.prefix_hash AS prefixHash, s.agent, CAST(f.dev AS REAL) AS dev, CAST(f.ino AS REAL) AS ino, + const statement = db.prepare( + // Numeric stat IDs may exceed SQLite's safe INTEGER-to-number read range. + `SELECT f.path, f.byte_offset AS byteOffset, f.prefix_hash AS prefixHash, s.agent, CAST(f.dev AS REAL) AS dev, CAST(f.ino AS REAL) AS ino, mtime_ms AS mtimeMs, size_bytes AS sizeBytes, state, fail_count AS failCount, failed_mtime_ms AS failedMtimeMs FROM files f LEFT JOIN sessions s ON s.id=f.session_row_id${path === undefined ? '' : ' WHERE f.path = ?'}` - ) - .all(...(path === undefined ? [] : [path])) as (Omit & { - dev: number | null - ino: number | null - })[] - ).map((row) => ({ + ) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this query explicitly selects the file-row fields and nullable numeric identity columns. + const rows = statement.all(...(path === undefined ? [] : [path])) as (Omit< + SessionSearchFileRow, + 'identity' + > & { + dev: number | null + ino: number | null + })[] + return rows.map((row) => ({ path: row.path, byteOffset: row.byteOffset, prefixHash: row.prefixHash, diff --git a/src/main/ai-vault-search/session-search-filesystem.ts b/src/main/ai-vault-search/session-search-filesystem.ts index b0a7a744b5c..cac1e15bdf0 100644 --- a/src/main/ai-vault-search/session-search-filesystem.ts +++ b/src/main/ai-vault-search/session-search-filesystem.ts @@ -1,10 +1,8 @@ -import { execFile } from 'node:child_process' +import { runProcess } from '@orca/process-host' import { statfs } from 'node:fs/promises' -import { promisify } from 'node:util' import { isWslUncPath } from '../../shared/wsl-paths' -const run = promisify(execFile) -let mounts: { at: number; result: Promise } | undefined +let mounts: { at: number; result: Promise } | undefined /** Native notifications cannot establish freshness when another host writes a shared mount. */ export async function sessionSearchRequiresPolling(path: string): Promise { @@ -16,17 +14,20 @@ export async function sessionSearchRequiresPolling(path: string): Promise 1_000) { mounts = { at: Date.now(), - result: run('/sbin/mount', [], { timeout: 2_000, maxBuffer: 256 * 1024 }) - .then(({ stdout }) => - stdout.split('\n').flatMap((line) => { - const match = / on (.+) \((?:nfs|smbfs|webdav|osxfuse|fusefs)[,)]/.exec(line) - return match ? [match[1]] : [] - }) + result: runProcess({ program: '/sbin/mount', timeoutMs: 2_000, maxOutputBytes: 256 * 1024 }) + .then(({ code, outputTruncated, stdout }) => + code !== 0 || outputTruncated + ? undefined + : stdout.split('\n').flatMap((line) => { + const match = / on (.+) \((?:nfs|smbfs|webdav|osxfuse|fusefs)[,)]/.exec(line) + return match ? [match[1]] : [] + }) ) - .catch(() => []) + .catch(() => undefined) } } - return (await mounts.result).some((root) => path === root || path.startsWith(`${root}/`)) + const shared = await mounts.result + return !shared || shared.some((root) => path === root || path.startsWith(`${root}/`)) } if (process.platform === 'linux') { try { @@ -34,7 +35,7 @@ export async function sessionSearchRequiresPolling(path: string): Promise path === root || path.startsWith(`${root}/`)).length ).toBeLessThanOrEqual(16) expect( - readdir.mock.calls.filter(([path]) => path === root || path.startsWith(`${root}/`)) + readdir.mock.calls.filter( + ([observed]) => + observed !== subagentTranscriptsDirFor(path) && + (observed === root || observed.startsWith(`${root}/`)) + ) ).toHaveLength(0) expect( harness.read((db) => @@ -76,7 +81,7 @@ it.each([40, 400])( ) .get() ) - ).toEqual({ n: 1 }) + ).toEqual({ n: 2 }) } finally { await subject.close() stat.mockRestore() @@ -157,7 +162,7 @@ it('indexes a delivered equal-stat rewrite outside the recent window', async () ) .get() ) - ).toEqual({ n: 1 }) + ).toEqual({ n: 2 }) expect( harness.read((db) => db diff --git a/src/main/ai-vault-search/session-search-indexer.test.ts b/src/main/ai-vault-search/session-search-indexer.test.ts index 330f3eae2bd..d26ee05ec71 100644 --- a/src/main/ai-vault-search/session-search-indexer.test.ts +++ b/src/main/ai-vault-search/session-search-indexer.test.ts @@ -595,7 +595,7 @@ it('retains a deferred recent file after it leaves the top-N window', async () = const paths = ids.map((id) => transcriptPath(id)) await writeClaudeTranscript(paths[0], ['deferredreviewneedle'], ids[0]) await writeClaudeTranscript(paths[1], ['firstreviewneedle'], ids[1]) - const time = Date.now() + const time = Date.UTC(2099, 0, 1) await utimes(paths[0], new Date(time), new Date(time)) await utimes(paths[1], new Date(time + 1000), new Date(time + 1000)) await nextCycle() diff --git a/src/main/ai-vault-search/session-search-metadata-admission.test.ts b/src/main/ai-vault-search/session-search-metadata-admission.test.ts new file mode 100644 index 00000000000..a19ab7777ff --- /dev/null +++ b/src/main/ai-vault-search/session-search-metadata-admission.test.ts @@ -0,0 +1,140 @@ +import './session-search-worker.test-fixture' +import { notifySessionSearchFileChange } from './session-search-watcher.test-fixture' +import { mkdir, stat, utimes, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { expect, it, vi } from 'vitest' +import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache' +import * as databaseReader from '../ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-open' +import { resetDevinSessionsIndexCacheForTests } from '../ai-vault/scanners/devin/session-scanner-devin-db' +import { + writeAntigravityHistory, + writeAntigravityTranscript +} from '../ai-vault/session-scanner-test-fixtures' +import SyncDatabase from '../sqlite/sync-database' +import { SessionSearchIndexer } from './session-search-indexer' +import { + FakeSessionSearchClock, + openSessionSearchIndexerHarness +} from './session-search-indexer-test-fixture' + +it('does not add a database-only Devin search wake and still enriches an admitted transcript read', async () => { + const harness = await openSessionSearchIndexerHarness('ss-devin-admission') + const clock = new FakeSessionSearchClock() + const root = harness.roots.devinTranscriptsDir! + const path = join(root, 'shown.json') + const dbPath = join(dirname(root), 'sessions.db') + await mkdir(root, { recursive: true }) + const db = new SyncDatabase(dbPath) + db.exec( + "CREATE TABLE sessions (id TEXT PRIMARY KEY, title TEXT, hidden INTEGER); INSERT INTO sessions VALUES ('shown', 'Original title', 0)" + ) + const write = (text: string) => + writeFile( + path, + JSON.stringify({ session_id: 'shown', steps: [{ source: 'user', message: text }] }) + ) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + const reads = vi.spyOn(parser, 'parseAgentSessionFileCached') + const accessor = vi.spyOn(databaseReader, 'readOpenCodeDatabase') + try { + await write('originaldevinneedle') + await subject.start() + reads.mockClear() + accessor.mockClear() + db.exec("UPDATE sessions SET title='Changed title'") + const previous = await stat(dbPath) + await utimes(dbPath, new Date(previous.mtimeMs + 1000), new Date(previous.mtimeMs + 1000)) + notifySessionSearchFileChange(dbPath) + clock.advance(20000) + await subject.settled() + expect(reads).not.toHaveBeenCalled() + expect(accessor).not.toHaveBeenCalled() + expect( + harness.read((db) => db.prepare("SELECT title FROM sessions WHERE session_id='shown'").get()) + ).toEqual({ title: 'Original title' }) + await write('changeddevinneedle') + notifySessionSearchFileChange(path) + clock.advance(20000) + await subject.settled() + expect( + harness.read((db) => db.prepare("SELECT title FROM sessions WHERE session_id='shown'").get()) + ).toEqual({ title: 'Changed title' }) + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'changeddevinneedle'" + ) + .get() + ) + ).toEqual({ n: 1 }) + } finally { + reads.mockRestore() + accessor.mockRestore() + db.close() + resetDevinSessionsIndexCacheForTests() + await subject.close() + await harness.cleanup() + } +}) + +it('does not fan out search reads for history metadata outside the Antigravity search parser', async () => { + const harness = await openSessionSearchIndexerHarness('ss-history-admission') + const clock = new FakeSessionSearchClock() + const root = harness.roots.antigravityBrainDir! + const id = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff' + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + const reads = vi.spyOn(parser, 'parseAgentSessionFileCached') + try { + const path = await writeAntigravityTranscript(root, id, [ + { + source: 'USER_EXPLICIT', + type: 'USER_INPUT', + created_at: '2026-07-15T11:39:10.000Z', + content: 'historyadmissionneedle' + } + ]) + await subject.start() + reads.mockClear() + await writeAntigravityHistory(root, [ + { display: 'historyadmissionneedle', timestamp: 1784115550100, workspace: '/repo/changed' } + ]) + notifySessionSearchFileChange(join(dirname(root), 'history.jsonl')) + clock.advance(20000) + await subject.settled() + expect(reads).not.toHaveBeenCalled() + expect(harness.read((db) => db.prepare('SELECT cwd FROM sessions').get())).toEqual({ + cwd: null + }) + await writeFile( + path, + `${JSON.stringify({ source: 'USER_EXPLICIT', type: 'USER_INPUT', created_at: '2026-07-15T11:39:10.000Z', content: 'changedhistoryneedle' })}\n` + ) + notifySessionSearchFileChange(path) + clock.advance(20000) + await subject.settled() + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'changedhistoryneedle'" + ) + .get() + ) + ).toEqual({ n: 1 }) + } finally { + reads.mockRestore() + await subject.close() + await harness.cleanup() + } +}) diff --git a/src/main/ai-vault-search/session-search-opencode-index.test.ts b/src/main/ai-vault-search/session-search-opencode-index.test.ts index 72250fd7ff8..d4e99aecffb 100644 --- a/src/main/ai-vault-search/session-search-opencode-index.test.ts +++ b/src/main/ai-vault-search/session-search-opencode-index.test.ts @@ -1,9 +1,12 @@ import './session-search-worker.test-fixture' +import { rename, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { afterEach, beforeEach, expect, it, vi } from 'vitest' // Replace worker transport while exercising the real SQLite readers. export const openCodeReadCalls: string[] = [] +export const openCodeListObservations: { paths: readonly string[]; limit: number; rows: number }[] = + [] vi.mock( '../ai-vault/scanners/opencode/reader-transport/session-scanner-opencode-sqlite-worker-spawn', async () => { @@ -16,12 +19,23 @@ vi.mock( const own = await import('./session-search-opencode-index.test') const { listOpenCode2SqliteSessions } = await import('../ai-vault/scanners/opencode2/session-scanner-opencode2-sqlite-list') + const parse2 = await import('../ai-vault/scanners/opencode2/session-scanner-opencode2-sqlite') return { + parseOpenCode2SqliteSessionViaWorker: parse2.parseOpenCode2SqliteSession, + captureOpenCode2SqliteSessionViaWorker: parse2.captureOpenCode2SqliteSession, listOpenCode2SqliteSessionsViaWorker: listOpenCode2SqliteSessions, resolveOpenCodeSqliteWorkerEntryPath: () => null, listOpenCodeSqliteSessionsViaWorker: ( args: Parameters[0] - ) => list.listOpenCodeSqliteSessions(args), + ) => + list.listOpenCodeSqliteSessions(args).then((candidates) => { + own.openCodeListObservations.push({ + paths: args.dbPaths, + limit: args.limit, + rows: candidates.length + }) + return candidates + }), parseOpenCodeSqliteSessionViaWorker: ( args: Parameters[0] ) => { @@ -43,7 +57,7 @@ import { appendOpenCodeSqliteTurn, writeOpenCodeSqliteDatabase } from '../ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-fixture' -import type SyncDatabase from '../sqlite/sync-database' +import SyncDatabase from '../sqlite/sync-database' import { SessionSearchEngine } from './session-search-engine' import { SessionSearchIndexer } from './session-search-indexer' import { openSessionSearchDatabase } from './session-search-schema' @@ -73,6 +87,7 @@ beforeEach(async () => { indexer = null engineDbs = [] openCodeReadCalls.length = 0 + openCodeListObservations.length = 0 }) afterEach(async () => { @@ -176,3 +191,152 @@ it('re-reads a session that gained a message and replaces its rows', async () => expect(engine.search({ query: ANSWER }).hits).toHaveLength(1) expect(openCodeReadCalls.filter((call) => call === `capture:${SESSION}`)).toHaveLength(2) }) + +it.each([40, 400])( + 'keeps recent database observations bounded at %s rows while old imports reach search on maintenance', + async (population) => { + writeOpenCodeSqliteDatabase( + dbPath(), + Array.from({ length: population }, (_, index) => ({ + id: `ses_${index}`, + turns: [{ role: 'user', parts: ['retainedarchive'] }] + })) + ) + const subject = await startIndexer() + appendOpenCodeSqliteTurn(dbPath(), 'ses_0', { + role: 'assistant', + parts: ['recentdatabaseneedle'] + }) + clock.advance(20000) + await subject.settled() + expect(openEngine().search({ query: 'recentdatabaseneedle' }).hits[0]?.sessionId).toBe('ses_0') + expect(openCodeListObservations.at(-1)).toMatchObject({ limit: 12, rows: 12 }) + writeOpenCodeSqliteDatabase(dbPath(), [ + { id: 'ses_imported', turns: [{ role: 'user', parts: ['oldimportneedle'] }] } + ]) + const source = new SyncDatabase(dbPath()) + source + .prepare("UPDATE session SET time_created=1000, time_updated=2000 WHERE id='ses_imported'") + .run() + source.close() + for (let cycle = 0; cycle < 14; cycle++) { + clock.advance(20000) + await subject.settled() + } + expect( + openCodeListObservations.filter((observation) => observation.limit === Infinity) + ).toHaveLength(2) + expect( + openCodeListObservations + .filter((observation) => observation.limit === 12) + .every((observation) => observation.rows === 12) + ).toBe(true) + expect(openEngine().search({ query: 'oldimportneedle' }).hits[0]?.sessionId).toBe( + 'ses_imported' + ) + } +) + +it('keeps a duplicate physical row and promotes it when the selected database disappears', async () => { + const alternative = join(harness.root, 'opencode-db', 'opencode-alternative.db') + writeOpenCodeSqliteDatabase(dbPath(), [ + { id: 'ses_shared', turns: [{ role: 'user', parts: ['winningcopyneedle'] }] }, + { id: 'ses_primary', turns: [{ role: 'user', parts: ['primaryonlyneedle'] }] } + ]) + writeOpenCodeSqliteDatabase(alternative, [ + { id: 'ses_shared', turns: [{ role: 'user', parts: ['alternativecopyneedle'] }] }, + { id: 'ses_alternative', turns: [{ role: 'user', parts: ['alternativeonlyneedle'] }] } + ]) + let paths = [dbPath(), alternative] + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: { ...harness.roots, opencodeDbPaths: paths }, + resolveRoots: async () => ({ ...harness.roots, opencodeDbPaths: paths }), + historyDays: null, + clock + }) + indexer = subject + await subject.start() + await subject.reconcile({ full: true }) + expect(openEngine().search({ query: 'winningcopyneedle' }).hits).toHaveLength(1) + expect(openEngine().search({ query: 'alternativecopyneedle' }).hits).toHaveLength(0) + await rm(dbPath()) + paths = [alternative] + await subject.reconcile({ full: true }) + expect(openEngine().search({ query: 'alternativecopyneedle' }).hits[0]?.sessionId).toBe( + 'ses_shared' + ) + expect(openEngine().search({ query: 'winningcopyneedle' }).hits).toHaveLength(0) + expect(openEngine().search({ query: 'primaryonlyneedle' }).hits).toHaveLength(0) +}) + +it('retains refused database membership while healthy owners settle and retries only the refused full observation', async () => { + const alternative = join(harness.root, 'opencode-db', 'opencode-alternative.db') + writeOpenCodeSqliteDatabase(dbPath(), [ + { id: 'ses_primary', turns: [{ role: 'user', parts: ['preservedneedle'] }] } + ]) + writeOpenCodeSqliteDatabase(alternative, [ + { id: 'ses_alternative', turns: [{ role: 'user', parts: ['healthyneedle'] }] } + ]) + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: { ...harness.roots, opencodeDbPaths: [dbPath(), alternative] }, + historyDays: null, + clock + }) + indexer = subject + await subject.start() + await rename(dbPath(), `${dbPath()}.saved`) + await writeFile(dbPath(), 'temporarily unreadable sqlite') + appendOpenCodeSqliteTurn(alternative, 'ses_alternative', { + role: 'assistant', + parts: ['healthyupdateneedle'] + }) + await subject.reconcile({ full: true }) + expect(openEngine().search({ query: 'preservedneedle' }).hits).toHaveLength(1) + expect(openEngine().search({ query: 'healthyupdateneedle' }).hits).toHaveLength(1) + await rm(dbPath()) + await rename(`${dbPath()}.saved`, dbPath()) + openCodeListObservations.length = 0 + clock.advance(20000) + await subject.settled() + expect( + openCodeListObservations.filter((observation) => observation.paths[0] === alternative) + ).toEqual([{ paths: [alternative], limit: 12, rows: 1 }]) + expect( + openCodeListObservations.filter((observation) => observation.paths[0] === dbPath()) + ).toEqual([{ paths: [dbPath()], limit: Infinity, rows: 1 }]) + expect(openEngine().search({ query: 'preservedneedle' }).hits).toHaveLength(1) +}) + +it('keeps full membership proofs separate when v1 and v2 share one physical database', async () => { + writeVault() + const subject = await startIndexer() + expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1) + const db = new SyncDatabase(dbPath()) + db.exec(`CREATE TABLE session_v2 (id TEXT PRIMARY KEY, time_created INTEGER, time_updated INTEGER); + INSERT INTO session_v2 VALUES ('ses_capture', 1740000000000, 1740000010000)`) + db.close() + const changed = new SyncDatabase(dbPath()) + changed.prepare('DELETE FROM session WHERE id=?').run(SESSION) + changed.close() + await subject.reconcile({ full: true }) + // The v2 copy of this ID is not positive membership evidence for the removed v1 row. + expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(0) + expect(openEngine().search({ query: OTHER }).hits).toHaveLength(1) +}) + +it('keeps indexed rows when an existing physical database becomes empty or unrecognized', async () => { + writeVault() + const subject = await startIndexer() + const db = new SyncDatabase(dbPath()) + db.exec('DELETE FROM session') + db.close() + await subject.reconcile({ full: true }) + expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1) + const changed = new SyncDatabase(dbPath()) + changed.exec('DROP TABLE session') + changed.close() + await subject.reconcile({ full: true }) + expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1) +}) diff --git a/src/main/ai-vault-search/session-search-pass.ts b/src/main/ai-vault-search/session-search-pass.ts index b3a90ee9415..0af260ab52f 100644 --- a/src/main/ai-vault-search/session-search-pass.ts +++ b/src/main/ai-vault-search/session-search-pass.ts @@ -17,6 +17,7 @@ import { runSessionSearchIndexPass } from './session-search-index-pass' import { discoverSessionSearchCandidates, sessionSearchEmptiedRoots, + isUnderScanRoot, type SessionSearchScanRoots } from './session-search-scan-roots' import type { SessionSearchIndexStore, SessionSearchFileRow } from './session-search-store' @@ -30,6 +31,7 @@ export class SessionSearchPassState { readonly queue = new SessionSearchWorkQueue() retirement: Iterator | null = null readonly retire = new Set() + readonly parkedRetire = new Set() needsRetirement = true constructor(entryPath?: string) { this.discovery = new SessionSearchDiscoveryInventory(entryPath) @@ -82,31 +84,47 @@ export async function runSessionSearchPass( } state.discovery.begin({ full: args.full, + maintenance: args.purge ?? args.full, recentLimit: args.recentPerAgent, overdue: args.discoveryOverdue ?? (() => false) }) if (args.full) { state.needsRetirement = true } + if (args.full || args.purge) { + for (const path of state.parkedRetire) { + state.retire.add(path) + } + state.parkedRetire.clear() + } return withCursorChatMetaScan(async () => { const swept = await withSessionDiscovery(state.discovery, () => discoverSessionSearchCandidates(args.roots, { limitPerAgent: Infinity, signal, raw: true }) ) + await state.discovery.end() const issues: AiVaultScanIssue[] = [...swept.issues] const update = await state.candidates.update( swept.discoveries, state.discovery.removed, { ...args.roots, signal }, - state.discovery.forced + state.discovery.forced, + state.discovery.removedCandidates ) for (const path of update.suppressed) { state.queue.remove(path) } - for (const path of state.discovery.removed) { + for (const path of new Set([ + ...state.discovery.removed, + ...state.discovery.removedCandidates.keys() + ])) { + if (state.candidates.has(path, store.fileRow(path)?.agent)) { + continue + } state.queue.remove(path) state.retire.add(path) } for (const candidate of update.candidates) { + state.parkedRetire.delete(candidate.file.path) state.queue.enqueue( candidate, store, @@ -128,7 +146,7 @@ export async function runSessionSearchPass( ? sessionSearchEmptiedRoots(args.previousRootsWithFiles, rootsWithFiles) : new Set() const degraded: SessionSearchDegradedRoot[] = [] - if (!state.discovery.pending && state.needsRetirement) { + if (!state.discovery.walking && state.needsRetirement) { state.retirement ??= store.fileRows() state.needsRetirement = false } @@ -150,7 +168,7 @@ export async function runSessionSearchPass( state.retirement = null break } - if (!state.candidates.has(row.value.path)) { + if (!state.candidates.has(row.value.path, row.value.agent)) { paths.push(row.value.path) } } @@ -170,9 +188,16 @@ export async function runSessionSearchPass( directoryLimit: 512, signal }) - for (const path of [...retirement.unverifiable, ...retirement.unchecked]) { + for (const path of retirement.unchecked) { retry.add(path) } + for (const path of retirement.unverifiable) { + if ([...emptiedRoots].some((root) => isUnderScanRoot(path, root))) { + retry.add(path) + } else { + state.parkedRetire.add(path) + } + } degraded.push(...retirement.degradedRoots) // Retry refused paths next cycle, but keep advancing the older retirement frontier now. batches++ @@ -192,7 +217,6 @@ export async function runSessionSearchPass( signal ) await store.flush?.() - await state.discovery.end() return { rootsWithFiles, degradedRoots: mergeDegradedRoots( diff --git a/src/main/ai-vault-search/session-search-root-watchers.ts b/src/main/ai-vault-search/session-search-root-watchers.ts index bf373713a44..4867b605990 100644 --- a/src/main/ai-vault-search/session-search-root-watchers.ts +++ b/src/main/ai-vault-search/session-search-root-watchers.ts @@ -10,6 +10,8 @@ export type SessionSearchRootWatch = { failed: boolean events: Map reconcile: boolean + attempts: number + retryAt: number } export class SessionSearchRootWatchers { @@ -36,10 +38,15 @@ export class SessionSearchRootWatchers { this.watches.clear() return this.supervisor.disposeAndWait() } - watch(path: string, signal?: AbortSignal): Promise { - let probe = this.probes.get(path) + watch( + path: string, + signal?: AbortSignal, + mode: 'recursive' | 'shallow' = 'recursive' + ): Promise { + const key = `${mode}:${path}` + let probe = this.probes.get(key) if (!probe) { - probe = this.probeWatch(path, signal).then((watch) => { + probe = this.probeWatch(path, key, mode, signal).then((watch) => { if (!watch) { return undefined } @@ -48,28 +55,38 @@ export class SessionSearchRootWatchers { watch.reconcile = false return delivery }) - this.probes.set(path, probe) + this.probes.set(key, probe) } return probe } private async probeWatch( path: string, + key: string, + mode: 'recursive' | 'shallow', signal?: AbortSignal ): Promise { + const previous = this.watches.get(key) let identity: string try { const stat = await wslGatedStat(path, 'scan', signal) if (!stat.isDirectory()) { + this.watches.delete(key) + await previous?.subscription?.unsubscribe() return undefined } identity = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}` } catch { throwIfAiVaultScanCancelled(signal) + this.watches.delete(key) + await previous?.subscription?.unsubscribe().catch(() => undefined) return undefined } - const previous = this.watches.get(path) - if (previous && !previous.failed && previous.identity === identity) { + if ( + previous && + previous.identity === identity && + (!previous.failed || Date.now() < previous.retryAt) + ) { return previous } await previous?.subscription?.unsubscribe().catch(() => undefined) @@ -77,10 +94,15 @@ export class SessionSearchRootWatchers { identity, failed: false, events: new Map(), - reconcile: true + reconcile: true, + attempts: previous?.attempts ?? 0, + retryAt: 0 } - this.watches.set(path, watch) + this.watches.set(key, watch) const fail = () => { + if (!watch.failed) { + watch.retryAt = Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(watch.attempts++, 6)) + } watch.failed = true watch.reconcile = true } @@ -102,10 +124,12 @@ export class SessionSearchRootWatchers { watch.events.set(event.path, event) } }, - { mode: 'recursive' }, + { mode }, { signal: this.owner.signal, - onInterruption: fail, + onInterruption: () => { + watch.reconcile = true + }, onOverflow: () => { watch.reconcile = true }, @@ -115,9 +139,11 @@ export class SessionSearchRootWatchers { ) if (this.owner.signal.aborted) { await watch.subscription.unsubscribe() + } else if (!watch.failed) { + watch.attempts = 0 } } catch { - watch.failed = true + fail() throwIfAiVaultScanCancelled(this.owner.signal) } return watch diff --git a/src/main/ai-vault-search/session-search-watcher.test-fixture.ts b/src/main/ai-vault-search/session-search-watcher.test-fixture.ts index 94a10e681e2..c25199020dd 100644 --- a/src/main/ai-vault-search/session-search-watcher.test-fixture.ts +++ b/src/main/ai-vault-search/session-search-watcher.test-fixture.ts @@ -1,16 +1,29 @@ import { vi } from 'vitest' import { readdirSync } from 'node:fs' import { join } from 'node:path' +import type { WatcherProcessHooks } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription' import type { WatcherProcessCallback } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription' -const watchers = vi.hoisted(() => new Map()) +const watchers = vi.hoisted( + () => + new Map< + object, + { path: string; callback: WatcherProcessCallback; hooks: WatcherProcessHooks } + >() +) vi.mock('../ipc/filesystem/watcher-process/parcel-watcher-process-supervisor', () => ({ WatcherProcessSupervisor: class { - async subscribe(path: string, callback: WatcherProcessCallback) { - watchers.set(path, callback) + async subscribe( + path: string, + callback: WatcherProcessCallback, + _options: unknown, + hooks: WatcherProcessHooks = {} + ) { + const token = {} + watchers.set(token, { path, callback, hooks }) return { unsubscribe: async () => { - watchers.delete(path) + watchers.delete(token) } } } @@ -21,7 +34,7 @@ vi.mock('../ipc/filesystem/watcher-process/parcel-watcher-process-supervisor', ( })) export function notifySessionSearchRootChanges(): void { - for (const [path, callback] of watchers) { + for (const { path, callback } of watchers.values()) { const directories = [path] for (let index = 0; index < directories.length; index++) { try { @@ -42,9 +55,17 @@ export function notifySessionSearchRootChanges(): void { } export function notifySessionSearchFileChange(path: string): void { - for (const [root, callback] of watchers) { + for (const { path: root, callback } of watchers.values()) { if (path.startsWith(`${root}/`) || path.startsWith(`${root}\\`)) { callback(null, [{ path, type: 'update', isDirectory: false }]) } } } + +export function notifySessionSearchOverflow(path: string): void { + for (const watch of watchers.values()) { + if (watch.path === path) { + watch.hooks.onOverflow?.() + } + } +} diff --git a/src/main/ai-vault-search/session-search-worker.test-fixture.ts b/src/main/ai-vault-search/session-search-worker.test-fixture.ts index 3a3ef931776..6e2e6daa5bb 100644 --- a/src/main/ai-vault-search/session-search-worker.test-fixture.ts +++ b/src/main/ai-vault-search/session-search-worker.test-fixture.ts @@ -1,3 +1,4 @@ +// @orca-isolated-worker -- Its worker factory mock must be installed in each importing module graph. import type * as WorkerClient from './session-search-worker-client' import { createRequire } from 'node:module' import { resolve } from 'node:path' diff --git a/src/main/ai-vault/scanners/antigravity/session-scanner-antigravity-sources.ts b/src/main/ai-vault/scanners/antigravity/session-scanner-antigravity-sources.ts index 835691e34b5..f1c2d1272a6 100644 --- a/src/main/ai-vault/scanners/antigravity/session-scanner-antigravity-sources.ts +++ b/src/main/ai-vault/scanners/antigravity/session-scanner-antigravity-sources.ts @@ -3,7 +3,7 @@ import { completeAntigravityTranscriptPairs } from './antigravity-transcript-can import { recordAntigravityArtifactPath } from './antigravity-session-artifacts' import { ANTIGRAVITY_HISTORY_ROOTS } from '../../../../shared/antigravity-session-origin' import { homedir } from 'node:os' -import { basename, dirname, join } from 'node:path' +import { join } from 'node:path' import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types' import { isAntigravityTranscriptPath, @@ -49,15 +49,6 @@ export function antigravityDiscoveries( agent: 'antigravity', issues, extensions: ['.jsonl'], - searchDependencies: [ - { - path: dirname(rootDir), - metadata: true, - fileKey: () => 'history', - eventKey: (path) => - ['history.jsonl'].includes(basename(path)) || path.endsWith('.meta') ? 'history' : null - } - ], filePredicate: isAntigravityTranscriptPath, directoryPredicate: shouldDescendAntigravityBrainDirectory, onDirectoryEntries: (dirPath, entries) => diff --git a/src/main/ai-vault/scanners/codex/session-scanner-codex-index-read.ts b/src/main/ai-vault/scanners/codex/session-scanner-codex-index-read.ts index 1136965ac4f..17eeaa49315 100644 --- a/src/main/ai-vault/scanners/codex/session-scanner-codex-index-read.ts +++ b/src/main/ai-vault/scanners/codex/session-scanner-codex-index-read.ts @@ -1,93 +1,20 @@ -import { createHash } from 'node:crypto' import type { Stats } from 'node:fs' -import { - openTranscriptReadStream, - readTranscriptSlice -} from '../../../native-chat/wsl-transcript-fs-access' -import { consumeCompleteJsonlLines } from '../../session-scanner-jsonl-reader' +import { readSessionIndexJsonl, type SessionIndexJsonlRead } from '../../session-index-jsonl-read' import { extractString, parseJsonObject } from '../../session-scanner-values' import { normalizeTitleText } from '../../session-scanner-text-normalization' -export type CodexIndexRead = { - titles: Map - offset: number - readThrough: number - dev: number - ino: number - firstRecord: { length: number; hash: string } | null -} - -async function firstRecordSignature(path: string, end: number) { - const hash = createHash('sha256') - let length = 0 - for await (const chunk of openTranscriptReadStream(path, { end: end - 1 }, 'scan')) { - const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) - const newline = bytes.indexOf(0x0a) - const prefix = newline === -1 ? bytes : bytes.subarray(0, newline + 1) - hash.update(prefix) - length += prefix.length - if (newline !== -1) { - return { length, hash: hash.digest('hex') } - } - } - return null -} - +export type CodexIndexRead = SessionIndexJsonlRead & { titles: Map } export async function readCodexIndexUpdate( path: string, stat: Stats, previous?: CodexIndexRead ): Promise { - let append = false - if ( - previous?.firstRecord && - previous.offset === previous.readThrough && - stat.size > previous.readThrough && - stat.dev !== 0 && - stat.ino !== 0 && - previous.dev !== 0 && - previous.ino !== 0 && - stat.dev === previous.dev && - stat.ino === previous.ino - ) { - try { - const prefix = await firstRecordSignature(path, previous.firstRecord.length) - const boundary = await readTranscriptSlice(path, previous.offset - 1, 1, 'scan') - // Growth alone cannot distinguish an append from Codex replacing the index on removal. - append = prefix?.hash === previous.firstRecord.hash && boundary[0] === 0x0a - } catch { - // A failed continuity check requires a complete reread. - } - } - const titles = append ? new Map(previous!.titles) : new Map() - const applyLine = (line: string) => { - const record = parseJsonObject(line) - const sessionId = record && extractString(record.id) - const title = record && normalizeTitleText(extractString(record.thread_name) ?? '') - if (sessionId && title) { - titles.set(sessionId, title) - } - } - const start = append ? previous!.offset : 0 - const { consumedThrough, trailingPartialLine, bytesRead } = await consumeCompleteJsonlLines({ - path, - start, - onLine: applyLine - }) - // EOF JSON remains readable; a tail forces the next growth to reread rather than retain it. - if (trailingPartialLine) { - applyLine(trailingPartialLine) - } - return { - titles, - offset: consumedThrough, - readThrough: start + bytesRead, - dev: stat.dev, - ino: stat.ino, - firstRecord: append - ? previous!.firstRecord - : consumedThrough > 0 - ? await firstRecordSignature(path, consumedThrough) - : null - } + const read = await readSessionIndexJsonl(path, stat, previous, parseCodexTitleIndexRecord) + return { ...read, titles: read.values } +} +export function parseCodexTitleIndexRecord(line: string): readonly [string, string] | null { + const record = parseJsonObject(line) + const id = record && extractString(record.id) + const title = record && normalizeTitleText(extractString(record.thread_name) ?? '') + return id && title ? [id, title] : null } diff --git a/src/main/ai-vault/scanners/codex/session-scanner-codex-title-index.ts b/src/main/ai-vault/scanners/codex/session-scanner-codex-title-index.ts index 9b97da7117e..085fb6a3292 100644 --- a/src/main/ai-vault/scanners/codex/session-scanner-codex-title-index.ts +++ b/src/main/ai-vault/scanners/codex/session-scanner-codex-title-index.ts @@ -1,3 +1,5 @@ +import { createSessionJsonlDependency } from '../../session-jsonl-dependency' +import { parseCodexTitleIndexRecord } from './session-scanner-codex-index-read' import type { SessionDiscoveryDependency } from '../../session-discovery-context' import { AsyncLocalStorage } from 'node:async_hooks' import { basename, dirname, join } from 'node:path' @@ -186,10 +188,15 @@ export const codexSearchDependencies: (root: string) => readonly SessionDiscover ) => [ { path: dirname(root), + mode: 'shallow', eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'titles' : null), fileKey: (path) => /([0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})\.jsonl$/i.exec(path)?.[1] ?? '*', - snapshot: () => readCodexSessionIndexTitles(dirname(root)), + observe: () => + createSessionJsonlDependency( + join(dirname(root), CODEX_SESSION_INDEX_FILE), + parseCodexTitleIndexRecord + ), metadata: true } ] diff --git a/src/main/ai-vault/scanners/devin/session-scanner-devin-db.ts b/src/main/ai-vault/scanners/devin/session-scanner-devin-db.ts index 5b71588c1e4..8c54fab529b 100644 --- a/src/main/ai-vault/scanners/devin/session-scanner-devin-db.ts +++ b/src/main/ai-vault/scanners/devin/session-scanner-devin-db.ts @@ -1,9 +1,8 @@ -import type { SessionDiscoveryDependency } from '../../session-discovery-context' import { parseWslUncPath } from '../../../../shared/wsl-paths' import { wasWslDistroLastSeenRunningSettled } from '../../../wsl/discovery/wsl-background-distro-gate' import { WSL_TRANSCRIPT_RUNNING_LEASE_MS } from '../../../native-chat/wsl-transcript-running-admission' import { AsyncLocalStorage } from 'node:async_hooks' -import { basename, dirname, join } from 'node:path' +import { dirname, join } from 'node:path' import { wslGatedStat } from '../../../native-chat/wsl-transcript-fs-access' import { columnExists, tableExists } from '../../../opencode-usage/schema-helpers' import { readOpenCodeDatabase } from '../opencode/sqlite/session-scanner-opencode-sqlite-open' @@ -177,14 +176,3 @@ function unixSecondsToIso(value: unknown): string | null { const date = new Date(seconds * 1000) return Number.isFinite(date.getTime()) ? date.toISOString() : null } - -export const devinSearchDependencies: (root: string) => readonly SessionDiscoveryDependency[] = ( - root -) => [ - { - path: dirname(root), - eventKey: (path) => (/^sessions\.db(?:-wal)?$/.test(basename(path)) ? 'db' : null), - fileKey: () => 'db', - metadata: true - } -] diff --git a/src/main/ai-vault/scanners/kimi/session-scanner-kimi-index-cache.ts b/src/main/ai-vault/scanners/kimi/session-scanner-kimi-index-cache.ts index 8e90ea863da..7ec492b5395 100644 --- a/src/main/ai-vault/scanners/kimi/session-scanner-kimi-index-cache.ts +++ b/src/main/ai-vault/scanners/kimi/session-scanner-kimi-index-cache.ts @@ -1,23 +1,25 @@ export type KimiSessionIndexIdentity = { + dev?: number + ino?: number changeTimeMs: number mtimeMs: number sizeBytes: number } -type KimiSessionIndexCacheEntry = { +type KimiSessionIndexCacheEntry = { expiresAt: number generation: number identity: KimiSessionIndexIdentity timer: NodeJS.Timeout | null - value: Promise> + value: Promise } export const KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS = 64 // Active scans refresh expiry; idle surfaces release parsed index maps. export const KIMI_WORK_DIR_CACHE_TTL_MS = 5 * 60_000 -export class KimiSessionIndexCache { - private readonly entries = new Map() +export class KimiSessionIndexCache> { + private readonly entries = new Map>() private minimumCacheGeneration = 0 private nextGeneration = 0 @@ -48,10 +50,10 @@ export class KimiSessionIndexCache { indexPath: string, identity: KimiSessionIndexIdentity, generation: number, - load: () => Promise> - ): Promise> { + load: (previous: Promise | undefined) => Promise + ): Promise { if (generation < this.minimumCacheGeneration) { - return load() + return load(undefined) } const cached = this.entries.get(indexPath) const now = Date.now() @@ -61,15 +63,15 @@ export class KimiSessionIndexCache { } if (cached && cached.generation > generation) { // An older stat must not overwrite a generation cached by a newer concurrent read. - return load() + return load(undefined) } - const entry: KimiSessionIndexCacheEntry = { + const entry: KimiSessionIndexCacheEntry = { expiresAt: now + KIMI_WORK_DIR_CACHE_TTL_MS, generation, identity, timer: null, - value: load() + value: load(cached?.value) } this.remember(indexPath, entry, now) return entry.value @@ -83,7 +85,7 @@ export class KimiSessionIndexCache { return this.entries.size } - private forget(indexPath: string, entry: KimiSessionIndexCacheEntry): void { + private forget(indexPath: string, entry: KimiSessionIndexCacheEntry): void { if (this.entries.get(indexPath) !== entry) { return } @@ -93,7 +95,7 @@ export class KimiSessionIndexCache { this.entries.delete(indexPath) } - private remember(indexPath: string, entry: KimiSessionIndexCacheEntry, now: number): void { + private remember(indexPath: string, entry: KimiSessionIndexCacheEntry, now: number): void { const replaced = this.entries.get(indexPath) if (replaced?.timer && replaced !== entry) { clearTimeout(replaced.timer) @@ -119,6 +121,8 @@ export class KimiSessionIndexCache { function identitiesMatch(left: KimiSessionIndexIdentity, right: KimiSessionIndexIdentity): boolean { return ( + left.dev === right.dev && + left.ino === right.ino && left.changeTimeMs === right.changeTimeMs && left.mtimeMs === right.mtimeMs && left.sizeBytes === right.sizeBytes diff --git a/src/main/ai-vault/scanners/kimi/session-scanner-kimi-paths.ts b/src/main/ai-vault/scanners/kimi/session-scanner-kimi-paths.ts index 7849ba01558..292268f0edb 100644 --- a/src/main/ai-vault/scanners/kimi/session-scanner-kimi-paths.ts +++ b/src/main/ai-vault/scanners/kimi/session-scanner-kimi-paths.ts @@ -1,14 +1,11 @@ +import { createSessionJsonlDependency } from '../../session-jsonl-dependency' import type { SessionDiscoveryDependency } from '../../session-discovery-context' import { throwIfAiVaultScanCancelled, waitForAiVaultScan } from '../../ai-vault-scan-cancellation' import { homedir } from 'node:os' import { basename, dirname, join } from 'node:path' -import { createInterface } from 'node:readline' import { resolveAbsoluteDirOverride } from '../../../../shared/absolute-dir-override' -import { - openTranscriptReadStream, - wslGatedStat -} from '../../../native-chat/wsl-transcript-fs-access' -import { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error' +import { wslGatedStat } from '../../../native-chat/wsl-transcript-fs-access' +import { readSessionIndexJsonl, type SessionIndexJsonlRead } from '../../session-index-jsonl-read' import { asRecord } from '../../session-scanner-record-value' import { extractString } from '../../session-scanner-values' import { KimiSessionIndexCache } from './session-scanner-kimi-index-cache' @@ -55,7 +52,7 @@ export function kimiPrimaryAgentWirePath( } // Share parsed indexes by path and file identity; one home can hold many sessions. -const workDirCacheByIndexPath = new KimiSessionIndexCache() +const workDirCacheByIndexPath = new KimiSessionIndexCache() export function clearKimiSessionIndexCache(): void { workDirCacheByIndexPath.clear() @@ -84,55 +81,42 @@ export async function readKimiWorkDirBySessionId( workDirCacheByIndexPath.get( indexPath, { + dev: identity.dev, + ino: identity.ino, changeTimeMs: identity.ctimeMs, mtimeMs: identity.mtimeMs, sizeBytes: identity.size }, generation, - () => - parseKimiSessionIndex(indexPath).then(({ map, refused }) => { - // Evict inside the shared loader so every concurrent caller retries refused reads without a signature change. - if (refused) { - workDirCacheByIndexPath.delete(indexPath, generation) - } - return map - }) + async (previous) => { + try { + return await readSessionIndexJsonl( + indexPath, + identity, + await previous, + parseKimiWorkDirRecord + ) + } catch { + // Refusal never becomes an accepted signature; the next caller must retry. + workDirCacheByIndexPath.delete(indexPath, generation) + return undefined + } + } ), signal - ) + ).then((read) => read?.values ?? new Map()) } -async function parseKimiSessionIndex( - indexPath: string -): Promise<{ map: Map; refused: boolean }> { - const map = new Map() - // Share partial results after read failure so one broken index does not hide every session under its home. +function parseKimiWorkDirRecord(line: string): readonly [string, string] | null { + let record: Record | null try { - const lines = createInterface({ - input: openTranscriptReadStream(indexPath, { encoding: 'utf-8' }, 'scan'), - crlfDelay: Infinity - }) - for await (const line of lines) { - if (!line.trim()) { - continue - } - let record: Record | null - try { - record = asRecord(JSON.parse(line) as unknown) - } catch { - continue - } - const sessionId = extractString(record?.sessionId) - const workDir = extractString(record?.workDir) - if (sessionId && workDir) { - // Later lines win so a resumed session reflects its most recent workDir. - map.set(sessionId, workDir) - } - } - } catch (error) { - return { map, refused: error instanceof WslTranscriptFsError } + record = asRecord(JSON.parse(line) as unknown) + } catch { + return null } - return { map, refused: false } + const id = extractString(record?.sessionId), + cwd = extractString(record?.workDir) + return id && cwd ? [id, cwd] : null } export const kimiSearchDependencies: (root: string) => readonly SessionDiscoveryDependency[] = ( @@ -142,11 +126,19 @@ export const kimiSearchDependencies: (root: string) => readonly SessionDiscovery path: root, fileKey: (path) => basename(dirname(path)), eventKey: (path) => - path.split(/[\\/]/).find((segment) => segment.startsWith('session_')) ?? null + basename(path) === 'wire.jsonl' + ? (path.split(/[\\/]/).find((segment) => segment.startsWith('session_')) ?? null) + : null }, { path: dirname(root), - fileKey: () => 'index', - eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'index' : null) + mode: 'shallow', + fileKey: (path) => basename(dirname(path)), + eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'index' : null), + observe: () => + createSessionJsonlDependency( + join(dirname(root), 'session_index.jsonl'), + parseKimiWorkDirRecord + ) } ] diff --git a/src/main/ai-vault/scanners/opencode/session-scanner-opencode-sources.ts b/src/main/ai-vault/scanners/opencode/session-scanner-opencode-sources.ts index 124f06ddecf..bfd2fb88611 100644 --- a/src/main/ai-vault/scanners/opencode/session-scanner-opencode-sources.ts +++ b/src/main/ai-vault/scanners/opencode/session-scanner-opencode-sources.ts @@ -1,4 +1,4 @@ -import { discoverSessionContainer } from '../../session-discovery-context' +import { currentSessionDiscovery, discoverSessionContainer } from '../../session-discovery-context' import { basename, dirname, join } from 'node:path' import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types' import type { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error' @@ -12,6 +12,7 @@ import { discoverOpenCodeSessions } from './sqlite/session-scanner-opencode-sqli import { listOpenCode2SqliteSessionsViaWorker } from './reader-transport/session-scanner-opencode-sqlite-worker-spawn' import { isOpenCodeV2DatabaseName } from '../../../../shared/opencode-database-name' import type { AiVaultScanOptions, SessionFileDiscovery } from '../../session-scanner-types' +import { isMissingTranscriptPathError } from '../../session-scanner-discovery' import { scopeWalksAgentFiles } from '../../session-scope-cwd' type OpenCodeSource = { @@ -55,12 +56,17 @@ function opencodeSources( const onRefusal = (path: string, error: WslTranscriptFsError): void => { recordSessionScanIssue(issues, { agent: 'opencode', path, ...scanIssueTextFromError(error) }) } + const onFsError = (path: string, error: unknown): void => { + if (currentSessionDiscovery() && !isMissingTranscriptPathError(error)) { + recordSessionScanIssue(issues, { agent: 'opencode', path, ...scanIssueTextFromError(error) }) + } + } const dataDirectorySource = (dataDirectory: string): OpenCodeSource => ({ storageDir: join(dataDirectory, 'storage'), dbPaths: async () => options.opencodeDbPaths ? [] - : listOpenCodeDatabasesInDirectory(dataDirectory, onRefusal, options.signal) + : listOpenCodeDatabasesInDirectory(dataDirectory, onRefusal, options.signal, onFsError) }) const storageDir = options.opencodeStorageDir ?? resolveOpenCodeStorageDirectory() const primary: OpenCodeSource = { @@ -71,8 +77,13 @@ function opencodeSources( } // Custom storage roots still place SQLite databases in the parent data directory. return options.opencodeStorageDir - ? listOpenCodeDatabasesInDirectory(dirname(storageDir), onRefusal, options.signal) - : listOpenCodeDatabases(onRefusal, undefined, options.signal) + ? listOpenCodeDatabasesInDirectory( + dirname(storageDir), + onRefusal, + options.signal, + onFsError + ) + : listOpenCodeDatabases(onRefusal, onFsError, options.signal) } } // Orca-managed accounts relocate XDG_DATA_HOME, so their stores are invisible to the default lookup. @@ -95,24 +106,30 @@ async function discoverOpenCode2Sessions( issues: AiVaultScanIssue[], options: AiVaultScanOptions ): Promise { - const list = (scopePaths?: readonly string[]) => + const list = (scopePaths?: readonly string[], paths = dbPaths, count = limit) => listOpenCode2SqliteSessionsViaWorker({ - dbPaths, - limit, + dbPaths: paths, + limit: count, issues, scopePaths, signal: options.signal }).then((candidates) => candidates.map((candidate) => candidate.file)) const [discovery, scopeFiles] = await Promise.all([ - discoverSessionContainer(`opencode2:${dbPaths.join('|')}`, dbPaths, async () => { - const before = issues.length - return { - agent: 'opencode2', - rootDir: storageDir, - files: await list(), - complete: !issues.slice(before).some((issue) => issue.kind !== 'notice') - } - }), + discoverSessionContainer( + `opencode2:${storageDir}`, + dbPaths, + async (paths, count) => { + const before = issues.length + return { + agent: 'opencode2', + rootDir: storageDir, + files: await list(undefined, paths, count), + complete: !issues.slice(before).some((issue) => issue.kind !== 'notice') + } + }, + limit, + issues + ), scopeWalksAgentFiles('opencode2', options.scopePaths) ? list(options.scopePaths) : undefined ]) return { diff --git a/src/main/ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-discovery.ts b/src/main/ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-discovery.ts index 1c5a7a0cb26..6be879f3d53 100644 --- a/src/main/ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-discovery.ts +++ b/src/main/ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-discovery.ts @@ -24,15 +24,18 @@ export async function discoverOpenCodeSessions(args: { signal?: AbortSignal }): Promise { const walksScope = scopeWalksAgentFiles('opencode', args.scopePaths) - const listSqlite = (scopePaths?: readonly string[]) => + const listSqlite = ( + scopePaths?: readonly string[], + dbPaths = args.dbPaths, + limit = args.limitPerAgent + ) => listOpenCodeSqliteSessionsViaWorker({ - dbPaths: args.dbPaths, - limit: args.limitPerAgent, + dbPaths, + limit, issues: args.issues, scopePaths, signal: args.signal }) - const containerKey = `opencode:${args.dbPaths.join('|')}` const [fileDiscovery, sqliteDiscovery, sqliteScopeCandidates] = await Promise.all([ discoverFiles({ rootDir: join(args.storageDir, 'session'), @@ -51,22 +54,28 @@ export async function discoverOpenCodeSessions(args: { keepScopeFiles: walksScope, signal: args.signal }), - discoverSessionContainer(containerKey, args.dbPaths, async () => { - const before = args.issues.length - const candidates = await listSqlite() - return { - agent: 'opencode', - rootDir: args.storageDir, - files: candidates.map((candidate) => candidate.file), - complete: !args.issues.slice(before).some((issue) => issue.kind !== 'notice') - } - }), + discoverSessionContainer( + `opencode:${args.storageDir}`, + args.dbPaths, + async (paths, limit) => { + const before = args.issues.length + const candidates = await listSqlite(undefined, paths, limit) + return { + agent: 'opencode', + rootDir: args.storageDir, + files: candidates.map((candidate) => candidate.file), + complete: !args.issues.slice(before).some((issue) => issue.kind !== 'notice') + } + }, + args.limitPerAgent, + args.issues + ), walksScope ? listSqlite(args.scopePaths) : undefined ]) const sqliteFiles = sqliteDiscovery.files const sqliteScopeFiles = sqliteScopeCandidates?.map((c) => c.file) - const sqliteSessionIds = currentSessionDiscovery()?.membership(containerKey) ?? new Set() + const sqliteSessionIds = new Set() for (const file of [...sqliteFiles, ...(sqliteScopeFiles ?? [])]) { const parsed = splitOpenCodeSqliteCandidate(file.path) if (parsed) { diff --git a/src/main/ai-vault/scanners/zcode/session-scanner-zcode-sources.ts b/src/main/ai-vault/scanners/zcode/session-scanner-zcode-sources.ts index ac3bafcc694..fb1c224a703 100644 --- a/src/main/ai-vault/scanners/zcode/session-scanner-zcode-sources.ts +++ b/src/main/ai-vault/scanners/zcode/session-scanner-zcode-sources.ts @@ -21,26 +21,47 @@ export function zcodeDiscoveries( : [homedir(), ...wslHomeDirs].map((home) => join(home, '.zcode', 'cli', 'db', 'db.sqlite')) return paths.map(async (dbPath) => { if (!(await zcodeDatabaseExists(dbPath, issues, options.signal))) { - return { agent: 'zcode' as const, rootDir: dbPath, files: [] } - } - const list = (scopePaths?: readonly string[]) => - listZcodeSqliteSessionsViaWorker({ - dbPaths: [dbPath], + return discoverSessionContainer( + `zcode:${dbPath}`, + [dbPath], + async () => ({ + agent: 'zcode', + rootDir: dbPath, + files: [], + complete: !issues.some((issue) => issue.path === dbPath && issue.kind !== 'notice') + }), limit, + issues + ) + } + const list = ( + scopePaths?: readonly string[], + paths: readonly string[] = [dbPath], + count = limit + ) => + listZcodeSqliteSessionsViaWorker({ + dbPaths: paths, + limit: count, issues, scopePaths, signal: options.signal }).then((candidates) => candidates.map((candidate) => candidate.file)) const [discovery, scopeFiles] = await Promise.all([ - discoverSessionContainer(`zcode:${dbPath}`, [dbPath], async () => { - const before = issues.length - return { - agent: 'zcode', - rootDir: dbPath, - files: await list(), - complete: !issues.slice(before).some((issue) => issue.kind !== 'notice') - } - }), + discoverSessionContainer( + `zcode:${dbPath}`, + [dbPath], + async (paths, count) => { + const before = issues.length + return { + agent: 'zcode', + rootDir: dbPath, + files: await list(undefined, paths, count), + complete: !issues.slice(before).some((issue) => issue.kind !== 'notice') + } + }, + limit, + issues + ), scopeWalksAgentFiles('zcode', options.scopePaths) ? list(options.scopePaths) : undefined ]) return { diff --git a/src/main/ai-vault/session-discovery-context.ts b/src/main/ai-vault/session-discovery-context.ts index 6e48fcc0c67..2767fab478c 100644 --- a/src/main/ai-vault/session-discovery-context.ts +++ b/src/main/ai-vault/session-discovery-context.ts @@ -1,13 +1,15 @@ import { AsyncLocalStorage } from 'node:async_hooks' import type { DiscoverSessionFilesArgs } from './session-scanner-discovery' +import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types' import type { SessionFileDiscovery } from './session-scanner-types' export type SessionDiscoveryDependency = { path: string + mode?: 'shallow' | 'recursive' fileKey?: (path: string) => string | null eventKey?: (path: string) => string | null metadata?: boolean - snapshot?: () => Promise> + observe?: () => (force: boolean) => Promise> } export type SessionDiscoveryContext = { @@ -15,10 +17,11 @@ export type SessionDiscoveryContext = { container( key: string, paths: readonly string[], - read: () => Promise + read: (paths: readonly string[], limit: number) => Promise, + limit: number, + issues: readonly AiVaultScanIssue[] ): Promise artifacts(root: string): Map - membership(key: string): ReadonlySet | undefined } const discoveryContext = new AsyncLocalStorage() @@ -34,7 +37,9 @@ export function withSessionDiscovery(context: SessionDiscoveryContext, read: export function discoverSessionContainer( key: string, paths: readonly string[], - read: () => Promise + read: (paths: readonly string[], limit: number) => Promise, + limit: number, + issues: readonly AiVaultScanIssue[] ): Promise { - return currentSessionDiscovery()?.container(key, paths, read) ?? read() + return currentSessionDiscovery()?.container(key, paths, read, limit, issues) ?? read(paths, limit) } diff --git a/src/main/ai-vault/session-file-discovery.ts b/src/main/ai-vault/session-file-discovery.ts index 4b2679ad902..7f48585ecb1 100644 --- a/src/main/ai-vault/session-file-discovery.ts +++ b/src/main/ai-vault/session-file-discovery.ts @@ -1,4 +1,4 @@ -import { dirname, extname, relative } from 'node:path' +import { dirname, extname, isAbsolute, relative } from 'node:path' import type { AiVaultAgentSource } from './session-scanner-types' export function isDiscoverableSessionFile( @@ -6,7 +6,10 @@ export function isDiscoverableSessionFile( rootDir: string, filePath: string ): boolean { - if (!source.extensions.includes(extname(filePath).toLowerCase())) { + if ( + !source.extensions.includes('*') && + !source.extensions.includes(extname(filePath).toLowerCase()) + ) { return false } if (source.filePredicate && !source.filePredicate(filePath)) { @@ -28,5 +31,5 @@ export function pathSegments(filePath: string): string[] { export function isInsideSessionRoot(root: string, path: string): boolean { const rel = relative(root, path) - return rel !== '..' && !rel.startsWith('../') && !rel.startsWith('..\\') && !rel.startsWith('/') + return !isAbsolute(rel) && rel !== '..' && !rel.startsWith('../') && !rel.startsWith('..\\') } diff --git a/src/main/ai-vault/session-file-inventory-directories.ts b/src/main/ai-vault/session-file-inventory-directories.ts index 259fe987464..946f4f82ab4 100644 --- a/src/main/ai-vault/session-file-inventory-directories.ts +++ b/src/main/ai-vault/session-file-inventory-directories.ts @@ -27,7 +27,11 @@ export class SessionInventoryDirectories extends Map void): void { + removeTree( + path: string, + removeFile: (path: string) => void, + removeDirectory?: (path: string) => void + ): void { const directory = this.get(path) if (!directory) { return @@ -36,8 +40,9 @@ export class SessionInventoryDirectories extends Map() + readonly dependencies = new Map>() + readonly failures = new Map() + readonly changes = new Map() + readonly removed = new Set() + readonly recent = new Map() + private readonly dependencyByFile = new Map() + constructor( + private readonly addFile: (path: string) => void, + private readonly scheduleParent: (path: string) => void, + private readonly forgetDirty: (path: string) => void + ) {} + async observe(path: string, args: DiscoverSessionFilesArgs, force: boolean, limit: number) { + const observation = await observeSessionFile( + path, + !force && this.dependencyByFile.has(path) + ? () => this.dependencyByFile.get(path) + : args.contentDependencyPath, + args.signal + ) + throwIfAiVaultScanCancelled(args.signal) + if ('issue' in observation) { + if (observation.missing && !this.files.has(path)) { + this.failures.delete(path) + return + } + if (!observation.missing) { + recordSessionScanIssue(args.issues, { agent: args.agent, ...observation.issue }) + } + this.failures.set(path, force || (this.failures.get(path) ?? false)) + // A successful parent listing, never a refused stat, can remove membership. + this.scheduleParent(path) + return + } + const metadataOnly = sidecarEnrichesWithoutReparse({ + agent: args.agent, + file: observation.file, + codexHome: null + }) + this.failures.delete(path) + if (!metadataOnly && observation.file.sidecar === 'unknown') { + this.failures.set(path, force) + } + const file = { + ...observation.file, + modifiedAt: new Date(observation.file.mtimeMs).toISOString() + } + const old = this.files.get(path) + const dependency = metadataOnly ? undefined : observation.sidecarPath + const previous = this.dependencyByFile.get(path) + if (previous !== dependency) { + if (previous) { + this.removeDependency(previous, path) + } + if (dependency) { + const owners = this.dependencies.get(dependency) ?? new Set() + owners.add(path) + this.dependencies.set(dependency, owners) + this.dependencyByFile.set(path, dependency) + } else { + this.dependencyByFile.delete(path) + } + } + const dependencyChanged = + !metadataOnly && + !!old && + file.sidecar !== 'unknown' && + !sidecarUnchanged(old.sidecar ?? 'none', file.sidecar ?? 'none') + this.files.set(path, file) + if (!old) { + this.addFile(path) + } + if ( + force || + !old || + dependencyChanged || + old.mtimeMs !== file.mtimeMs || + old.sizeBytes !== file.sizeBytes || + old.dev !== file.dev || + old.ino !== file.ino + ) { + this.changes.set(path, { + file, + force: + dependencyChanged || + (force && + !!old && + old.mtimeMs === file.mtimeMs && + old.sizeBytes === file.sizeBytes && + old.dev === file.dev && + old.ino === file.ino) + }) + } + retainRecentSessionFile(this.recent, file, limit) + } + + forget(path: string): void { + this.files.delete(path) + this.failures.delete(path) + this.forgetDirty(path) + this.recent.delete(path) + const dependency = this.dependencyByFile.get(path) + if (dependency) { + this.removeDependency(dependency, path) + } + this.dependencyByFile.delete(path) + this.changes.delete(path) + this.removed.add(path) + } + + private removeDependency(dependency: string, path: string): void { + const owners = this.dependencies.get(dependency) + owners?.delete(path) + if (!owners?.size) { + this.dependencies.delete(dependency) + } + } +} diff --git a/src/main/ai-vault/session-file-inventory.ts b/src/main/ai-vault/session-file-inventory.ts index dac0ada8cff..e756927385d 100644 --- a/src/main/ai-vault/session-file-inventory.ts +++ b/src/main/ai-vault/session-file-inventory.ts @@ -8,16 +8,13 @@ import { wslGatedReaddir, wslGatedStat } from '../native-chat/wsl-transcript-fs- import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation' import { isMissingTranscriptPathError, - observeSessionFile, type DiscoverSessionFilesArgs } from './session-scanner-discovery' -import type { FileWithMtime } from './session-scanner-types' import { isDiscoverableSessionFile, isInsideSessionRoot } from './session-file-discovery' import { recordSessionScanIssue, scanIssueTextFromError } from './session-scan-issues' -import { retainRecentSessionFile } from './session-file-observation' -import { sidecarUnchanged } from './parse-cache/session-sidecar-stat' +import { SessionInventoryFiles, type SessionFileChange } from './session-file-inventory-files' -export type SessionFileChange = { file: FileWithMtime; force: boolean } +export type { SessionFileChange } from './session-file-inventory-files' type Walk = { path: string depth: number @@ -29,34 +26,56 @@ type Walk = { /** Retains discovery position independently of the read queue and of watcher delivery. */ export class SessionFileInventory { - readonly files = new Map() - readonly dependencies = new Map>() + private readonly observations: SessionInventoryFiles + get files() { + return this.observations.files + } + get dependencies() { + return this.observations.dependencies + } private readonly directories: SessionInventoryDirectories private readonly dirty = new Map() private readonly walks: Walk[] = [] private readonly scheduled = new Set() private readonly trailing = new Set() - private readonly recent = new Map() - private readonly dependencyByFile = new Map() - private readonly changes = new Map() - private readonly removed = new Set() private readonly failures = new Set() private reconcileAgain = false + private reconciling = false + private chooseDirty = true + private readonly removedDirectories = new Set() constructor(readonly root: string) { this.directories = new SessionInventoryDirectories(root) + this.observations = new SessionInventoryFiles( + (path) => this.directories.addFile(path), + (path) => this.schedule(dirname(path)), + (path) => this.dirty.delete(path) + ) this.reconcile() } get pending(): boolean { - return this.walks.length > 0 || this.dirty.size > 0 || this.reconcileAgain + return ( + this.walks.length > 0 || + this.dirty.size > 0 || + this.reconcileAgain || + this.failures.size > 0 || + this.observations.failures.size > 0 + ) + } + + get walking(): boolean { + return this.walks.length > 0 || this.reconcileAgain } reconcile(): void { // A burst during a crawl leaves one trailing obligation, never restarts its prefix. - if (!this.scheduled.has(this.root)) { - this.schedule(this.root, true) + if (this.reconciling) { + this.reconcileAgain = true + return } + this.reconciling = true + this.schedule(this.root, true) } invalidate(path: string, force = true): void { @@ -69,7 +88,7 @@ export class SessionFileInventory { } private schedule(path: string, reconcile = false): void { - if (this.scheduled.has(path)) { + if (!isInsideSessionRoot(this.root, path) || this.scheduled.has(path)) { return } const rel = relative(this.root, path) @@ -79,9 +98,11 @@ export class SessionFileInventory { async pollRecent(args: DiscoverSessionFilesArgs, limit: number): Promise { // This set is bounded on admission; never project/sort the archive to find recent paths. - const paths = [...this.recent.keys()] + const paths = [...this.observations.recent.keys()] for (const path of paths) { - await this.observe(path, args, false, limit) + if (!this.dirty.has(path)) { + await this.observations.observe(path, args, false, limit) + } } } @@ -89,77 +110,52 @@ export class SessionFileInventory { args: DiscoverSessionFilesArgs, overdue: () => boolean, recentLimit: number - ): Promise<{ changes: SessionFileChange[]; removed: string[] }> { + ): Promise<{ changes: SessionFileChange[]; removed: string[]; removedDirectories: string[] }> { let consumed = 0 - // Events arriving during an await stay in dirty; deleting before reading preserves their version. - for (let index = 0; index < 16_384 && this.dirty.size; index++) { - if (consumed > 0 && overdue()) { - break + while ((this.dirty.size || this.walks.length) && (consumed === 0 || !overdue())) { + if (this.dirty.size && (!this.walks.length || this.chooseDirty)) { + this.chooseDirty = false + await this.drainDirty(args, recentLimit) + consumed++ + continue } - const first = this.dirty.entries().next().value - if (!first) { - break - } - const [path, force] = first - this.dirty.delete(path) - const owners = this.dependencies.get(path) - if (owners) { - for (const owner of owners) { - this.invalidate(owner, force) - } - } - if (this.accepts(path, args)) { - await this.observe(path, args, force, recentLimit) - } else if (isInsideSessionRoot(this.root, path)) { - try { - if ((await wslGatedStat(path, 'scan', args.signal)).isDirectory()) { - if (this.scheduled.has(path)) { - this.trailing.add(path) - } else { - this.schedule(path) - } - } else if (this.directories.has(path)) { - this.schedule(dirname(path)) - } - } catch (error) { - throwIfAiVaultScanCancelled(args.signal) - if (isMissingTranscriptPathError(error)) { - this.schedule(dirname(path)) - } else { - this.issue(path, error, args) - } - } - } - consumed++ - } - while (this.walks.length && (consumed === 0 || !overdue())) { + this.chooseDirty = true throwIfAiVaultScanCancelled(args.signal) - const walk = this.walks.at(-1)! + const walk = this.walks[0] if (!walk.entries) { try { walk.entries = await wslGatedReaddir(walk.path, 'scan', args.signal) args.onDirectoryEntries?.(walk.path, walk.entries) - const names = new Set(walk.entries.map((entry) => entry.name)) + const entries = new Map(walk.entries.map((entry) => [entry.name, entry])) const old = this.directories.get(walk.path) walk.previous = old if (old) { for (const path of old.files) { - if (!names.has(relative(walk.path, path))) { - this.forget(path) + if (!entries.get(relative(walk.path, path))?.isFile()) { + this.observations.forget(path) + old.files.delete(path) } } for (const child of old.children) { - if (!names.has(relative(walk.path, child))) { - this.directories.removeTree(child, (path) => this.forget(path)) + if (!entries.get(relative(walk.path, child))?.isDirectory()) { + this.removeDirectory(child) } } } - this.directories.set(walk.path, { files: new Set(), children: new Set() }) + this.directories.set(walk.path, old ?? { files: new Set(), children: new Set() }) this.failures.delete(walk.path) } catch (error) { throwIfAiVaultScanCancelled(args.signal) - this.issue(walk.path, error, args) - this.walks.pop() + if (isMissingTranscriptPathError(error)) { + this.failures.delete(walk.path) + this.removeDirectory(walk.path) + } else { + this.issue(walk.path, error, args) + } + const index = this.walks.indexOf(walk) + if (index !== -1) { + this.walks.splice(index, 1) + } this.scheduled.delete(walk.path) continue } @@ -170,15 +166,17 @@ export class SessionFileInventory { const entry = walk.entries[walk.offset] const path = join(walk.path, entry.name) if (entry.isDirectory() && (args.directoryPredicate?.(entry.name, walk.depth) ?? true)) { + const known = walk.previous?.children.has(path) node.children.add(path) - if (walk.reconcile || !walk.previous?.children.has(path)) { + if (walk.reconcile || !known) { this.schedule(path, walk.reconcile) } } else if (entry.isFile() && this.accepts(path, args)) { + const known = walk.previous?.files.has(path) node.files.add(path) // Do not advance past an aborted observation. - if (walk.reconcile || !walk.previous?.files.has(path)) { - await this.observe(path, args, false, recentLimit) + if (walk.reconcile || !known) { + await this.observations.observe(path, args, false, recentLimit) } } walk.offset++ @@ -193,20 +191,92 @@ export class SessionFileInventory { } } } - if (!this.walks.length && this.reconcileAgain) { - this.reconcileAgain = false - this.schedule(this.root, true) + if (!this.walks.length) { + this.reconciling = false + if (this.reconcileAgain) { + this.reconcileAgain = false + this.reconcile() + } } - const changes = [...this.changes.values()] - const removed = [...this.removed] - this.changes.clear() - this.removed.clear() - return { changes, removed } + const changes = [...this.observations.changes.values()] + const removed = [...this.observations.removed] + this.observations.changes.clear() + this.observations.removed.clear() + const removedDirectories = [...this.removedDirectories] + this.removedDirectories.clear() + return { changes, removed, removedDirectories } + } + + private async drainDirty(args: DiscoverSessionFilesArgs, recentLimit: number): Promise { + const first = this.dirty.entries().next().value + if (!first) { + return + } + const [path, force] = first + this.dirty.delete(path) + const owners = this.dependencies.get(path) + if (owners) { + for (const owner of owners) { + this.invalidate(owner, force) + } + } + if (this.accepts(path, args)) { + await this.observations.observe(path, args, force, recentLimit) + } else if (isInsideSessionRoot(this.root, path)) { + try { + const stat = await wslGatedStat(path, 'scan', args.signal) + this.failures.delete(path) + if (stat.isDirectory()) { + if (!isInsideSessionRoot(this.root, path) || this.scheduled.has(path)) { + this.trailing.add(path) + } else { + this.schedule(path) + } + } else if (this.directories.has(path)) { + this.schedule(dirname(path)) + } + } catch (error) { + throwIfAiVaultScanCancelled(args.signal) + if (isMissingTranscriptPathError(error)) { + this.removeDirectory(path) + this.schedule(dirname(path)) + } else { + this.issue(path, error, args) + } + } + } + } + + private removeDirectory(path: string): void { + for (let index = this.walks.length - 1; index >= 0; index--) { + const walk = this.walks[index] + if (isInsideSessionRoot(path, walk.path)) { + this.walks.splice(index, 1) + this.scheduled.delete(walk.path) + this.trailing.delete(walk.path) + } + } + for (const failed of this.failures) { + if (isInsideSessionRoot(path, failed)) { + this.failures.delete(failed) + } + } + this.directories.get(dirname(path))?.children.delete(path) + this.directories.removeTree( + path, + (file) => this.observations.forget(file), + (directory) => { + this.removedDirectories.add(directory) + } + ) } retryFailures(): void { for (const path of this.failures) { - this.schedule(path) + this.schedule(path, true) + } + for (const [path, force] of this.observations.failures) { + this.invalidate(path, force) } } @@ -214,94 +284,6 @@ export class SessionFileInventory { return isInsideSessionRoot(this.root, path) && isDiscoverableSessionFile(args, this.root, path) } - private async observe( - path: string, - args: DiscoverSessionFilesArgs, - force: boolean, - limit: number - ) { - const observation = await observeSessionFile( - path, - !force && this.dependencyByFile.has(path) - ? () => this.dependencyByFile.get(path) - : args.contentDependencyPath, - args.signal - ) - throwIfAiVaultScanCancelled(args.signal) - if ('issue' in observation) { - if (!observation.missing) { - recordSessionScanIssue(args.issues, { agent: args.agent, ...observation.issue }) - } - // Only a successful parent listing removes membership; an unreadable stat is never deletion proof. - if (this.files.has(path)) { - this.schedule(dirname(path)) - } - return - } - const file = { - ...observation.file, - modifiedAt: new Date(observation.file.mtimeMs).toISOString() - } - const old = this.files.get(path) - const dependency = observation.sidecarPath - const previous = this.dependencyByFile.get(path) - if (previous !== dependency) { - if (previous) { - this.dependencies.get(previous)?.delete(path) - } - if (dependency) { - const owners = this.dependencies.get(dependency) ?? new Set() - owners.add(path) - this.dependencies.set(dependency, owners) - this.dependencyByFile.set(path, dependency) - } else { - this.dependencyByFile.delete(path) - } - } - const dependencyChanged = - !!old && - file.sidecar !== 'unknown' && - !sidecarUnchanged(old.sidecar ?? 'none', file.sidecar ?? 'none') - this.files.set(path, file) - if (!old) { - this.directories.addFile(path) - } - if ( - force || - !old || - dependencyChanged || - old.mtimeMs !== file.mtimeMs || - old.sizeBytes !== file.sizeBytes || - old.dev !== file.dev || - old.ino !== file.ino - ) { - this.changes.set(path, { - file, - force: - dependencyChanged || - (force && - !!old && - old.mtimeMs === file.mtimeMs && - old.sizeBytes === file.sizeBytes && - old.dev === file.dev && - old.ino === file.ino) - }) - } - retainRecentSessionFile(this.recent, file, limit) - } - - private forget(path: string): void { - this.files.delete(path) - this.recent.delete(path) - const dependency = this.dependencyByFile.get(path) - if (dependency) { - this.dependencies.get(dependency)?.delete(path) - } - this.dependencyByFile.delete(path) - this.changes.delete(path) - this.removed.add(path) - } - private issue(path: string, error: unknown, args: DiscoverSessionFilesArgs) { this.failures.add(path) recordSessionScanIssue(args.issues, { diff --git a/src/main/ai-vault/session-index-jsonl-read.ts b/src/main/ai-vault/session-index-jsonl-read.ts new file mode 100644 index 00000000000..ad991be0bd5 --- /dev/null +++ b/src/main/ai-vault/session-index-jsonl-read.ts @@ -0,0 +1,108 @@ +import { createHash } from 'node:crypto' +import type { Stats } from 'node:fs' +import { + openTranscriptReadStream, + readTranscriptSlice +} from '../native-chat/wsl-transcript-fs-access' +import { consumeCompleteJsonlLines } from './session-scanner-jsonl-reader' + +export type SessionIndexJsonlRead = { + values: Map + changedKeys: ReadonlySet + offset: number + readThrough: number + dev: number + ino: number + firstRecord: { length: number; hash: string } | null +} + +async function firstRecordSignature(path: string, end: number) { + const hash = createHash('sha256') + let length = 0 + for await (const chunk of openTranscriptReadStream(path, { end: end - 1 }, 'scan')) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + const newline = bytes.indexOf(0x0a) + const prefix = newline === -1 ? bytes : bytes.subarray(0, newline + 1) + hash.update(prefix) + length += prefix.length + if (newline !== -1) { + return { length, hash: hash.digest('hex') } + } + } + return null +} + +export async function readSessionIndexJsonl( + path: string, + stat: Stats, + previous: SessionIndexJsonlRead | undefined, + parse: (line: string) => readonly [string, string] | null +): Promise { + let append = false + if ( + previous?.firstRecord && + previous.offset === previous.readThrough && + stat.size > previous.readThrough && + stat.dev !== 0 && + stat.ino !== 0 && + previous.dev !== 0 && + previous.ino !== 0 && + stat.dev === previous.dev && + stat.ino === previous.ino + ) { + try { + const prefix = await firstRecordSignature(path, previous.firstRecord.length) + const boundary = await readTranscriptSlice(path, previous.offset - 1, 1, 'scan') + // Growth alone cannot distinguish an append from Codex replacing the index on removal. + append = prefix?.hash === previous.firstRecord.hash && boundary[0] === 0x0a + } catch { + // A failed continuity check requires a complete reread. + } + } + const updates = new Map() + const applyLine = (line: string) => { + const entry = parse(line) + if (entry) { + updates.set(entry[0], entry[1]) + } + } + const start = append ? previous!.offset : 0 + const { consumedThrough, trailingPartialLine, bytesRead } = await consumeCompleteJsonlLines({ + path, + start, + onLine: applyLine + }) + // EOF JSON remains readable; a tail forces the next growth to reread rather than retain it. + if (trailingPartialLine) { + applyLine(trailingPartialLine) + } + const firstRecord = append + ? previous!.firstRecord + : consumedThrough > 0 + ? await firstRecordSignature(path, consumedThrough) + : null + const values = append ? previous!.values : new Map() + const changedKeys = new Set() + for (const [key, value] of updates) { + if (previous?.values.get(key) !== value) { + changedKeys.add(key) + } + values.set(key, value) + } + if (!append) { + for (const key of previous?.values.keys() ?? []) { + if (!values.has(key)) { + changedKeys.add(key) + } + } + } + return { + values, + changedKeys, + offset: consumedThrough, + readThrough: start + bytesRead, + dev: stat.dev, + ino: stat.ino, + firstRecord + } +} diff --git a/src/main/ai-vault/session-jsonl-dependency.ts b/src/main/ai-vault/session-jsonl-dependency.ts new file mode 100644 index 00000000000..f7d88eac3fa --- /dev/null +++ b/src/main/ai-vault/session-jsonl-dependency.ts @@ -0,0 +1,35 @@ +import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access' +import { isMissingTranscriptPathError } from './session-file-observation' +import { readSessionIndexJsonl, type SessionIndexJsonlRead } from './session-index-jsonl-read' + +/** Each owner keeps its own accepted cursor, independent of foreground History cache reads. */ +export function createSessionJsonlDependency( + path: string, + parse: (line: string) => readonly [string, string] | null +): (force: boolean) => Promise> { + let accepted: SessionIndexJsonlRead | undefined + let signature: string | undefined + return async (force) => { + let stat + try { + stat = await wslGatedStat(path, 'scan') + } catch (error) { + if (!isMissingTranscriptPathError(error)) { + throw error + } + const removed = new Set(accepted?.values.keys()) + accepted = undefined + signature = undefined + return removed + } + const nextSignature = `${stat.dev}:${stat.ino}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}` + if (!force && signature === nextSignature) { + return new Set() + } + // Commit the cursor only after a complete successful read; refusal leaves the input owed. + const next = await readSessionIndexJsonl(path, stat, accepted, parse) + accepted = next + signature = nextSignature + return next.changedKeys + } +} diff --git a/src/main/ai-vault/session-scanner-agent-sources.ts b/src/main/ai-vault/session-scanner-agent-sources.ts index c787b6d4253..bea3398038b 100644 --- a/src/main/ai-vault/session-scanner-agent-sources.ts +++ b/src/main/ai-vault/session-scanner-agent-sources.ts @@ -2,10 +2,7 @@ import { kimiSearchDependencies, resolveKimiSessionsDir } from './scanners/kimi/session-scanner-kimi-paths' -import { - devinSearchDependencies, - devinSessionsDbDependencyPath -} from './scanners/devin/session-scanner-devin-db' +import * as devin from './scanners/devin/session-scanner-devin-db' import { cursorSearchDependencies, cursorChatMetaPath @@ -78,7 +75,6 @@ const DROID_PROJECTS_DIR = join(homedir(), '.factory', 'projects') const CLINE_SESSIONS_DIR = process.env.CLINE_SESSION_DATA_DIR?.trim() || join(homedir(), '.cline', 'data', 'sessions') -// OpenCode and Antigravity use dedicated discovery for their storage layouts. export const AI_VAULT_AGENT_SOURCES: Record & Partial> = { claude: { @@ -175,8 +171,7 @@ export const AI_VAULT_AGENT_SOURCES: Record @@ -316,3 +311,5 @@ export function sessionRowRootDirs( } export { isDiscoverableSessionFile } from './session-file-discovery' + +export type { AiVaultAgentSource } from './session-scanner-types'