diff --git a/config/scripts/session-search-write-benchmark.ts b/config/scripts/session-search-write-benchmark.ts index 26c4832f079..db09275cd98 100644 --- a/config/scripts/session-search-write-benchmark.ts +++ b/config/scripts/session-search-write-benchmark.ts @@ -15,9 +15,7 @@ import { writeSyntheticTranscriptCorpus } from '../../src/main/ai-vault-search/s import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures' import SyncDatabase from '../../src/main/sqlite/sync-database' -// The cost model owed to the two-FTS-table decision. Everything runs through the -// real transcript reader and SessionSearchStore over a synthetic corpus, so the -// numbers include tokenization, the identifier shadow column and both FTS tables. +// Measures the real transcript reader and search store over a synthetic corpus. // Never point this at a real transcript tree. /** @@ -73,6 +71,21 @@ function tableBytes(db: SyncDatabase): Record { } } +function assertIndexedMessages(db: SyncDatabase, expected: number): number { + const { n } = db + .prepare('SELECT count(*) AS n FROM messages m JOIN sessions s ON s.id = m.session_row_id') + .get() as { n: number } + assert.equal(n, expected, 'indexed message count') + return n +} + +async function checkpointedFileBytes(db: SyncDatabase, path: string): Promise { + // Flush committed WAL pages before reporting the final database footprint. + const [checkpoint] = db.pragma('wal_checkpoint(TRUNCATE)') as { busy: number }[] + assert.equal(checkpoint?.busy, 0, 'storage measurement requires a completed checkpoint') + return (await stat(path)).size +} + // The default corpus puts tool output at about half the message text; set this // far higher to price the tool-row cap against the real 80-97 % band. const toolResultWords = Number(process.env.ORCA_SEARCH_BENCH_TOOL_WORDS ?? 200) @@ -105,11 +118,7 @@ try { const reader = new SyncDatabase(indexPath, { readonly: true }) try { - const rows = ( - reader.prepare('SELECT count(*) AS n FROM messages').get() as { - n: number - } - ).n + const rows = assertIndexedMessages(reader, corpus.messageCount) const sessions = ( reader.prepare('SELECT count(*) AS n FROM sessions').get() as { n: number @@ -119,7 +128,7 @@ try { const bytes = tableBytes(reader) const perMb = (value: number): number => Math.round((value / (corpus.transcriptBytes / (1024 * 1024))) * 10) / 10 - const fileBytes = (await stat(indexPath)).size + const fileBytes = await checkpointedFileBytes(store.connection, indexPath) stalls.sort((a, b) => a - b) transactions.sort((a, b) => a - b) console.log( @@ -195,6 +204,7 @@ try { const indexMs = performance.now() - started restoreExec() assert.deepEqual(errors, []) + assertIndexedMessages(store.connection, large.messageCount) transactions.sort((a, b) => a - b) // The same file again, over a generation the index already holds. That is @@ -219,6 +229,7 @@ try { const reclaimMs = performance.now() - reclaimStarted restoreReplaceExec() assert.deepEqual(errors, []) + assertIndexedMessages(store.connection, large.messageCount) replaceTransactions.sort((a, b) => a - b) console.log( @@ -233,7 +244,11 @@ try { replaceTransactions: replaceTransactions.length, maxReplaceTransactionMs: Math.round(replaceTransactions.at(-1) ?? 0), reclaimMs: Math.round(reclaimMs), - indexMb: Math.round(((await stat(largeIndexPath)).size / (1024 * 1024)) * 100) / 100 + indexMb: + Math.round( + ((await checkpointedFileBytes(store.connection, largeIndexPath)) / (1024 * 1024)) * + 100 + ) / 100 }, null, 2 diff --git a/src/main/ai-vault-search/session-search-file-records.ts b/src/main/ai-vault-search/session-search-file-records.ts index d1ee6df122d..2ee79cbdc13 100644 --- a/src/main/ai-vault-search/session-search-file-records.ts +++ b/src/main/ai-vault-search/session-search-file-records.ts @@ -1,3 +1,4 @@ +import { fileIdentity } from './session-search-file-cursor' import type { AiVaultSession } from '../../shared/ai-vault-types' import type { TranscriptSessionIdentity } from '../ai-vault/session-transcript-consumers' import type { SessionFileCandidate } from '../ai-vault/session-scanner-types' @@ -106,22 +107,24 @@ export class SessionSearchFileRecords { sessionRowId: number | null ): void { const { file } = candidate + const identity = fileIdentity(file) this.db .prepare( `INSERT INTO files(path, dev, ino, byte_offset, mtime_ms, size_bytes, session_row_id) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET - -- Why COALESCE: a host that cannot stat dev/ino (SSH, WSL, a degraded - -- Windows stat) must not erase an identity an earlier scan proved, or - -- the next rename-replace of this path becomes undetectable. - dev = COALESCE(excluded.dev, files.dev), ino = COALESCE(excluded.ino, files.ino), + -- Partial observations must never create a pair that no stat proved. + dev = CASE WHEN excluded.dev IS NOT NULL AND excluded.ino IS NOT NULL + THEN excluded.dev ELSE files.dev END, + ino = CASE WHEN excluded.dev IS NOT NULL AND excluded.ino IS NOT NULL + THEN excluded.ino ELSE files.ino END, byte_offset = excluded.byte_offset, mtime_ms = excluded.mtime_ms, size_bytes = excluded.size_bytes, session_row_id = excluded.session_row_id` ) .run( file.path, - file.dev ?? null, - file.ino ?? null, + identity?.dev ?? null, + identity?.ino ?? null, byteOffset, file.mtimeMs, file.sizeBytes ?? null, diff --git a/src/main/ai-vault-search/session-search-index-writer.test.ts b/src/main/ai-vault-search/session-search-index-writer.test.ts index ad6cadb1ff4..46d147dcce0 100644 --- a/src/main/ai-vault-search/session-search-index-writer.test.ts +++ b/src/main/ai-vault-search/session-search-index-writer.test.ts @@ -172,8 +172,7 @@ it("hands the read's identity accessor to the store", () => { }) it('treats half a recorded identity as no identity at all', () => { - // A host that could stat dev but not ino: `remote-session-file-stat` spreads - // the two independently, and `upsertFile` preserves the half it was given. + // New partial observations are not stored as identities. const partial = { ...syntheticCandidate({ dev: 7 }), agent: 'claude' as const @@ -188,9 +187,11 @@ it('treats half a recorded identity as no identity at all', () => { incomplete: false }) expect(index.db.prepare('SELECT dev, ino FROM files').get()).toEqual({ - dev: 7, + dev: null, ino: null }) + // Older indexes may still carry a half-pair. + index.db.exec('UPDATE files SET dev = 7') // One matching number is not proof of sameness, and one mismatching number is // not proof of replacement. Neither compares, so neither declines. @@ -198,3 +199,30 @@ it('treats half a recorded identity as no identity at all', () => { expect(store.indexedFile(SYNTHETIC_TRANSCRIPT, { dev: 8, ino: 99 })?.byteOffset).toBe(100) expect(store.beginWrite(syntheticCandidate({ dev: 8, ino: 99 }), 'append', 100)).not.toBeNull() }) + +it.each([ + [null, { dev: null, ino: null }], + [ + { dev: 7, ino: 11 }, + { dev: 7, ino: 11 } + ] +])('never combines partial stats with the previous identity %j', (initial, expected) => { + const observations = [initial ?? {}, { dev: 9 }, { ino: 13 }, { dev: 17, ino: 19 }] + for (const [position, identity] of observations.entries()) { + const write = store.beginWrite( + syntheticCandidate(identity), + position ? 'append' : 'replace', + position * 100 + )! + expect( + write.commit({ + session: syntheticSession(), + byteOffset: (position + 1) * 100, + incomplete: false + }) + ).toBe(true) + expect(index.db.prepare('SELECT dev, ino FROM files').get()).toEqual( + position === 3 ? { dev: 17, ino: 19 } : expected + ) + } +}) diff --git a/src/main/ai-vault-search/session-search-index-writer.ts b/src/main/ai-vault-search/session-search-index-writer.ts index 6451262a372..a5c981b5bb2 100644 --- a/src/main/ai-vault-search/session-search-index-writer.ts +++ b/src/main/ai-vault-search/session-search-index-writer.ts @@ -117,15 +117,7 @@ export class SessionSearchIndexWriter { if (!row) { return null } - // A recorded identity that no longer matches is a different file at the same - // path; the rows describe the old one. - // - // Half an identity is no identity: `remote-session-file-stat` spreads dev and - // ino independently, so a host can record one without the other, and the - // COALESCE in `upsertFile` now preserves that half across a later read. One - // number cannot tell a rename-replace from a same-file re-read, so comparing - // it would decline healthy resumes on the strength of a coincidence, and - // `fileIdentity` refuses to build a half identity for the same reason. + // Older indexes can carry half-pairs; only a complete identity can prove replacement. if (identity && row.dev !== null && row.ino !== null) { if (row.dev !== identity.dev || row.ino !== identity.ino) { return null diff --git a/src/main/ai-vault-search/session-search-synthetic-corpus.test.ts b/src/main/ai-vault-search/session-search-synthetic-corpus.test.ts new file mode 100644 index 00000000000..9b6a48beb4e --- /dev/null +++ b/src/main/ai-vault-search/session-search-synthetic-corpus.test.ts @@ -0,0 +1,44 @@ +import { rm } from 'node:fs/promises' +import { join } from 'node:path' +import { expect, it } from 'vitest' +import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache' +import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers' +import { registerSessionSearchIndexConsumer } from './session-search-index-consumer' +import { SessionSearchStore } from './session-search-store' +import { writeSyntheticTranscriptCorpus } from './session-search-synthetic-corpus' +import { parseTranscript } from './session-search-transcript-fixtures' + +it.each([Infinity, -Infinity, Number.NaN, -1, 1.5])( + 'rejects invalid corpus loop bounds: %s', + async (value) => { + for (const field of ['sessions', 'turnsPerSession', 'toolResultWords']) { + await expect(writeSyntheticTranscriptCorpus({ [field]: value })).rejects.toThrow(RangeError) + } + } +) + +it.each([0, 200, 2000])( + 'counts the indexed messages with %s tool words', + async (toolResultWords) => { + const corpus = await writeSyntheticTranscriptCorpus({ + sessions: 1, + turnsPerSession: 1, + toolResultWords + }) + const store = new SessionSearchStore(join(corpus.root, 'index.sqlite')) + const unregister = registerSessionSearchIndexConsumer(store) + try { + await parseTranscript(corpus.files[0]!) + expect(corpus.messageCount).toBe(toolResultWords === 0 ? 3 : 4) + expect(store.connection.prepare('SELECT count(*) AS n FROM messages').get()).toEqual({ + n: corpus.messageCount + }) + } finally { + unregister() + resetTranscriptConsumersForTests() + resetSessionParseCacheForTests() + store.close() + await rm(corpus.root, { recursive: true, force: true }) + } + } +) diff --git a/src/main/ai-vault-search/session-search-synthetic-corpus.ts b/src/main/ai-vault-search/session-search-synthetic-corpus.ts index c22061a796b..14e8a27fe5f 100644 --- a/src/main/ai-vault-search/session-search-synthetic-corpus.ts +++ b/src/main/ai-vault-search/session-search-synthetic-corpus.ts @@ -74,6 +74,15 @@ export async function writeSyntheticTranscriptCorpus( const sessions = options.sessions ?? 40 const turns = options.turnsPerSession ?? 60 const toolWords = options.toolResultWords ?? 200 + for (const [name, value] of Object.entries({ + sessions, + turnsPerSession: turns, + toolResultWords: toolWords + })) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a finite non-negative safe integer`) + } + } const random = mulberry32(options.seed ?? 1) const root = await mkdtemp(join(tmpdir(), 'orca-search-corpus-')) const files: string[] = [] @@ -131,8 +140,8 @@ export async function writeSyntheticTranscriptCorpus( } }) ) - // One user turn, one assistant turn, one tool call, one tool result. - messageCount += 4 + // Empty tool results emit no searchable message. + messageCount += toolWords === 0 ? 3 : 4 } const path = join(root, `${sessionId}.jsonl`) const body = `${lines.join('\n')}\n`