fix(ai-vault-search): preserve stat pairs and validate benchmark results

config/scripts/session-search-write-benchmark.ts

@@ -15,9 +15,7 @@ import { writeSyntheticTranscriptCorpus } from '../../src/main/ai-vault-search/s
 import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
 import SyncDatabase from '../../src/main/sqlite/sync-database'
 
-// The cost model owed to the two-FTS-table decision. Everything runs through the
-// real transcript reader and SessionSearchStore over a synthetic corpus, so the
-// numbers include tokenization, the identifier shadow column and both FTS tables.
+// Measures the real transcript reader and search store over a synthetic corpus.
 // Never point this at a real transcript tree.
 
 /**
@@ -73,6 +71,21 @@ function tableBytes(db: SyncDatabase): Record<string, number> {
   }
 }
 
+function assertIndexedMessages(db: SyncDatabase, expected: number): number {
+  const { n } = db
+    .prepare('SELECT count(*) AS n FROM messages m JOIN sessions s ON s.id = m.session_row_id')
+    .get() as { n: number }
+  assert.equal(n, expected, 'indexed message count')
+  return n
+}
+
+async function checkpointedFileBytes(db: SyncDatabase, path: string): Promise<number> {
+  // Flush committed WAL pages before reporting the final database footprint.
+  const [checkpoint] = db.pragma('wal_checkpoint(TRUNCATE)') as { busy: number }[]
+  assert.equal(checkpoint?.busy, 0, 'storage measurement requires a completed checkpoint')
+  return (await stat(path)).size
+}
+
 // The default corpus puts tool output at about half the message text; set this
 // far higher to price the tool-row cap against the real 80-97 % band.
 const toolResultWords = Number(process.env.ORCA_SEARCH_BENCH_TOOL_WORDS ?? 200)
@@ -105,11 +118,7 @@ try {
 
     const reader = new SyncDatabase(indexPath, { readonly: true })
     try {
-      const rows = (
-        reader.prepare('SELECT count(*) AS n FROM messages').get() as {
-          n: number
-        }
-      ).n
+      const rows = assertIndexedMessages(reader, corpus.messageCount)
       const sessions = (
         reader.prepare('SELECT count(*) AS n FROM sessions').get() as {
           n: number
@@ -119,7 +128,7 @@ try {
       const bytes = tableBytes(reader)
       const perMb = (value: number): number =>
         Math.round((value / (corpus.transcriptBytes / (1024 * 1024))) * 10) / 10
-      const fileBytes = (await stat(indexPath)).size
+      const fileBytes = await checkpointedFileBytes(store.connection, indexPath)
       stalls.sort((a, b) => a - b)
       transactions.sort((a, b) => a - b)
       console.log(
@@ -195,6 +204,7 @@ try {
     const indexMs = performance.now() - started
     restoreExec()
     assert.deepEqual(errors, [])
+    assertIndexedMessages(store.connection, large.messageCount)
     transactions.sort((a, b) => a - b)
 
     // The same file again, over a generation the index already holds. That is
@@ -219,6 +229,7 @@ try {
     const reclaimMs = performance.now() - reclaimStarted
     restoreReplaceExec()
     assert.deepEqual(errors, [])
+    assertIndexedMessages(store.connection, large.messageCount)
     replaceTransactions.sort((a, b) => a - b)
 
     console.log(
@@ -233,7 +244,11 @@ try {
           replaceTransactions: replaceTransactions.length,
           maxReplaceTransactionMs: Math.round(replaceTransactions.at(-1) ?? 0),
           reclaimMs: Math.round(reclaimMs),
-          indexMb: Math.round(((await stat(largeIndexPath)).size / (1024 * 1024)) * 100) / 100
+          indexMb:
+            Math.round(
+              ((await checkpointedFileBytes(store.connection, largeIndexPath)) / (1024 * 1024)) *
+                100
+            ) / 100
         },
         null,
         2

src/main/ai-vault-search/session-search-file-records.ts

@@ -1,3 +1,4 @@
+import { fileIdentity } from './session-search-file-cursor'
 import type { AiVaultSession } from '../../shared/ai-vault-types'
 import type { TranscriptSessionIdentity } from '../ai-vault/session-transcript-consumers'
 import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
@@ -106,22 +107,24 @@ export class SessionSearchFileRecords {
     sessionRowId: number | null
   ): void {
     const { file } = candidate
+    const identity = fileIdentity(file)
     this.db
       .prepare(
         `INSERT INTO files(path, dev, ino, byte_offset, mtime_ms, size_bytes, session_row_id)
          VALUES (?, ?, ?, ?, ?, ?, ?)
          ON CONFLICT(path) DO UPDATE SET
-           -- Why COALESCE: a host that cannot stat dev/ino (SSH, WSL, a degraded
-           -- Windows stat) must not erase an identity an earlier scan proved, or
-           -- the next rename-replace of this path becomes undetectable.
-           dev = COALESCE(excluded.dev, files.dev), ino = COALESCE(excluded.ino, files.ino),
+           -- Partial observations must never create a pair that no stat proved.
+           dev = CASE WHEN excluded.dev IS NOT NULL AND excluded.ino IS NOT NULL
+             THEN excluded.dev ELSE files.dev END,
+           ino = CASE WHEN excluded.dev IS NOT NULL AND excluded.ino IS NOT NULL
+             THEN excluded.ino ELSE files.ino END,
            byte_offset = excluded.byte_offset, mtime_ms = excluded.mtime_ms,
            size_bytes = excluded.size_bytes, session_row_id = excluded.session_row_id`
       )
       .run(
         file.path,
-        file.dev ?? null,
-        file.ino ?? null,
+        identity?.dev ?? null,
+        identity?.ino ?? null,
         byteOffset,
         file.mtimeMs,
         file.sizeBytes ?? null,

src/main/ai-vault-search/session-search-index-writer.test.ts

@@ -172,8 +172,7 @@ it("hands the read's identity accessor to the store", () => {
 })
 
 it('treats half a recorded identity as no identity at all', () => {
-  // A host that could stat dev but not ino: `remote-session-file-stat` spreads
-  // the two independently, and `upsertFile` preserves the half it was given.
+  // New partial observations are not stored as identities.
   const partial = {
     ...syntheticCandidate({ dev: 7 }),
     agent: 'claude' as const
@@ -188,9 +187,11 @@ it('treats half a recorded identity as no identity at all', () => {
     incomplete: false
   })
   expect(index.db.prepare('SELECT dev, ino FROM files').get()).toEqual({
-    dev: 7,
+    dev: null,
     ino: null
   })
+  // Older indexes may still carry a half-pair.
+  index.db.exec('UPDATE files SET dev = 7')
 
   // One matching number is not proof of sameness, and one mismatching number is
   // not proof of replacement. Neither compares, so neither declines.
@@ -198,3 +199,30 @@ it('treats half a recorded identity as no identity at all', () => {
   expect(store.indexedFile(SYNTHETIC_TRANSCRIPT, { dev: 8, ino: 99 })?.byteOffset).toBe(100)
   expect(store.beginWrite(syntheticCandidate({ dev: 8, ino: 99 }), 'append', 100)).not.toBeNull()
 })
+
+it.each([
+  [null, { dev: null, ino: null }],
+  [
+    { dev: 7, ino: 11 },
+    { dev: 7, ino: 11 }
+  ]
+])('never combines partial stats with the previous identity %j', (initial, expected) => {
+  const observations = [initial ?? {}, { dev: 9 }, { ino: 13 }, { dev: 17, ino: 19 }]
+  for (const [position, identity] of observations.entries()) {
+    const write = store.beginWrite(
+      syntheticCandidate(identity),
+      position ? 'append' : 'replace',
+      position * 100
+    )!
+    expect(
+      write.commit({
+        session: syntheticSession(),
+        byteOffset: (position + 1) * 100,
+        incomplete: false
+      })
+    ).toBe(true)
+    expect(index.db.prepare('SELECT dev, ino FROM files').get()).toEqual(
+      position === 3 ? { dev: 17, ino: 19 } : expected
+    )
+  }
+})

src/main/ai-vault-search/session-search-index-writer.ts

@@ -117,15 +117,7 @@ export class SessionSearchIndexWriter {
     if (!row) {
       return null
     }
-    // A recorded identity that no longer matches is a different file at the same
-    // path; the rows describe the old one.
-    //
-    // Half an identity is no identity: `remote-session-file-stat` spreads dev and
-    // ino independently, so a host can record one without the other, and the
-    // COALESCE in `upsertFile` now preserves that half across a later read. One
-    // number cannot tell a rename-replace from a same-file re-read, so comparing
-    // it would decline healthy resumes on the strength of a coincidence, and
-    // `fileIdentity` refuses to build a half identity for the same reason.
+    // Older indexes can carry half-pairs; only a complete identity can prove replacement.
     if (identity && row.dev !== null && row.ino !== null) {
       if (row.dev !== identity.dev || row.ino !== identity.ino) {
         return null

src/main/ai-vault-search/session-search-synthetic-corpus.test.ts

@@ -0,0 +1,44 @@
+import { rm } from 'node:fs/promises'
+import { join } from 'node:path'
+import { expect, it } from 'vitest'
+import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
+import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
+import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
+import { SessionSearchStore } from './session-search-store'
+import { writeSyntheticTranscriptCorpus } from './session-search-synthetic-corpus'
+import { parseTranscript } from './session-search-transcript-fixtures'
+
+it.each([Infinity, -Infinity, Number.NaN, -1, 1.5])(
+  'rejects invalid corpus loop bounds: %s',
+  async (value) => {
+    for (const field of ['sessions', 'turnsPerSession', 'toolResultWords']) {
+      await expect(writeSyntheticTranscriptCorpus({ [field]: value })).rejects.toThrow(RangeError)
+    }
+  }
+)
+
+it.each([0, 200, 2000])(
+  'counts the indexed messages with %s tool words',
+  async (toolResultWords) => {
+    const corpus = await writeSyntheticTranscriptCorpus({
+      sessions: 1,
+      turnsPerSession: 1,
+      toolResultWords
+    })
+    const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'))
+    const unregister = registerSessionSearchIndexConsumer(store)
+    try {
+      await parseTranscript(corpus.files[0]!)
+      expect(corpus.messageCount).toBe(toolResultWords === 0 ? 3 : 4)
+      expect(store.connection.prepare('SELECT count(*) AS n FROM messages').get()).toEqual({
+        n: corpus.messageCount
+      })
+    } finally {
+      unregister()
+      resetTranscriptConsumersForTests()
+      resetSessionParseCacheForTests()
+      store.close()
+      await rm(corpus.root, { recursive: true, force: true })
+    }
+  }
+)

src/main/ai-vault-search/session-search-synthetic-corpus.ts

@@ -74,6 +74,15 @@ export async function writeSyntheticTranscriptCorpus(
   const sessions = options.sessions ?? 40
   const turns = options.turnsPerSession ?? 60
   const toolWords = options.toolResultWords ?? 200
+  for (const [name, value] of Object.entries({
+    sessions,
+    turnsPerSession: turns,
+    toolResultWords: toolWords
+  })) {
+    if (!Number.isSafeInteger(value) || value < 0) {
+      throw new RangeError(`${name} must be a finite non-negative safe integer`)
+    }
+  }
   const random = mulberry32(options.seed ?? 1)
   const root = await mkdtemp(join(tmpdir(), 'orca-search-corpus-'))
   const files: string[] = []
@@ -131,8 +140,8 @@ export async function writeSyntheticTranscriptCorpus(
           }
         })
       )
-      // One user turn, one assistant turn, one tool call, one tool result.
-      messageCount += 4
+      // Empty tool results emit no searchable message.
+      messageCount += toolWords === 0 ? 3 : 4
     }
     const path = join(root, `${sessionId}.jsonl`)
     const body = `${lines.join('\n')}\n`