fix(ai-vault-search): answer from a version-1 index, and keep a repaired literal whole

Two smaller findings.

An engine can be handed a connection to a version-1 file that another handle is
still answering from, and this PR is what first makes that reachable. It now
probes once for the two tables version 2 added and names what it cannot serve
on every result, instead of throwing at the first query that reaches for a
vocabulary that is not there. The route ladder simply skips its repair rung.
Which process may unlink and rebuild the index is PR 3b's decision and is not
solved here.

Typo repair re-planned the corrected query from scratch, and a corrected
spelling can read as prose even when what was typed was a literal:
`parseJsonn(the, data)` has the punctuation, `parsejson the data` does not, so
the re-plan dropped `the` as a stop word. The repaired query searched for less
than was asked and `repairedTerms` reported a body nobody typed. The re-plan is
now told what the original decided, because a repair changes spellings, not the
query's character.

`repairedTerms` is documented as the whole body the repaired plan ran, with the
index's own case-folded spelling for the terms it corrected.

src/main/ai-vault-search/session-search-engine-types.ts

@@ -1,5 +1,6 @@
 import type { AiVaultAgent } from '../../shared/ai-vault-types'
 import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers'
+import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities'
 
 // ENGINE types, deliberately not in src/shared: nothing here is a wire type.
 // PR 5 owns the public contract and lifts what a caller may actually receive;
@@ -59,7 +60,13 @@ export type SessionSearchRoute = 'phrase' | 'and' | 'or' | 'typo+phrase' | 'typo
  */
 export type SessionSearchPlannerReport = {
   route: SessionSearchRoute
-  /** Query terms after typo repair, when any were changed. */
+  /**
+   * The whole body the repaired plan searched, in query order, when any term
+   * was changed. Not just the corrected terms: a caller rendering "searched
+   * for" needs the query it actually ran, and a repair never drops a term the
+   * original kept. A corrected term carries the index's own spelling, which the
+   * tokenizer has case-folded; untouched terms keep the case they were typed in.
+   */
   repairedTerms?: string[]
   /** The corpus the route ran against; today always the requested scope. */
   tier: SessionSearchScope
@@ -120,6 +127,12 @@ export type SessionSearchTruncation = {
 
 export type SessionSearchResponse = {
   hits: SessionSearchHit[]
+  /**
+   * Engine features the index on disk cannot serve, empty on a current index.
+   * A route ladder missing its repair rung still answers; saying so is what
+   * keeps the answer honest.
+   */
+  unavailable: readonly SessionSearchUnavailableFeature[]
   planner: SessionSearchPlannerReport
   page: SessionSearchPage
   truncated: SessionSearchTruncation

src/main/ai-vault-search/session-search-engine.test.ts

@@ -1,4 +1,5 @@
 import { afterEach, describe, expect, it, vi } from 'vitest'
+import { SessionSearchEngine } from './session-search-engine'
 import { SESSION_SEARCH_QUERY_MAX_LENGTH } from './session-search-engine-types'
 import type { SessionSearchRequest, SessionSearchResponse } from './session-search-engine-types'
 import {
@@ -63,6 +64,20 @@ describe('the route ladder tries phrase, then AND, then repair, then OR', () =>
     expect(ids(result).sort()).toEqual(['1', '2'])
   })
 
+  it('keeps every term a repaired literal was typed with', async () => {
+    const { db, engine } = await open('ss-engine-typo-literal')
+    addSyntheticSession(db, { id: 1, text: 'parseJson the data' })
+    addSyntheticSession(db, { id: 2, text: 'parseJson the data again' })
+    // `parseJsonn(the, data)` is literal because of its punctuation; the
+    // corrected spelling read on its own is prose. Re-planning without carrying
+    // the original decision across would drop `the` and report a body that was
+    // never typed.
+    // A corrected term comes back in the index's own spelling, which unicode61
+    // has folded; the terms the repair left alone keep the case they were typed.
+    const result = engine.search({ query: 'parseJsonn(the, data)' })
+    expect(result.planner.repairedTerms).toEqual(['parsejson', 'the', 'data'])
+  })
+
   it('does not repair a term the index already holds', async () => {
     const { db, engine } = await open('ss-engine-no-typo')
     addSyntheticSession(db, { id: 1, text: 'coalesces' })
@@ -255,6 +270,39 @@ describe('source presence comes from the files table, never a stat', () => {
   })
 })
 
+describe('an index that predates version 2 is answered from, not thrown at', () => {
+  async function version1(): Promise<SessionSearchHarness> {
+    const opened = await open('ss-engine-v1')
+    addSyntheticSession(opened.db, { id: 1, text: 'the coalesces path is slow' })
+    addSyntheticSession(opened.db, { id: 2, text: 'coalesces again here' })
+    // A real v1 file: neither table version 2 added exists in it.
+    opened.db.exec('DROP TABLE messages_vocab; DROP TABLE search_log')
+    return opened
+  }
+
+  it('still searches, and names the feature it cannot serve', async () => {
+    const { store } = await version1()
+    const engine = new SessionSearchEngine(store, { logQueries: true })
+    const result = engine.search({ query: 'coalesces' })
+    expect(ids(result).sort()).toEqual(['1', '2'])
+    expect(result.unavailable).toEqual(['typo-repair', 'query-log'])
+  })
+
+  it('skips the repair rung rather than reaching for a vocabulary that is gone', async () => {
+    const { store } = await version1()
+    const result = new SessionSearchEngine(store).search({ query: 'coalescs' })
+    expect(result.planner.route).toBe('or')
+    expect(result.planner.repairedTerms).toBeUndefined()
+    expect(result.hits).toEqual([])
+  })
+
+  it('claims nothing unavailable on a current index', async () => {
+    const { db, engine } = await open('ss-engine-v2')
+    addSyntheticSession(db, { id: 1 })
+    expect(engine.search({ query: 'needle' }).unavailable).toEqual([])
+  })
+})
+
 describe('the engine is the only thing that warms the index', () => {
   it('warms on the first search and leans on the store to memoize the rest', async () => {
     const { db, store, engine } = await open('ss-engine-warm')

src/main/ai-vault-search/session-search-engine.ts

@@ -12,6 +12,7 @@ import {
   type SessionSearchResponse,
   type SessionSearchSourcePresence
 } from './session-search-engine-types'
+import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities'
 import {
   rankSessionHits,
   type MessageRow,
@@ -32,6 +33,7 @@ import {
   type Retrieved
 } from './session-search-retrieval'
 import { sessionRowFilter } from './session-search-row-filter'
+import { sessionSearchUnavailableFeatures } from './session-search-index-capabilities'
 import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet'
 import { sessionSourcePresence } from './session-search-source-presence'
 import type { SessionSearchStore } from './session-search-store'
@@ -79,13 +81,16 @@ export class SessionSearchEngine {
   private readonly db: SyncDatabase
   private readonly retrieval: SessionSearchRetrieval
   private readonly candidateLimit: number
+  /** Probed once: a version-1 file has neither vocabulary nor query log. */
+  private readonly unavailable: readonly SessionSearchUnavailableFeature[]
 
   constructor(
     private readonly store: SessionSearchStore,
     private readonly options: SessionSearchEngineOptions = {}
   ) {
     this.db = store.connection
-    this.retrieval = new SessionSearchRetrieval(this.db)
+    this.unavailable = sessionSearchUnavailableFeatures(this.db)
+    this.retrieval = new SessionSearchRetrieval(this.db, !this.unavailable.includes('typo-repair'))
     this.candidateLimit = options.sessionCandidateLimit ?? SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT
   }
 
@@ -129,6 +134,7 @@ export class SessionSearchEngine {
     const hasMore = ranked.length > offset + limit
     const response: SessionSearchResponse = {
       hits,
+      unavailable: this.unavailable,
       planner: {
         route: retrieved?.route ?? 'or',
         tier: scope,
@@ -147,7 +153,7 @@ export class SessionSearchEngine {
       generation,
       durationMs: performance.now() - startedAt
     }
-    if (this.options.logQueries) {
+    if (this.options.logQueries && !this.unavailable.includes('query-log')) {
       logSessionSearchQuery(this.db, {
         query: request.query,
         route: response.planner.route,

src/main/ai-vault-search/session-search-index-capabilities.ts

@@ -0,0 +1,38 @@
+import type SyncDatabase from '../sqlite/sync-database'
+
+/** An engine feature the index on disk cannot serve. */
+export type SessionSearchUnavailableFeature = 'typo-repair' | 'query-log'
+
+/**
+ * What this index can answer, probed once when an engine opens over it.
+ *
+ * Why probe at all: schema version 2 added `messages_vocab` and `search_log`,
+ * and a mismatched file is normally dropped and rebuilt when it is opened. But
+ * an engine can be handed a connection to a version-1 file that another handle
+ * is still answering from, and PR 4 is what first makes that reachable. Reading
+ * the tables that do exist and saying plainly which feature is missing is a
+ * better answer than throwing on the first query that reaches for one.
+ *
+ * Which process may open, unlink and rebuild the index is PR 3b's decision, not
+ * this file's; this only keeps a reader useful while that is unsettled.
+ */
+export function sessionSearchUnavailableFeatures(
+  db: SyncDatabase
+): SessionSearchUnavailableFeature[] {
+  const unavailable: SessionSearchUnavailableFeature[] = []
+  if (!hasTable(db, 'messages_vocab')) {
+    unavailable.push('typo-repair')
+  }
+  if (!hasTable(db, 'search_log')) {
+    unavailable.push('query-log')
+  }
+  return unavailable
+}
+
+function hasTable(db: SyncDatabase, name: string): boolean {
+  return (
+    db
+      .prepare("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?")
+      .get(name) !== undefined
+  )
+}

src/main/ai-vault-search/session-search-query-planner.ts

@@ -53,11 +53,21 @@ export function indexTokens(query: string, limit = Number.POSITIVE_INFINITY): st
   return out
 }
 
-export function planSessionSearchQuery(query: string): SessionSearchQueryPlan {
+/**
+ * `literal` overrides the shape test. Typo repair re-plans the query it
+ * corrected, and a corrected spelling can look like ordinary prose even though
+ * what was typed was a literal: `parseJsonn(the, data)` has the punctuation that
+ * makes it literal, `parsejson the data` does not. Without the override the
+ * re-plan would drop `the` as a stop word, so the repaired query would search
+ * for less than the original asked for and `repairedTerms` would report a body
+ * the user never typed.
+ */
+export function planSessionSearchQuery(
+  query: string,
+  literal = isLiteralQuery(query)
+): SessionSearchQueryPlan {
   const raw = indexTokens(query, MAX_BODY_TERMS)
-  let body = isLiteralQuery(query)
-    ? raw
-    : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase()))
+  let body = literal ? raw : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase()))
   if (body.length < 2) {
     body = raw
   }
@@ -71,7 +81,7 @@ export function planSessionSearchQuery(query: string): SessionSearchQueryPlan {
     }
   }
   return {
-    literal: isLiteralQuery(query),
+    literal,
     terms: [...terms, ...extra].slice(0, MAX_TERMS),
     body: body.slice(0, MAX_BODY_TERMS)
   }

src/main/ai-vault-search/session-search-retrieval.ts

@@ -43,10 +43,14 @@ export function ftsTableFor(scope: SessionSearchScope): 'messages_fts' | 'conver
 
 /** The FTS half of a search: the route ladder and the SQL each rung runs. */
 export class SessionSearchRetrieval {
-  private readonly typoRepair: SessionSearchTypoRepair
+  /** Null when this index has no vocabulary to repair against; the rung is skipped. */
+  private readonly typoRepair: SessionSearchTypoRepair | null
 
-  constructor(private readonly db: SyncDatabase) {
-    this.typoRepair = new SessionSearchTypoRepair(db)
+  constructor(
+    private readonly db: SyncDatabase,
+    canRepairTypos = true
+  ) {
+    this.typoRepair = canRepairTypos ? new SessionSearchTypoRepair(db) : null
   }
 
   /**
@@ -100,16 +104,23 @@ export class SessionSearchRetrieval {
   }
 
   private repair(plan: SessionSearchQueryPlan): SessionSearchQueryPlan | null {
+    if (!this.typoRepair) {
+      return null
+    }
+    const typoRepair = this.typoRepair
     let changed = false
     const body = plan.body.map((term) => {
-      const fix = this.typoRepair.correct(term)
+      const fix = typoRepair.correct(term)
       if (fix && fix !== term.toLowerCase()) {
         changed = true
         return fix
       }
       return term
     })
-    return changed ? { ...planSessionSearchQuery(body.join(' ')), literal: plan.literal } : null
+    // The repair changes spellings, not the query's character: the re-plan is
+    // told what the original decided so a corrected literal keeps every term it
+    // was typed with.
+    return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null
   }
 
   /** Phrase, then AND, for literal-looking queries; null when neither matches. */