From 96efb319bc30366fc11a03201e998b95f58c03b0 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 9 Sep 2026 12:35:12 -0400 Subject: [PATCH] fix(ai-vault-search): answer from a version-1 index, and keep a repaired literal whole Two smaller findings. An engine can be handed a connection to a version-1 file that another handle is still answering from, and this PR is what first makes that reachable. It now probes once for the two tables version 2 added and names what it cannot serve on every result, instead of throwing at the first query that reaches for a vocabulary that is not there. The route ladder simply skips its repair rung. Which process may unlink and rebuild the index is PR 3b's decision and is not solved here. Typo repair re-planned the corrected query from scratch, and a corrected spelling can read as prose even when what was typed was a literal: `parseJsonn(the, data)` has the punctuation, `parsejson the data` does not, so the re-plan dropped `the` as a stop word. The repaired query searched for less than was asked and `repairedTerms` reported a body nobody typed. The re-plan is now told what the original decided, because a repair changes spellings, not the query's character. `repairedTerms` is documented as the whole body the repaired plan ran, with the index's own case-folded spelling for the terms it corrected. --- .../session-search-engine-types.ts | 15 +++++- .../session-search-engine.test.ts | 48 +++++++++++++++++++ .../ai-vault-search/session-search-engine.ts | 10 +++- .../session-search-index-capabilities.ts | 38 +++++++++++++++ .../session-search-query-planner.ts | 20 ++++++-- .../session-search-retrieval.ts | 21 ++++++-- 6 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 src/main/ai-vault-search/session-search-index-capabilities.ts diff --git a/src/main/ai-vault-search/session-search-engine-types.ts b/src/main/ai-vault-search/session-search-engine-types.ts index a90fe1be34e..709ed9ccca5 100644 --- a/src/main/ai-vault-search/session-search-engine-types.ts +++ b/src/main/ai-vault-search/session-search-engine-types.ts @@ -1,5 +1,6 @@ import type { AiVaultAgent } from '../../shared/ai-vault-types' import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers' +import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities' // ENGINE types, deliberately not in src/shared: nothing here is a wire type. // PR 5 owns the public contract and lifts what a caller may actually receive; @@ -59,7 +60,13 @@ export type SessionSearchRoute = 'phrase' | 'and' | 'or' | 'typo+phrase' | 'typo */ export type SessionSearchPlannerReport = { route: SessionSearchRoute - /** Query terms after typo repair, when any were changed. */ + /** + * The whole body the repaired plan searched, in query order, when any term + * was changed. Not just the corrected terms: a caller rendering "searched + * for" needs the query it actually ran, and a repair never drops a term the + * original kept. A corrected term carries the index's own spelling, which the + * tokenizer has case-folded; untouched terms keep the case they were typed in. + */ repairedTerms?: string[] /** The corpus the route ran against; today always the requested scope. */ tier: SessionSearchScope @@ -120,6 +127,12 @@ export type SessionSearchTruncation = { export type SessionSearchResponse = { hits: SessionSearchHit[] + /** + * Engine features the index on disk cannot serve, empty on a current index. + * A route ladder missing its repair rung still answers; saying so is what + * keeps the answer honest. + */ + unavailable: readonly SessionSearchUnavailableFeature[] planner: SessionSearchPlannerReport page: SessionSearchPage truncated: SessionSearchTruncation diff --git a/src/main/ai-vault-search/session-search-engine.test.ts b/src/main/ai-vault-search/session-search-engine.test.ts index 1b2a6d246cd..7fbcd5e113a 100644 --- a/src/main/ai-vault-search/session-search-engine.test.ts +++ b/src/main/ai-vault-search/session-search-engine.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' +import { SessionSearchEngine } from './session-search-engine' import { SESSION_SEARCH_QUERY_MAX_LENGTH } from './session-search-engine-types' import type { SessionSearchRequest, SessionSearchResponse } from './session-search-engine-types' import { @@ -63,6 +64,20 @@ describe('the route ladder tries phrase, then AND, then repair, then OR', () => expect(ids(result).sort()).toEqual(['1', '2']) }) + it('keeps every term a repaired literal was typed with', async () => { + const { db, engine } = await open('ss-engine-typo-literal') + addSyntheticSession(db, { id: 1, text: 'parseJson the data' }) + addSyntheticSession(db, { id: 2, text: 'parseJson the data again' }) + // `parseJsonn(the, data)` is literal because of its punctuation; the + // corrected spelling read on its own is prose. Re-planning without carrying + // the original decision across would drop `the` and report a body that was + // never typed. + // A corrected term comes back in the index's own spelling, which unicode61 + // has folded; the terms the repair left alone keep the case they were typed. + const result = engine.search({ query: 'parseJsonn(the, data)' }) + expect(result.planner.repairedTerms).toEqual(['parsejson', 'the', 'data']) + }) + it('does not repair a term the index already holds', async () => { const { db, engine } = await open('ss-engine-no-typo') addSyntheticSession(db, { id: 1, text: 'coalesces' }) @@ -255,6 +270,39 @@ describe('source presence comes from the files table, never a stat', () => { }) }) +describe('an index that predates version 2 is answered from, not thrown at', () => { + async function version1(): Promise { + const opened = await open('ss-engine-v1') + addSyntheticSession(opened.db, { id: 1, text: 'the coalesces path is slow' }) + addSyntheticSession(opened.db, { id: 2, text: 'coalesces again here' }) + // A real v1 file: neither table version 2 added exists in it. + opened.db.exec('DROP TABLE messages_vocab; DROP TABLE search_log') + return opened + } + + it('still searches, and names the feature it cannot serve', async () => { + const { store } = await version1() + const engine = new SessionSearchEngine(store, { logQueries: true }) + const result = engine.search({ query: 'coalesces' }) + expect(ids(result).sort()).toEqual(['1', '2']) + expect(result.unavailable).toEqual(['typo-repair', 'query-log']) + }) + + it('skips the repair rung rather than reaching for a vocabulary that is gone', async () => { + const { store } = await version1() + const result = new SessionSearchEngine(store).search({ query: 'coalescs' }) + expect(result.planner.route).toBe('or') + expect(result.planner.repairedTerms).toBeUndefined() + expect(result.hits).toEqual([]) + }) + + it('claims nothing unavailable on a current index', async () => { + const { db, engine } = await open('ss-engine-v2') + addSyntheticSession(db, { id: 1 }) + expect(engine.search({ query: 'needle' }).unavailable).toEqual([]) + }) +}) + describe('the engine is the only thing that warms the index', () => { it('warms on the first search and leans on the store to memoize the rest', async () => { const { db, store, engine } = await open('ss-engine-warm') diff --git a/src/main/ai-vault-search/session-search-engine.ts b/src/main/ai-vault-search/session-search-engine.ts index d00c880f8f6..961dddea5c8 100644 --- a/src/main/ai-vault-search/session-search-engine.ts +++ b/src/main/ai-vault-search/session-search-engine.ts @@ -12,6 +12,7 @@ import { type SessionSearchResponse, type SessionSearchSourcePresence } from './session-search-engine-types' +import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities' import { rankSessionHits, type MessageRow, @@ -32,6 +33,7 @@ import { type Retrieved } from './session-search-retrieval' import { sessionRowFilter } from './session-search-row-filter' +import { sessionSearchUnavailableFeatures } from './session-search-index-capabilities' import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet' import { sessionSourcePresence } from './session-search-source-presence' import type { SessionSearchStore } from './session-search-store' @@ -79,13 +81,16 @@ export class SessionSearchEngine { private readonly db: SyncDatabase private readonly retrieval: SessionSearchRetrieval private readonly candidateLimit: number + /** Probed once: a version-1 file has neither vocabulary nor query log. */ + private readonly unavailable: readonly SessionSearchUnavailableFeature[] constructor( private readonly store: SessionSearchStore, private readonly options: SessionSearchEngineOptions = {} ) { this.db = store.connection - this.retrieval = new SessionSearchRetrieval(this.db) + this.unavailable = sessionSearchUnavailableFeatures(this.db) + this.retrieval = new SessionSearchRetrieval(this.db, !this.unavailable.includes('typo-repair')) this.candidateLimit = options.sessionCandidateLimit ?? SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT } @@ -129,6 +134,7 @@ export class SessionSearchEngine { const hasMore = ranked.length > offset + limit const response: SessionSearchResponse = { hits, + unavailable: this.unavailable, planner: { route: retrieved?.route ?? 'or', tier: scope, @@ -147,7 +153,7 @@ export class SessionSearchEngine { generation, durationMs: performance.now() - startedAt } - if (this.options.logQueries) { + if (this.options.logQueries && !this.unavailable.includes('query-log')) { logSessionSearchQuery(this.db, { query: request.query, route: response.planner.route, diff --git a/src/main/ai-vault-search/session-search-index-capabilities.ts b/src/main/ai-vault-search/session-search-index-capabilities.ts new file mode 100644 index 00000000000..ecaa8eb34e1 --- /dev/null +++ b/src/main/ai-vault-search/session-search-index-capabilities.ts @@ -0,0 +1,38 @@ +import type SyncDatabase from '../sqlite/sync-database' + +/** An engine feature the index on disk cannot serve. */ +export type SessionSearchUnavailableFeature = 'typo-repair' | 'query-log' + +/** + * What this index can answer, probed once when an engine opens over it. + * + * Why probe at all: schema version 2 added `messages_vocab` and `search_log`, + * and a mismatched file is normally dropped and rebuilt when it is opened. But + * an engine can be handed a connection to a version-1 file that another handle + * is still answering from, and PR 4 is what first makes that reachable. Reading + * the tables that do exist and saying plainly which feature is missing is a + * better answer than throwing on the first query that reaches for one. + * + * Which process may open, unlink and rebuild the index is PR 3b's decision, not + * this file's; this only keeps a reader useful while that is unsettled. + */ +export function sessionSearchUnavailableFeatures( + db: SyncDatabase +): SessionSearchUnavailableFeature[] { + const unavailable: SessionSearchUnavailableFeature[] = [] + if (!hasTable(db, 'messages_vocab')) { + unavailable.push('typo-repair') + } + if (!hasTable(db, 'search_log')) { + unavailable.push('query-log') + } + return unavailable +} + +function hasTable(db: SyncDatabase, name: string): boolean { + return ( + db + .prepare("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?") + .get(name) !== undefined + ) +} diff --git a/src/main/ai-vault-search/session-search-query-planner.ts b/src/main/ai-vault-search/session-search-query-planner.ts index 965dea17481..07566bfaa49 100644 --- a/src/main/ai-vault-search/session-search-query-planner.ts +++ b/src/main/ai-vault-search/session-search-query-planner.ts @@ -53,11 +53,21 @@ export function indexTokens(query: string, limit = Number.POSITIVE_INFINITY): st return out } -export function planSessionSearchQuery(query: string): SessionSearchQueryPlan { +/** + * `literal` overrides the shape test. Typo repair re-plans the query it + * corrected, and a corrected spelling can look like ordinary prose even though + * what was typed was a literal: `parseJsonn(the, data)` has the punctuation that + * makes it literal, `parsejson the data` does not. Without the override the + * re-plan would drop `the` as a stop word, so the repaired query would search + * for less than the original asked for and `repairedTerms` would report a body + * the user never typed. + */ +export function planSessionSearchQuery( + query: string, + literal = isLiteralQuery(query) +): SessionSearchQueryPlan { const raw = indexTokens(query, MAX_BODY_TERMS) - let body = isLiteralQuery(query) - ? raw - : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase())) + let body = literal ? raw : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase())) if (body.length < 2) { body = raw } @@ -71,7 +81,7 @@ export function planSessionSearchQuery(query: string): SessionSearchQueryPlan { } } return { - literal: isLiteralQuery(query), + literal, terms: [...terms, ...extra].slice(0, MAX_TERMS), body: body.slice(0, MAX_BODY_TERMS) } diff --git a/src/main/ai-vault-search/session-search-retrieval.ts b/src/main/ai-vault-search/session-search-retrieval.ts index fc99ed0e6f2..b93bb0c04bf 100644 --- a/src/main/ai-vault-search/session-search-retrieval.ts +++ b/src/main/ai-vault-search/session-search-retrieval.ts @@ -43,10 +43,14 @@ export function ftsTableFor(scope: SessionSearchScope): 'messages_fts' | 'conver /** The FTS half of a search: the route ladder and the SQL each rung runs. */ export class SessionSearchRetrieval { - private readonly typoRepair: SessionSearchTypoRepair + /** Null when this index has no vocabulary to repair against; the rung is skipped. */ + private readonly typoRepair: SessionSearchTypoRepair | null - constructor(private readonly db: SyncDatabase) { - this.typoRepair = new SessionSearchTypoRepair(db) + constructor( + private readonly db: SyncDatabase, + canRepairTypos = true + ) { + this.typoRepair = canRepairTypos ? new SessionSearchTypoRepair(db) : null } /** @@ -100,16 +104,23 @@ export class SessionSearchRetrieval { } private repair(plan: SessionSearchQueryPlan): SessionSearchQueryPlan | null { + if (!this.typoRepair) { + return null + } + const typoRepair = this.typoRepair let changed = false const body = plan.body.map((term) => { - const fix = this.typoRepair.correct(term) + const fix = typoRepair.correct(term) if (fix && fix !== term.toLowerCase()) { changed = true return fix } return term }) - return changed ? { ...planSessionSearchQuery(body.join(' ')), literal: plan.literal } : null + // The repair changes spellings, not the query's character: the re-plan is + // told what the original decided so a corrected literal keeps every term it + // was typed with. + return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null } /** Phrase, then AND, for literal-looking queries; null when neither matches. */