diff --git a/src/main/ai-vault-search/session-search-engine-types.ts b/src/main/ai-vault-search/session-search-engine-types.ts index a90fe1be34e..709ed9ccca5 100644 --- a/src/main/ai-vault-search/session-search-engine-types.ts +++ b/src/main/ai-vault-search/session-search-engine-types.ts @@ -1,5 +1,6 @@ import type { AiVaultAgent } from '../../shared/ai-vault-types' import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers' +import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities' // ENGINE types, deliberately not in src/shared: nothing here is a wire type. // PR 5 owns the public contract and lifts what a caller may actually receive; @@ -59,7 +60,13 @@ export type SessionSearchRoute = 'phrase' | 'and' | 'or' | 'typo+phrase' | 'typo */ export type SessionSearchPlannerReport = { route: SessionSearchRoute - /** Query terms after typo repair, when any were changed. */ + /** + * The whole body the repaired plan searched, in query order, when any term + * was changed. Not just the corrected terms: a caller rendering "searched + * for" needs the query it actually ran, and a repair never drops a term the + * original kept. A corrected term carries the index's own spelling, which the + * tokenizer has case-folded; untouched terms keep the case they were typed in. + */ repairedTerms?: string[] /** The corpus the route ran against; today always the requested scope. */ tier: SessionSearchScope @@ -120,6 +127,12 @@ export type SessionSearchTruncation = { export type SessionSearchResponse = { hits: SessionSearchHit[] + /** + * Engine features the index on disk cannot serve, empty on a current index. + * A route ladder missing its repair rung still answers; saying so is what + * keeps the answer honest. + */ + unavailable: readonly SessionSearchUnavailableFeature[] planner: SessionSearchPlannerReport page: SessionSearchPage truncated: SessionSearchTruncation diff --git a/src/main/ai-vault-search/session-search-engine.test.ts b/src/main/ai-vault-search/session-search-engine.test.ts index 1b2a6d246cd..7fbcd5e113a 100644 --- a/src/main/ai-vault-search/session-search-engine.test.ts +++ b/src/main/ai-vault-search/session-search-engine.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' +import { SessionSearchEngine } from './session-search-engine' import { SESSION_SEARCH_QUERY_MAX_LENGTH } from './session-search-engine-types' import type { SessionSearchRequest, SessionSearchResponse } from './session-search-engine-types' import { @@ -63,6 +64,20 @@ describe('the route ladder tries phrase, then AND, then repair, then OR', () => expect(ids(result).sort()).toEqual(['1', '2']) }) + it('keeps every term a repaired literal was typed with', async () => { + const { db, engine } = await open('ss-engine-typo-literal') + addSyntheticSession(db, { id: 1, text: 'parseJson the data' }) + addSyntheticSession(db, { id: 2, text: 'parseJson the data again' }) + // `parseJsonn(the, data)` is literal because of its punctuation; the + // corrected spelling read on its own is prose. Re-planning without carrying + // the original decision across would drop `the` and report a body that was + // never typed. + // A corrected term comes back in the index's own spelling, which unicode61 + // has folded; the terms the repair left alone keep the case they were typed. + const result = engine.search({ query: 'parseJsonn(the, data)' }) + expect(result.planner.repairedTerms).toEqual(['parsejson', 'the', 'data']) + }) + it('does not repair a term the index already holds', async () => { const { db, engine } = await open('ss-engine-no-typo') addSyntheticSession(db, { id: 1, text: 'coalesces' }) @@ -255,6 +270,39 @@ describe('source presence comes from the files table, never a stat', () => { }) }) +describe('an index that predates version 2 is answered from, not thrown at', () => { + async function version1(): Promise { + const opened = await open('ss-engine-v1') + addSyntheticSession(opened.db, { id: 1, text: 'the coalesces path is slow' }) + addSyntheticSession(opened.db, { id: 2, text: 'coalesces again here' }) + // A real v1 file: neither table version 2 added exists in it. + opened.db.exec('DROP TABLE messages_vocab; DROP TABLE search_log') + return opened + } + + it('still searches, and names the feature it cannot serve', async () => { + const { store } = await version1() + const engine = new SessionSearchEngine(store, { logQueries: true }) + const result = engine.search({ query: 'coalesces' }) + expect(ids(result).sort()).toEqual(['1', '2']) + expect(result.unavailable).toEqual(['typo-repair', 'query-log']) + }) + + it('skips the repair rung rather than reaching for a vocabulary that is gone', async () => { + const { store } = await version1() + const result = new SessionSearchEngine(store).search({ query: 'coalescs' }) + expect(result.planner.route).toBe('or') + expect(result.planner.repairedTerms).toBeUndefined() + expect(result.hits).toEqual([]) + }) + + it('claims nothing unavailable on a current index', async () => { + const { db, engine } = await open('ss-engine-v2') + addSyntheticSession(db, { id: 1 }) + expect(engine.search({ query: 'needle' }).unavailable).toEqual([]) + }) +}) + describe('the engine is the only thing that warms the index', () => { it('warms on the first search and leans on the store to memoize the rest', async () => { const { db, store, engine } = await open('ss-engine-warm') diff --git a/src/main/ai-vault-search/session-search-engine.ts b/src/main/ai-vault-search/session-search-engine.ts index d00c880f8f6..961dddea5c8 100644 --- a/src/main/ai-vault-search/session-search-engine.ts +++ b/src/main/ai-vault-search/session-search-engine.ts @@ -12,6 +12,7 @@ import { type SessionSearchResponse, type SessionSearchSourcePresence } from './session-search-engine-types' +import type { SessionSearchUnavailableFeature } from './session-search-index-capabilities' import { rankSessionHits, type MessageRow, @@ -32,6 +33,7 @@ import { type Retrieved } from './session-search-retrieval' import { sessionRowFilter } from './session-search-row-filter' +import { sessionSearchUnavailableFeatures } from './session-search-index-capabilities' import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet' import { sessionSourcePresence } from './session-search-source-presence' import type { SessionSearchStore } from './session-search-store' @@ -79,13 +81,16 @@ export class SessionSearchEngine { private readonly db: SyncDatabase private readonly retrieval: SessionSearchRetrieval private readonly candidateLimit: number + /** Probed once: a version-1 file has neither vocabulary nor query log. */ + private readonly unavailable: readonly SessionSearchUnavailableFeature[] constructor( private readonly store: SessionSearchStore, private readonly options: SessionSearchEngineOptions = {} ) { this.db = store.connection - this.retrieval = new SessionSearchRetrieval(this.db) + this.unavailable = sessionSearchUnavailableFeatures(this.db) + this.retrieval = new SessionSearchRetrieval(this.db, !this.unavailable.includes('typo-repair')) this.candidateLimit = options.sessionCandidateLimit ?? SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT } @@ -129,6 +134,7 @@ export class SessionSearchEngine { const hasMore = ranked.length > offset + limit const response: SessionSearchResponse = { hits, + unavailable: this.unavailable, planner: { route: retrieved?.route ?? 'or', tier: scope, @@ -147,7 +153,7 @@ export class SessionSearchEngine { generation, durationMs: performance.now() - startedAt } - if (this.options.logQueries) { + if (this.options.logQueries && !this.unavailable.includes('query-log')) { logSessionSearchQuery(this.db, { query: request.query, route: response.planner.route, diff --git a/src/main/ai-vault-search/session-search-index-capabilities.ts b/src/main/ai-vault-search/session-search-index-capabilities.ts new file mode 100644 index 00000000000..ecaa8eb34e1 --- /dev/null +++ b/src/main/ai-vault-search/session-search-index-capabilities.ts @@ -0,0 +1,38 @@ +import type SyncDatabase from '../sqlite/sync-database' + +/** An engine feature the index on disk cannot serve. */ +export type SessionSearchUnavailableFeature = 'typo-repair' | 'query-log' + +/** + * What this index can answer, probed once when an engine opens over it. + * + * Why probe at all: schema version 2 added `messages_vocab` and `search_log`, + * and a mismatched file is normally dropped and rebuilt when it is opened. But + * an engine can be handed a connection to a version-1 file that another handle + * is still answering from, and PR 4 is what first makes that reachable. Reading + * the tables that do exist and saying plainly which feature is missing is a + * better answer than throwing on the first query that reaches for one. + * + * Which process may open, unlink and rebuild the index is PR 3b's decision, not + * this file's; this only keeps a reader useful while that is unsettled. + */ +export function sessionSearchUnavailableFeatures( + db: SyncDatabase +): SessionSearchUnavailableFeature[] { + const unavailable: SessionSearchUnavailableFeature[] = [] + if (!hasTable(db, 'messages_vocab')) { + unavailable.push('typo-repair') + } + if (!hasTable(db, 'search_log')) { + unavailable.push('query-log') + } + return unavailable +} + +function hasTable(db: SyncDatabase, name: string): boolean { + return ( + db + .prepare("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?") + .get(name) !== undefined + ) +} diff --git a/src/main/ai-vault-search/session-search-query-planner.ts b/src/main/ai-vault-search/session-search-query-planner.ts index 965dea17481..07566bfaa49 100644 --- a/src/main/ai-vault-search/session-search-query-planner.ts +++ b/src/main/ai-vault-search/session-search-query-planner.ts @@ -53,11 +53,21 @@ export function indexTokens(query: string, limit = Number.POSITIVE_INFINITY): st return out } -export function planSessionSearchQuery(query: string): SessionSearchQueryPlan { +/** + * `literal` overrides the shape test. Typo repair re-plans the query it + * corrected, and a corrected spelling can look like ordinary prose even though + * what was typed was a literal: `parseJsonn(the, data)` has the punctuation that + * makes it literal, `parsejson the data` does not. Without the override the + * re-plan would drop `the` as a stop word, so the repaired query would search + * for less than the original asked for and `repairedTerms` would report a body + * the user never typed. + */ +export function planSessionSearchQuery( + query: string, + literal = isLiteralQuery(query) +): SessionSearchQueryPlan { const raw = indexTokens(query, MAX_BODY_TERMS) - let body = isLiteralQuery(query) - ? raw - : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase())) + let body = literal ? raw : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase())) if (body.length < 2) { body = raw } @@ -71,7 +81,7 @@ export function planSessionSearchQuery(query: string): SessionSearchQueryPlan { } } return { - literal: isLiteralQuery(query), + literal, terms: [...terms, ...extra].slice(0, MAX_TERMS), body: body.slice(0, MAX_BODY_TERMS) } diff --git a/src/main/ai-vault-search/session-search-retrieval.ts b/src/main/ai-vault-search/session-search-retrieval.ts index fc99ed0e6f2..b93bb0c04bf 100644 --- a/src/main/ai-vault-search/session-search-retrieval.ts +++ b/src/main/ai-vault-search/session-search-retrieval.ts @@ -43,10 +43,14 @@ export function ftsTableFor(scope: SessionSearchScope): 'messages_fts' | 'conver /** The FTS half of a search: the route ladder and the SQL each rung runs. */ export class SessionSearchRetrieval { - private readonly typoRepair: SessionSearchTypoRepair + /** Null when this index has no vocabulary to repair against; the rung is skipped. */ + private readonly typoRepair: SessionSearchTypoRepair | null - constructor(private readonly db: SyncDatabase) { - this.typoRepair = new SessionSearchTypoRepair(db) + constructor( + private readonly db: SyncDatabase, + canRepairTypos = true + ) { + this.typoRepair = canRepairTypos ? new SessionSearchTypoRepair(db) : null } /** @@ -100,16 +104,23 @@ export class SessionSearchRetrieval { } private repair(plan: SessionSearchQueryPlan): SessionSearchQueryPlan | null { + if (!this.typoRepair) { + return null + } + const typoRepair = this.typoRepair let changed = false const body = plan.body.map((term) => { - const fix = this.typoRepair.correct(term) + const fix = typoRepair.correct(term) if (fix && fix !== term.toLowerCase()) { changed = true return fix } return term }) - return changed ? { ...planSessionSearchQuery(body.join(' ')), literal: plan.literal } : null + // The repair changes spellings, not the query's character: the re-plan is + // told what the original decided so a corrected literal keeps every term it + // was typed with. + return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null } /** Phrase, then AND, for literal-looking queries; null when neither matches. */