diff --git a/.gitignore b/.gitignore index 913dfc4a045..5bb1fedcaee 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ docs/** !docs/agent-skill-sharing-implementation-checklist.md !docs/mobile-terminal-shortcut-bar.md !docs/reference/ +!docs/reference/agent-session-search-query-tuning.md !docs/reference/agent-status-store.md !docs/reference/git-compatibility.md !docs/reference/headless-linux-server.md diff --git a/config/scripts/session-search-query-benchmark.ts b/config/scripts/session-search-query-benchmark.ts new file mode 100644 index 00000000000..1471a0a17bd --- /dev/null +++ b/config/scripts/session-search-query-benchmark.ts @@ -0,0 +1,192 @@ +import { rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { + createSessionParseStats, + parseAgentSessionFileCached, + resetSessionParseCacheForTests +} from '../../src/main/ai-vault/session-scanner-parse-cache' +import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers' +import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine' +import type { + SessionSearchRequest, + SessionSearchScope +} from '../../src/main/ai-vault-search/session-search-engine-types' +import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer' +import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store' +import type SyncDatabase from '../../src/main/sqlite/sync-database' +import { + writeSyntheticTranscriptCorpus, + type SyntheticCorpus, + type SyntheticCorpusOptions +} from '../../src/main/ai-vault-search/session-search-synthetic-corpus' +import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures' + +// What a query costs, and what the session candidate limit buys. Everything +// runs through the real store and the real engine over a synthetic corpus; +// never point this at a real transcript tree. + +const WARMUP = 5 +const SAMPLES = 25 + +// One query per rung the ladder can take, plus the two shapes that skip it. +const QUERIES: { name: string; request: SessionSearchRequest }[] = [ + { name: 'phrase', request: { query: '"terminal reattach"' } }, + { name: 'identifier', request: { query: 'resolveTerminalPath' } }, + { name: 'path', request: { query: 'src/main/ai-vault/session-transcript-reader.ts' } }, + { name: 'prose', request: { query: 'why is the daemon snapshot stale' } }, + { name: 'typo', request: { query: 'reattahc worktre' } }, + { name: 'common-term', request: { query: 'index' } }, + { name: 'operator-only', request: { query: 'repo:app-3' } }, + { name: 'scoped', request: { query: 'worktree', filters: { scopePaths: ['/repo/app-3'] } } } +] + +type Timing = { p50: number; p95: number } + +function percentile(sorted: readonly number[], fraction: number): number { + const at = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction)) + return Math.round((sorted[at] ?? 0) * 100) / 100 +} + +function timing(samples: number[]): Timing { + const sorted = [...samples].sort((left, right) => left - right) + return { p50: percentile(sorted, 0.5), p95: percentile(sorted, 0.95) } +} + +function time(engine: SessionSearchEngine, request: SessionSearchRequest): number { + const started = performance.now() + engine.search(request) + return performance.now() - started +} + +async function indexCorpus( + options: SyntheticCorpusOptions +): Promise<{ corpus: SyntheticCorpus; db: SyncDatabase; release: () => void }> { + resetSessionParseCacheForTests() + const corpus = await writeSyntheticTranscriptCorpus(options) + const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => { + throw error + }) + const unregister = registerSessionSearchIndexConsumer(store) + const stats = createSessionParseStats() + for (const path of corpus.files) { + await parseAgentSessionFileCached( + await sessionCandidate('claude', path), + process.platform, + stats + ) + } + return { + corpus, + // The handle a composed reader gets. Every read here is one synchronous + // statement, which is the contract that comes with it. + db: store.connection, + release: () => { + unregister() + resetTranscriptConsumersForTests() + resetSessionParseCacheForTests() + store.close() + } + } +} + +/** Per-query and overall latency for one scope. */ +function scopeReport(db: SyncDatabase, scope: SessionSearchScope): Record { + const engine = new SessionSearchEngine(db) + const everything: number[] = [] + const perQuery: Record = {} + for (const { name, request } of QUERIES) { + const scoped = { ...request, scope } + for (let run = 0; run < WARMUP; run++) { + engine.search(scoped) + } + const samples = Array.from({ length: SAMPLES }, () => time(engine, scoped)) + everything.push(...samples) + const result = engine.search(scoped) + perQuery[name] = { ...timing(samples), hits: result.hits.length, route: result.planner.route } + } + return { ...timing(everything), perQuery } +} + +/** + * The candidate limit only costs anything once there are more matching sessions + * than the limit, so this runs over many short sessions rather than the wide + * corpus above. Limits are interleaved sample by sample: run back to back, the + * first configuration pays for every page the OS cache had not seen yet and the + * ordering alone moves p95 by more than the limit does. + */ +function candidateSweep(db: SyncDatabase, limits: readonly number[]): Record { + const request: SessionSearchRequest = { query: 'index', limit: 20 } + const engines = new Map( + limits.map((limit) => [limit, new SessionSearchEngine(db, { sessionCandidateLimit: limit })]) + ) + const samples = new Map(limits.map((limit) => [limit, [] as number[]])) + for (let run = 0; run < WARMUP; run++) { + for (const engine of engines.values()) { + engine.search(request) + } + } + for (let run = 0; run < SAMPLES; run++) { + for (const limit of limits) { + samples.get(limit)!.push(time(engines.get(limit)!, request)) + } + } + const report: Record = {} + for (const limit of limits) { + const result = engines.get(limit)!.search(request) + report[String(limit)] = { + ...timing(samples.get(limit)!), + truncated: result.truncated.candidates, + // Pages a caller could walk before the limit stops handing out sessions. + reachablePages: Math.ceil(limit / (request.limit ?? 20)) + } + } + return report +} + +const wide = await indexCorpus({ sessions: Number(process.env.SESSIONS ?? 40) }) +let report: string +try { + const scope = { + all: scopeReport(wide.db, 'all'), + conversation: scopeReport(wide.db, 'conversation') + } + wide.release() + await rm(wide.corpus.root, { recursive: true, force: true }) + + // Many short sessions: what makes the candidate limit binding is the session + // count, not the byte count. + const many = await indexCorpus({ sessions: 2500, turnsPerSession: 1, seed: 7 }) + try { + report = JSON.stringify( + { + scopeCorpus: { + sessions: wide.corpus.files.length, + transcriptMb: Math.round((wide.corpus.transcriptBytes / 1024 / 1024) * 100) / 100, + messages: wide.corpus.messageCount + }, + scope, + candidateCorpus: { + sessions: many.corpus.files.length, + transcriptMb: Math.round((many.corpus.transcriptBytes / 1024 / 1024) * 100) / 100 + }, + candidateSweep: candidateSweep(many.db, [200, 600, 1200, 2400]) + }, + null, + 2 + ) + } finally { + many.release() + await rm(many.corpus.root, { recursive: true, force: true }) + } +} catch (error) { + await rm(wide.corpus.root, { recursive: true, force: true }) + throw error +} + +// Why a file as well as stdout: a runner that intercepts console output +// (vitest does) would otherwise swallow the whole report. +const out = process.env.BENCH_OUT +if (out) { + await writeFile(out, `${report}\n`) +} +console.log(report) diff --git a/config/scripts/session-search-scope-benchmark.ts b/config/scripts/session-search-scope-benchmark.ts new file mode 100644 index 00000000000..306387cbdda --- /dev/null +++ b/config/scripts/session-search-scope-benchmark.ts @@ -0,0 +1,205 @@ +import { rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { + createSessionParseStats, + parseAgentSessionFileCached, + resetSessionParseCacheForTests +} from '../../src/main/ai-vault/session-scanner-parse-cache' +import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers' +import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine' +import type { + SessionSearchRequest, + SessionSearchScope +} from '../../src/main/ai-vault-search/session-search-engine-types' +import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer' +import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store' +import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures' +import type SyncDatabase from '../../src/main/sqlite/sync-database' +import { writeToolHeavyCorpus, type ToolHeavyCorpus } from './session-search-tool-heavy-corpus' + +// What each scope costs on an index the size of a real transcript tree. +// +// The 10.5 MB corpus in `session-search-query-benchmark.ts` sizes the route +// ladder; this one sizes the corpus. `conversation` is a column filter over the +// one FTS table rather than a second table of its own, and the whole cost of +// that decision is how much of `messages_fts` a conversation query has to read +// past — which is set by how much of a transcript is tool output. +// +// Synthetic, always: this must never be pointed at a real transcript. + +const WARMUP = 5 + +/** Conversation-shaped queries; every term is one the prose actually uses. */ +const QUERIES = [ + 'terminal reattach', + 'stale snapshot', + 'daemon cursor', + 'worktree index', + 'publish transaction', + 'relay daemon', + 'session cursor', + 'because stale', + 'terminal worktree', + 'index snapshot', + 'reattach cursor', + 'transaction relay', + 'snapshot session', + 'daemon publish', + 'worktree terminal', + 'cursor index', + 'stale relay', + 'session transaction', + 'publish snapshot', + 'reattach daemon' +] + +async function indexCorpus( + corpus: ToolHeavyCorpus +): Promise<{ db: SyncDatabase; release: () => void }> { + resetSessionParseCacheForTests() + const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => { + throw error + }) + const unregister = registerSessionSearchIndexConsumer(store) + const stats = createSessionParseStats() + for (const path of corpus.files) { + await parseAgentSessionFileCached( + await sessionCandidate('claude', path), + process.platform, + stats + ) + } + return { + // The store's own handle, which is what a composed reader gets: every + // retrieval is one synchronous statement, so nothing pins a WAL snapshot. + db: store.connection, + release: () => { + unregister() + resetTranscriptConsumersForTests() + resetSessionParseCacheForTests() + store.close() + } + } +} + +type Timing = { p50: number; p95: number } + +function timing(samples: readonly number[]): Timing { + const sorted = [...samples].sort((left, right) => left - right) + const at = (fraction: number): number => { + const index = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction)) + return Math.round((sorted[index] ?? 0) * 100) / 100 + } + return { p50: at(0.5), p95: at(0.95) } +} + +/** + * The query sets, one per rung of the ladder the engine may take. + * + * Which rung each one reaches is not forced, it is observed: samples are + * bucketed by the route the engine reports, so the table says what was measured + * rather than what was intended, and a query that lands on a different rung + * than expected shows up as a bucket rather than as a wrong number. + */ +function queries(): string[] { + const run = (index: number, length: number): string => + Array.from({ length }, (_unused, step) => QUERIES[(index + step) % QUERIES.length]).join(' ') + return [ + // Two terms, unquoted: not literal, so straight to OR. + ...QUERIES, + // Two terms, quoted: literal, and on this corpus any two of fourteen words + // sit next to each other somewhere, so the phrase rung answers. + ...QUERIES.map((query) => `"${query}"`), + // Eight terms, quoted: an ordered run that long does not occur in 105 MB of + // draws from fourteen words, so the phrase rung misses and AND answers. + ...QUERIES.map((_query, index) => `"${run(index, 4)}"`) + ] +} + +type Bucket = { samples: number[]; hits: number } + +/** + * Both scopes over the same queries, interleaved scope by scope: run back to + * back, the first one pays for every page the OS cache had not seen and the + * ordering moves p95 more than the scope does. + */ +function scopeReport(db: SyncDatabase): Record { + const engine = new SessionSearchEngine(db) + const scopes: SessionSearchScope[] = ['all', 'conversation'] + const requests: SessionSearchRequest[] = queries().map((query) => ({ query })) + const buckets = new Map() + for (let run = 0; run < WARMUP; run++) { + for (const scope of scopes) { + for (const request of requests) { + engine.search({ ...request, scope }) + } + } + } + for (const request of requests) { + for (const scope of scopes) { + const started = performance.now() + const result = engine.search({ ...request, scope }) + const elapsed = performance.now() - started + const key = `${result.planner.route}/${scope}` + const bucket = buckets.get(key) ?? { samples: [], hits: 0 } + bucket.samples.push(elapsed) + bucket.hits += result.hits.length + buckets.set(key, bucket) + } + } + const report: Record = {} + for (const [key, bucket] of [...buckets].sort(([left], [right]) => left.localeCompare(right))) { + report[key] = { ...timing(bucket.samples), samples: bucket.samples.length, hits: bucket.hits } + } + return report +} + +/** Bytes the FTS table occupies, which is the cost the deleted second table saved. */ +function indexBytes(db: SyncDatabase): Record | { unavailable: string } { + try { + const sum = (where: string, ...values: string[]): number => + Number( + ( + db + .prepare(`SELECT COALESCE(SUM(pgsize),0) AS bytes FROM dbstat ${where}`) + .get(...values) as { bytes: number } + ).bytes + ) + return { total: sum(''), messagesFts: sum('WHERE name LIKE ?', 'messages_fts%') } + } catch { + // dbstat is a compile-time option; the latency numbers stand without it. + return { unavailable: 'no dbstat' } + } +} + +const corpus = await writeToolHeavyCorpus({ + targetBytes: Number(process.env.CORPUS_MB ?? 100) * 1024 * 1024, + toolShare: Number(process.env.TOOL_SHARE ?? 0.9) +}) +let report: string +const indexed = await indexCorpus(corpus) +try { + report = JSON.stringify( + { + corpus: { + sessions: corpus.files.length, + transcriptMb: Math.round((corpus.transcriptBytes / 1024 / 1024) * 100) / 100, + toolShareOfMessageText: + Math.round((corpus.toolBytes / (corpus.toolBytes + corpus.proseBytes)) * 1000) / 1000 + }, + indexBytes: indexBytes(indexed.db), + route: scopeReport(indexed.db) + }, + null, + 2 + ) +} finally { + indexed.release() + await rm(corpus.root, { recursive: true, force: true }) +} + +const out = process.env.BENCH_OUT +if (out) { + await writeFile(out, `${report}\n`) +} +console.log(report) diff --git a/config/scripts/session-search-tool-heavy-corpus.ts b/config/scripts/session-search-tool-heavy-corpus.ts new file mode 100644 index 00000000000..050535a00cf --- /dev/null +++ b/config/scripts/session-search-tool-heavy-corpus.ts @@ -0,0 +1,152 @@ +import { mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +// The corpus the scope benchmark runs over. Written here rather than by +// `session-search-synthetic-corpus.ts` because what it costs to answer a +// conversation query out of the one FTS table turns on the property that +// generator fixes: how much of a transcript is tool output. +// +// Synthetic, always. This must never be pointed at a real transcript. + +const PROSE = [ + 'terminal', + 'reattach', + 'worktree', + 'the', + 'index', + 'cursor', + 'publish', + 'transaction', + 'relay', + 'daemon', + 'snapshot', + 'because', + 'stale', + 'session' +] +// Tool output is paths, hashes and log lines — and the same words the +// conversation uses, because a `rg` over this repository prints them. That +// overlap is what the benchmark turns on: it is what makes a conversation +// term's posting list carry rows the column filter then has to discard. A tool +// vocabulary disjoint from the prose would leave nothing to discard and measure +// the wrong thing. +const TOOL_ONLY = [ + 'src/main/ai-vault/session-transcript-reader.ts', + 'node_modules/.pnpm/typescript@5.9.2', + '0x00007ff8', + 'ENOENT', + 'drwxr-xr-x', + '2026-09-10T00:00:00.000Z', + 'sha256:9f2c1a', + 'chunk-VHQ4NWQK.js', + 'warning:', + 'resolveTerminalPath', + 'byteOffset', + 'MAX_RETRIES' +] +// Half the tool tokens are conversation words. Deliberately pessimistic: the +// more of a query term lives in `tool_text`, the more the column filter costs, +// so a number measured here holds on a real transcript tree. +const TOOL = [...PROSE, ...TOOL_ONLY] + +function mulberry32(seed: number): () => number { + let state = seed >>> 0 + return () => { + state = (state + 0x6d2b79f5) >>> 0 + let t = Math.imul(state ^ (state >>> 15), 1 | state) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +function words(random: () => number, vocabulary: readonly string[], count: number): string { + const out: string[] = [] + for (let index = 0; index < count; index++) { + out.push(vocabulary[Math.floor(random() * vocabulary.length)]!) + } + return out.join(' ') +} + +export type ToolHeavyCorpus = { + root: string + files: string[] + transcriptBytes: number + toolBytes: number + proseBytes: number +} + +/** + * Claude JSONL transcripts whose tool output is `toolShare` of the message text. + * One turn is a user question, an assistant answer, a tool call and its output; + * only the last one grows with the share. + */ +export async function writeToolHeavyCorpus(args: { + targetBytes: number + toolShare: number + seed?: number +}): Promise { + const random = mulberry32(args.seed ?? 11) + const root = await mkdtemp(join(tmpdir(), 'orca-search-convfts-')) + const files: string[] = [] + const proseWordsPerTurn = 160 + // Tool and prose words are not the same length, so the share is over bytes. + const proseBytesPerTurn = proseWordsPerTurn * 6 + const toolWordCount = Math.max( + 1, + Math.round((proseBytesPerTurn * args.toolShare) / (1 - args.toolShare) / 22) + ) + let transcriptBytes = 0 + let toolBytes = 0 + let proseBytes = 0 + for (let session = 0; transcriptBytes < args.targetBytes; session++) { + const sessionId = `00000000-0000-4000-8000-${String(session).padStart(12, '0')}` + const lines: string[] = [] + for (let turn = 0; turn < 40; turn++) { + const at = new Date(1740000000000 + turn * 60_000).toISOString() + const question = words(random, PROSE, 40) + const answer = words(random, PROSE, proseWordsPerTurn - 40) + const output = words(random, TOOL, toolWordCount) + proseBytes += Buffer.byteLength(question) + Buffer.byteLength(answer) + toolBytes += Buffer.byteLength(output) + lines.push( + JSON.stringify({ + type: 'user', + sessionId, + timestamp: at, + cwd: `/repo/app-${session % 7}`, + gitBranch: 'main', + message: { role: 'user', content: question } + }), + JSON.stringify({ + type: 'assistant', + sessionId, + timestamp: at, + message: { + role: 'assistant', + model: 'claude-fable-5', + content: [ + { type: 'text', text: answer }, + { type: 'tool_use', name: 'Bash', input: { command: 'rg needle' } } + ] + } + }), + JSON.stringify({ + type: 'user', + sessionId, + timestamp: at, + message: { + role: 'user', + content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: output }] + } + }) + ) + } + const path = join(root, `${sessionId}.jsonl`) + const body = `${lines.join('\n')}\n` + await writeFile(path, body) + transcriptBytes += Buffer.byteLength(body) + files.push(path) + } + return { root, files, transcriptBytes, toolBytes, proseBytes } +} diff --git a/docs/reference/agent-session-search-query-tuning.md b/docs/reference/agent-session-search-query-tuning.md new file mode 100644 index 00000000000..fcae799f8a6 --- /dev/null +++ b/docs/reference/agent-session-search-query-tuning.md @@ -0,0 +1,218 @@ +# Agent session search: query tuning + +What a search costs, and what the knobs in `src/main/ai-vault-search/session-search-engine.ts` +buy. Every number here comes from `config/scripts/session-search-query-benchmark.ts` +over the synthetic corpus in `session-search-synthetic-corpus.ts`, except the +`conversation_fts` shoot-out, which writes its own corpus because the answer +turns on how much of a transcript is tool output. Nothing in this file was +measured against a real transcript, and neither benchmark must ever be pointed +at one. + +## Running it + +The benchmark is a top-level-await module that imports the main-process tree by +extensionless path, so it needs a bundler-backed runner rather than bare `node`: + +```sh +cat > src/main/ai-vault-search/zz-bench.test.ts <<'EOF' +import { it } from 'vitest' +it('runs', { timeout: 1_800_000 }, async () => { + await import('../../../config/scripts/session-search-query-benchmark') +}) +EOF +BENCH_OUT=/tmp/ss-query-bench.json pnpm test src/main/ai-vault-search/zz-bench.test.ts +rm src/main/ai-vault-search/zz-bench.test.ts +``` + +The `conversation_fts` shoot-out below runs the same way, importing +`config/scripts/session-search-conversation-fts-benchmark` instead, with +`CORPUS_MB` and `TOOL_SHARE` to size and shape its corpus. `config/scripts` is +not inside any typecheck project, so while that throwaway test exists `tsc` +reports TS6307 for each script it pulls in; delete it and the run is clean +again. + +`BENCH_OUT` exists because vitest intercepts `console.log`; the report is written +to that path as well as printed. + +## Scope: what the second FTS table buys a reader + +Corpus: 40 synthetic Claude transcripts, 10.5 MB, 9,600 messages, indexed through +the real store. Eight queries, one per rung of the route ladder plus the two +shapes that skip it; 5 warm-up runs and 25 samples each. Apple silicon, warm page +cache, machine otherwise idle. Milliseconds, and p95 over 25 samples moves +several milliseconds run to run if anything else is competing for the disk. + +| Scope | p50 | p95 | +| -------------- | ---- | ---- | +| `all` | 7.22 | 8.94 | +| `conversation` | 5.33 | 7.86 | + +Per query, `all` then `conversation` (p50 / p95): + +| Query | `all` | `conversation` | +| ------------------------------------------------ | ------------ | -------------- | +| `"terminal reattach"` (phrase) | 5.24 / 8.42 | 2.97 / 3.24 | +| `resolveTerminalPath` (identifier) | 7.55 / 8.94 | 6.47 / 6.72 | +| `src/main/…/session-transcript-reader.ts` (path) | 8.69 / 10.12 | 7.78 / 8.04 | +| `why is the daemon snapshot stale` (prose) | 7.84 / 8.57 | 5.90 / 7.01 | +| `reattahc worktre` (typo repair) | 7.30 / 7.39 | 5.53 / 5.89 | +| `index` (common term) | 5.45 / 5.66 | 3.81 / 4.02 | +| `repo:app-3` (operator only) | 0.12 / 0.16 | 0.10 / 0.10 | +| `worktree` scoped to one cwd | 1.47 / 1.63 | 1.25 / 1.49 | + +Reading it: + +- `conversation` is about 1.4x faster at p50 and 1.1x at p95, and it is a column + filter over the same table rather than a table of its own. Narrowing to the + two prose columns is what buys the gap: fewer postings to score. It is also + the scope where a match is something a person wrote rather than something a + tool printed. +- A `scopePaths` query is the cheapest real search on the page. It is the one + narrowing SQL can express exactly, so it seeks `sessions_cwd_key` and hands + ranking a small candidate set. +- The operator-only figure is a floor, not a typical cost. `repo:` and `path:` + are applied in JS over retrieved rows (see `session-search-row-filter` for why + they cannot be pushed into SQL), so their cost tracks how many sessions the + walk has to read before it fills a candidate set. This corpus has 40 sessions, + which is one page of that walk; an index where few sessions match the operator + will read up to the ceiling in `session-search-retrieval` instead. + +## What the conversation scope costs at real corpus size + +`conversation` was a second FTS table holding a copy of the two prose columns. +It is a column filter now — `{user_text assistant_text}: (…)` with bm25 weights +that zero the other two — and PR 2 deleted the table on the strength of the +shoot-out this section used to hold: the filter came in at 1.16-1.36x the p95 of +the dedicated table, under the 2x bar, while the table cost a tenth of the index +to maintain. What follows is what the shipped schema actually does, measured +again on the same corpus after the table went and tool rows were capped. + +Corpus: Claude transcripts from `config/scripts/session-search-tool-heavy-corpus.ts`, +105 MB, indexed through the real store, at two points in the 80-97% band a real +transcript tree sits in. Half the tokens in tool output are words the +conversation also uses, so a conversation term really does have postings the +filter must discard. Twenty queries per rung, both scopes interleaved query by +query, warm cache; `config/scripts/session-search-scope-benchmark.ts`, run twice. + +| Tool share | Rung | `all` p50 / p95 | `conversation` p50 / p95 | +| ---------- | ------ | --------------- | ------------------------ | +| 86% | phrase | 16.69 / 17.48 | 13.08 / 13.52 | +| 86% | or | 31.91 / 35.74 | 22.25 / 23.87 | +| 86% | and | 70.04 / 74.00 | 53.47 / 59.39 | +| 93% | phrase | 9.14 / 13.36 | 7.23 / 8.51 | +| 93% | or | 16.46 / 18.70 | 12.34 / 14.88 | +| 93% | and | 39.65 / 43.44 | 31.05 / 32.92 | + +Three things to read out of it. + +**The filter is a win, not a cost.** Every rung is faster narrow than wide, by +1.2x to 1.4x at p50. The shoot-out compared the filter against a table built for +exactly this query; against the wide table it replaces, it does what the second +table did, which is read fewer postings. + +**The `and` rung is where the corpus size shows.** Those queries are eight terms, +chosen so no ordered run that long occurs and the phrase rung has to miss; a +real two-term AND sits nearer the phrase row. It is also the noisiest: the +second run's p95 reached 140 ms on one bucket, which is what twenty samples of a +70 ms query buys. Read the p50 column. + +**The index is far smaller than the shoot-out's was.** 57 MB at 93% tool output +and 103 MB at 86%, against roughly 150 MB for `messages_fts` alone before PR 2 +capped an indexed tool row at 3,072 characters. Most of a tool-heavy transcript +is now not in the index at all, which moves every number above and is the larger +effect of the two. + +What is **not** measured here is relevance, and the column filter does carry one +ranking difference the deleted table did not. FTS5's bm25 normalises by the +whole row's length and has no per-column length, so two rows with identical +prose score differently when one also holds tool output. The rowid set is +unchanged, which is what the deletion was decided on; the order within it can +move. `session-search-engine.test.ts` pins the direction. + +## `sessionCandidateLimit` + +The reviewer's F13: this is a tunable default, not a constant. It bounds how many +sessions the SQL hands ranking, so it bounds both retrieval cost and how deep a +caller can page before the answer simply stops. + +The limit only costs anything once more sessions match than the limit allows, so +this is measured over a second corpus: 2,500 one-turn transcripts, 10.9 MB, every +one of them matching the query. Limits are interleaved sample by sample, because +run back to back the first configuration pays for every page the OS cache had not +seen and the ordering alone moves p95 further than the limit does. + +| Limit | p50 | p95 | Pages of 20 a caller can reach | +| ----- | ----- | ----- | ------------------------------ | +| 200 | 6.85 | 7.21 | 10 | +| 600 | 7.93 | 8.36 | 30 | +| 1200 | 9.55 | 10.53 | 60 | +| 2400 | 12.32 | 13.45 | 120 | + +600 is the default: it costs about 16% over 200 at p50 and buys three times the +reachable depth, and the curve only turns steep past 1200. A host with a much +larger index can raise it; the result's `truncated.candidates` says when the limit +was the thing that cut the answer, so a caller never has to guess. + +What is **not** measured here is relevance. These numbers say what a limit costs, +not what it retrieves. The MRR figures quoted in the BM25 weights +(`session-search-retrieval.ts`) and in the identifier shadow column +(`session-search-identifier-split.ts`) come from the original retrieval shoot-out +on real transcripts and are not reproducible from this repository. Any change to +the limit justified on relevance grounds needs an eval set, not this benchmark. + +## What typo repair costs + +The repair is the one rung whose cost tracks the size of the vocabulary rather +than the size of a result. It only runs for a term the scope has no posting for, +so an ordinary query never pays it; a query of nonsense pays it once per term. + +Measured over a synthetic vocabulary of 1.6 M distinct terms, every term in two +rows so none is filtered out: + +| Query | p50 | +| -------------------------------------- | ------ | +| one known term (no repair) | 11 ms | +| one unknown term | 10 ms | +| 39 unknown 12-character terms (480 ch) | 387 ms | +| 12 unknown 40-character terms | 99 ms | + +Two things follow. The cost is linear in unknown terms and in vocabulary size, +and `search` is synchronous, so a 512-character query of nonsense holds the +thread for a third of a second on an index that large. And the scoped-count fix +made this cheaper rather than dearer — it was 737 ms before — because ordering +the vocabulary scan by term drops the sort that ordering by `doc` required, and +the counts it added are at most eight bounded probes per prefix. A cap on +unknown terms per query is recorded as a follow-up in the split plan. + +## Page warmup, dropped + +PR 2 deferred `warm()` — a sliced read of `messages` that pulls its pages into +the OS cache before the first query — to whoever knew which pages a read +touches. It is not re-added here, for two reasons. The measurement that +justified it (first query 1.3 s to 0.45 s) was on a 4 GB index, and neither +corpus in this file is within an order of magnitude of that, so PR 4 cannot +show a win: removing the call moved the 10.5 MB corpus's p50 by less than the +run-to-run spread. And it is a cancellable background pass, which needs an owner +with a lifecycle; a query library that holds no timers has nothing to hang the +`stopped()` on, and a fire-and-forget async read from a synchronous `search` is +a rejection nothing can supervise. It belongs with the indexer in PR 3b, which +already owns starting and stopping work. + +## Not settled here + +Which process may open, unlink and rebuild the index is PR 3b's decision. A +second handle that finds an older schema version replaces the file while a live +store keeps answering from the unlinked inode, and this PR is what first makes +that reachable, because it is the first thing that reads. What PR 4 does is +refuse to make it worse. The engine carries its own schema — the vocabulary, the +query log and the generation triggers — and re-creates whatever of it is missing +on every search, so a dropped object heals rather than degrading. + +The one it cannot re-create is the vocabulary's source, because `messages_fts` +is the store's. With one FTS table that is also the end of the degrade: there is +no second corpus to answer from, so an engine over an index mid-rebuild names +typo repair as unavailable and then fails on the table it cannot read, which is +the honest outcome — an empty page would read as an answer. `unavailable` can +therefore no longer be reported alongside a successful search, and PR 5 should +decide whether the field survives into the contract; it becomes reachable again +the day something opens the index read-only. diff --git a/src/main/ai-vault-search/session-search-engine-test-fixture.ts b/src/main/ai-vault-search/session-search-engine-test-fixture.ts new file mode 100644 index 00000000000..694a3d6397f --- /dev/null +++ b/src/main/ai-vault-search/session-search-engine-test-fixture.ts @@ -0,0 +1,113 @@ +import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers' +import type SyncDatabase from '../sqlite/sync-database' +import { SessionSearchEngine, type SessionSearchEngineOptions } from './session-search-engine' +import { cwdKey } from './session-search-file-records' +import { identifierShadowText } from './session-search-identifier-split' +import { SessionSearchStore } from './session-search-store' +import { + openSessionSearchIndexFile, + type SessionSearchIndexFile +} from './session-search-index-test-fixture' + +// Synthetic index rows for the query tests. The write path has its own tests; +// driving it here would make every retrieval assertion depend on the parser. + +export type SessionSearchHarness = { + /** The engine's own connection; the store next to it keeps a second, private one. */ + db: SyncDatabase + /** A real writer on the same file, so a test can move the index under the engine. */ + store: SessionSearchStore + engine: SessionSearchEngine + close: () => Promise +} + +export async function openSessionSearchHarness( + name: string, + options: SessionSearchEngineOptions = {} +): Promise { + const index: SessionSearchIndexFile = await openSessionSearchIndexFile(name) + const store = new SessionSearchStore(index.path, (error) => { + throw error + }) + // Constructed before any row is planted, because constructing it is what + // installs the generation triggers the planted rows have to move. + const engine = new SessionSearchEngine(index.db, options) + return { + db: index.db, + store, + engine, + close: async () => { + store.close() + await index.close() + } + } +} + +export type SyntheticSession = { + id: number + cwd?: string | null + text?: string + /** Rows of `text` to write; one session with many rows is one hit. */ + rows?: number + role?: TranscriptMessageRole + /** + * Written into `tool_text` alongside `text`, which is the one row shape the + * conversation scope has to exclude while the `all` scope keeps it. + */ + toolText?: string + agent?: string + updatedAt?: string + messageCount?: number + /** Written into `files`, which is what makes the source `present`. */ + filePath?: string | null + /** `sessions.file_path`: the transcript `path:` searches alongside cwd. */ + sessionFilePath?: string +} + +/** One session and its message rows, in both FTS tables the way the writer does. */ +export function addSyntheticSession(db: SyncDatabase, session: SyntheticSession): void { + const { + id, + cwd = '/repo/app', + text = 'needle', + rows = 1, + role = 'user', + toolText = '', + agent = 'claude', + updatedAt = `2026-09-${String((id % 28) + 1).padStart(2, '0')}T00:00:00.000Z`, + messageCount = rows, + filePath = `/synthetic/${id}.jsonl`, + sessionFilePath = `/synthetic/${id}.jsonl` + } = session + db.prepare( + `INSERT INTO sessions(id,agent,session_id,file_path,title,cwd,cwd_key,updated_at,message_count,resume_command) + VALUES (?,?,?,?,'fixture',?,?,?,?,'resume')` + ).run(id, agent, String(id), sessionFilePath, cwd, cwdKey(cwd), updatedAt, messageCount) + if (filePath !== null) { + db.prepare( + 'INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES (?,0,1740000000000,?)' + ).run(filePath, id) + } + for (let row = 0; row < rows; row++) { + const messageId = Number( + db + .prepare('INSERT INTO messages(session_row_id,role,ts) VALUES (?,?,?)') + .run(id, role, updatedAt).lastInsertRowid + ) + const user = role === 'user' ? text : '' + const assistant = role === 'assistant' ? text : '' + const tool = role === 'tool' ? `${text} ${toolText}`.trim() : toolText + db.prepare( + 'INSERT INTO messages_fts(rowid,user_text,assistant_text,tool_text,identifiers) VALUES (?,?,?,?,?)' + ).run(messageId, user, assistant, tool, identifierShadowText(`${text} ${toolText}`)) + } +} + +export function markFork(db: SyncDatabase, ids: readonly number[], hash: string): void { + for (const id of ids) { + db.prepare('UPDATE sessions SET content_hash = ?, content_hash_count = 8 WHERE id = ?').run( + hash, + id + ) + } +} diff --git a/src/main/ai-vault-search/session-search-engine-types.ts b/src/main/ai-vault-search/session-search-engine-types.ts new file mode 100644 index 00000000000..861130be59b --- /dev/null +++ b/src/main/ai-vault-search/session-search-engine-types.ts @@ -0,0 +1,157 @@ +import type { AiVaultAgent } from '../../shared/ai-vault-types' +import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers' +import type { SessionSearchUnavailableFeature } from './session-search-query-schema' + +// ENGINE types, deliberately not in src/shared: nothing here is a wire type. +// PR 5 owns the public contract and lifts what a caller may actually receive; +// until then a field can be added, renamed or dropped without a compat story. + +export const SESSION_SEARCH_LIMIT_DEFAULT = 20 +export const SESSION_SEARCH_LIMIT_MAX = 100 +// Longer than this is not a query, and FTS5 pays for every term it plans. +export const SESSION_SEARCH_QUERY_MAX_LENGTH = 512 + +// Snippet match markers. Why doubled: single brackets are everywhere in code +// transcripts (`arr[0]`, regex classes, markdown links) and would read as +// matches; doubled ones are rare. +export const SESSION_SEARCH_SNIPPET_MARK_OPEN = '[[' +export const SESSION_SEARCH_SNIPPET_MARK_CLOSE = ']]' + +/** + * Which corpus answers the query. + * + * - `conversation`: user and assistant turns only, as a column filter over + * `messages_fts` (see `scopedExpression`). + * - `all`: those turns plus tool calls and tool output, and the identifier + * shadow column, from `messages_fts`. + * + * The engine searches exactly the scope it is given. Switching corpus as the + * user types is a UI policy and lives in the panel (PR 7); an engine that + * second-guessed the scope would make a result impossible to reproduce from + * its own request. + */ +export type SessionSearchScope = 'conversation' | 'all' + +export type SessionSearchSort = 'relevance' | 'newest' + +export type SessionSearchFilters = { + agents?: readonly AiVaultAgent[] + /** Only sessions whose cwd is that path or inside it. */ + scopePaths?: readonly string[] + /** ISO timestamp; only sessions updated at or after it. */ + since?: string + sort?: SessionSearchSort +} + +export type SessionSearchRequest = { + query: string + /** Default `all`. */ + scope?: SessionSearchScope + limit?: number + /** From a previous response's `page.cursor`; only valid in its own generation. */ + cursor?: string + filters?: SessionSearchFilters +} + +export type SessionSearchRoute = 'phrase' | 'and' | 'or' | 'typo+phrase' | 'typo+and' | 'typo+or' + +/** + * How the query was executed. Diagnostics, not an answer: PR 5 decides which of + * these a caller ever sees (the reviewer's F5/F7 want them behind `debug`). + */ +export type SessionSearchPlannerReport = { + route: SessionSearchRoute + /** + * The whole body the repaired plan searched, in query order, when any term + * was changed. Not just the corrected terms: a caller rendering "searched + * for" needs the query it actually ran, and a repair never drops a term the + * original kept. A corrected term carries the index's own spelling, which the + * tokenizer has case-folded; untouched terms keep the case they were typed in. + */ + repairedTerms?: string[] + /** The corpus the route ran against; today always the requested scope. */ + tier: SessionSearchScope +} + +/** + * Where a source stands according to the index's own `files` table. The query + * path never stats a transcript, so it can report that the index has a live + * file record for a session or that it has none, and never that a source is + * gone: only a proven deletion may claim `missing`, and proving one is the + * indexer's job (docs/reference/ssh-execution-boundary.md). + */ +export type SessionSearchSourcePresence = 'present' | 'unverifiable' + +export type SessionSearchEvidence = { + role: TranscriptMessageRole + timestamp: string | null + /** FTS5 snippet with the matched terms wrapped in `[[` `]]`. */ + snippet: string + /** The snippet hit the engine's per-hit ceiling and was cut. */ + snippetTruncated?: boolean +} + +export type SessionSearchHit = { + agent: AiVaultAgent + sessionId: string + filePath: string + codexHome: string | null + title: string + cwd: string | null + branch: string | null + updatedAt: string | null + messageCount: number + resumeCommand: string + score: number + /** Sessions folded into this hit (forks sharing an opening prefix); absent when unique. */ + duplicateCount?: number + source: SessionSearchSourcePresence + /** Null when the operators alone put this session on the page, with no text match. */ + evidence: SessionSearchEvidence | null +} + +export type SessionSearchPage = { + /** Null when this page is the last one. */ + cursor: string | null + hasMore: boolean +} + +export type SessionSearchTruncation = { + /** + * Ranking saw only the first `sessionCandidateLimit` sessions, so a session + * past that cut cannot appear on any page of this query. + */ + candidates: boolean + /** Hits on this page whose snippet was cut. */ + snippets: number + /** + * The query itself was cut before it was searched: past the length ceiling, + * or past the number of terms the planner will plan. The terms that survived + * were searched in full, so a hit is still a hit; a miss is not proof of + * absence. + */ + query: boolean +} + +export type SessionSearchResponse = { + hits: SessionSearchHit[] + /** + * Engine features the index on disk cannot serve, empty on a current index. + * A route ladder missing its repair rung still answers; saying so is what + * keeps the answer honest. + */ + unavailable: readonly SessionSearchUnavailableFeature[] + planner: SessionSearchPlannerReport + page: SessionSearchPage + truncated: SessionSearchTruncation + /** The index snapshot these hits came from; a cursor is only valid within it. */ + generation: number + durationMs: number +} + +export function resolveSessionSearchLimit(limit: number | undefined): number { + // Why clamped here and not at the caller: a non-positive limit becomes + // `slice(0, -1)`, which silently drops the last hit of every page. + const requested = Number.isInteger(limit) ? (limit as number) : SESSION_SEARCH_LIMIT_DEFAULT + return Math.min(Math.max(1, requested), SESSION_SEARCH_LIMIT_MAX) +} diff --git a/src/main/ai-vault-search/session-search-engine.test.ts b/src/main/ai-vault-search/session-search-engine.test.ts new file mode 100644 index 00000000000..140f9449c3d --- /dev/null +++ b/src/main/ai-vault-search/session-search-engine.test.ts @@ -0,0 +1,471 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { SESSION_SEARCH_QUERY_MAX_LENGTH } from './session-search-engine-types' +import type { SessionSearchRequest, SessionSearchResponse } from './session-search-engine-types' +import { planSessionSearchQuery } from './session-search-query-planner' +import { ensureSessionSearchQuerySchema } from './session-search-query-schema' +import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet' +import { + addSyntheticSession, + markFork, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +async function open(name: string, options = {}): Promise { + harness = await openSessionSearchHarness(name, options) + return harness +} + +function ids(result: SessionSearchResponse): string[] { + return result.hits.map((hit) => hit.sessionId) +} + +describe('the route ladder tries phrase, then AND, then repair, then OR', () => { + async function routeFor( + text: string, + request: SessionSearchRequest + ): Promise { + const { db, engine } = await open('ss-engine-route') + addSyntheticSession(db, { id: 1, text }) + return engine.search(request) + } + + it('takes the phrase route when the tokens are adjacent and in order', async () => { + const result = await routeFor('the alpha beta gamma line', { query: '"alpha beta"' }) + expect(result.planner.route).toBe('phrase') + expect(ids(result)).toEqual(['1']) + }) + + it('falls to AND when the tokens are present but not adjacent', async () => { + const result = await routeFor('beta separated alpha', { query: '"alpha beta"' }) + expect(result.planner.route).toBe('and') + expect(ids(result)).toEqual(['1']) + }) + + it('falls to OR for prose, where no phrase was ever claimed', async () => { + const result = await routeFor('the relay dropped a frame', { query: 'relay frames dropped' }) + expect(result.planner.route).toBe('or') + expect(ids(result)).toEqual(['1']) + }) + + it('repairs a typo before the OR fallback, and says which terms it changed', async () => { + const { db, engine } = await open('ss-engine-typo') + // Two copies: the repair only suggests a term the index really holds. + addSyntheticSession(db, { id: 1, text: 'the coalesces path is slow' }) + addSyntheticSession(db, { id: 2, text: 'coalesces again here' }) + const result = engine.search({ query: 'coalescs' }) + expect(result.planner.route).toBe('typo+or') + expect(result.planner.repairedTerms).toEqual(['coalesces']) + expect(ids(result).sort()).toEqual(['1', '2']) + }) + + it('keeps every term a repaired literal was typed with', async () => { + const { db, engine } = await open('ss-engine-typo-literal') + addSyntheticSession(db, { id: 1, text: 'parseJson the data' }) + addSyntheticSession(db, { id: 2, text: 'parseJson the data again' }) + // `parseJsonn(the, data)` is literal because of its punctuation; the + // corrected spelling read on its own is prose. Re-planning without carrying + // the original decision across would drop `the` and report a body that was + // never typed. + // A corrected term comes back in the index's own spelling, which unicode61 + // has folded; the terms the repair left alone keep the case they were typed. + const result = engine.search({ query: 'parseJsonn(the, data)' }) + expect(result.planner.repairedTerms).toEqual(['parsejson', 'the', 'data']) + }) + + it('does not repair a term the index already holds', async () => { + const { db, engine } = await open('ss-engine-no-typo') + addSyntheticSession(db, { id: 1, text: 'coalesces' }) + const result = engine.search({ query: 'coalesces' }) + expect(result.planner.repairedTerms).toBeUndefined() + expect(result.planner.route).toBe('or') + }) + + it('reports the scope it searched as the planner tier', async () => { + const { db, engine } = await open('ss-engine-tier') + addSyntheticSession(db, { id: 1, text: 'needle' }) + expect(engine.search({ query: 'needle' }).planner.tier).toBe('all') + expect(engine.search({ query: 'needle', scope: 'conversation' }).planner.tier).toBe( + 'conversation' + ) + }) +}) + +describe('scope picks the corpus and never switches it', () => { + async function corpus(): Promise { + const opened = await open('ss-engine-scope') + addSyntheticSession(opened.db, { id: 1, text: 'harbor pilot manifest', role: 'user' }) + addSyntheticSession(opened.db, { id: 2, text: 'harbor tool output line', role: 'tool' }) + return opened + } + + it('searches conversation turns only under `conversation`', async () => { + const { engine } = await corpus() + expect(ids(engine.search({ query: 'harbor', scope: 'conversation' }))).toEqual(['1']) + }) + + it('includes tool output under `all`, which is the default', async () => { + const { engine } = await corpus() + expect(ids(engine.search({ query: 'harbor', scope: 'all' })).sort()).toEqual(['1', '2']) + expect(ids(engine.search({ query: 'harbor' })).sort()).toEqual(['1', '2']) + }) + + it('returns nothing rather than widening when the narrow scope misses', async () => { + // The panel's two-tier typing is a UI policy (PR 7). An engine that widened + // here would make a result impossible to reproduce from its own request. + const { engine } = await corpus() + const result = engine.search({ query: 'output', scope: 'conversation' }) + expect(result.hits).toEqual([]) + expect(result.planner.tier).toBe('conversation') + }) + + it('matches an identifier through its pieces only in the full corpus', async () => { + const { db, engine } = await open('ss-engine-identifiers') + addSyntheticSession(db, { id: 1, text: 'resolveTerminalPath' }) + // The identifier shadow column lives in messages_fts alone. + expect(ids(engine.search({ query: 'terminal path' }))).toEqual(['1']) + expect(engine.search({ query: 'terminal path', scope: 'conversation' }).hits).toEqual([]) + }) +}) + +describe('the conversation scope is a column filter, and it binds the whole query', () => { + it('refuses an AND whose second term lives only in tool output', async () => { + // The filter binds to the expression it prefixes. `{cols}: (a AND b)` + // filters both terms; `{cols}: a AND b` filters only `a` and searches tool + // output for the rest, which is a conversation search answering from a + // column it promised not to read. + const { db, engine } = await open('ss-engine-scope-binding') + addSyntheticSession(db, { id: 1, text: 'alpha gamma beta' }) + addSyntheticSession(db, { id: 2, text: 'alpha gamma', toolText: 'beta' }) + // Quoted, so the query is literal; not adjacent, so the phrase rung misses + // and the AND rung is the one that answers. + const query = '"alpha" beta' + + const wide = engine.search({ query, scope: 'all' }) + expect(wide.planner.route).toBe('and') + expect(ids(wide).sort()).toEqual(['1', '2']) + + const narrowed = engine.search({ query, scope: 'conversation' }) + expect(narrowed.planner.route).toBe('and') + expect(ids(narrowed)).toEqual(['1']) + }) + + it('ranks a conversation hit down for tool output it will not show', async () => { + // The one behavioural difference the column filter carries, pinned rather + // than wished away. FTS5's bm25 normalises by the whole row's length and + // has no per-column length, so two rows with identical prose do not score + // identically when one of them also holds tool output. A dedicated + // two-column table scored them the same. The rowid set is unchanged, which + // is what the decision was measured on; the order within it can move. + const { db, engine } = await open('ss-engine-scope-weights') + addSyntheticSession(db, { id: 1, text: 'harbor pilot' }) + addSyntheticSession(db, { id: 2, text: 'harbor pilot', toolText: 'unrelated '.repeat(40) }) + const narrowed = engine.search({ query: 'harbor', scope: 'conversation' }) + expect(ids(narrowed)).toEqual(['1', '2']) + expect(narrowed.hits[0]!.score).toBeGreaterThan(narrowed.hits[1]!.score) + }) + + it('never snippets a conversation hit out of tool output', async () => { + const { db, engine } = await open('ss-engine-scope-snippet') + addSyntheticSession(db, { id: 1, text: 'harbor pilot', toolText: 'harbor tool output line' }) + const [hit] = engine.search({ query: 'harbor', scope: 'conversation' }).hits + expect(hit?.evidence?.snippet).toContain('pilot') + expect(hit?.evidence?.snippet).not.toContain('output') + // And asked for a tool-only row directly, it has nothing to show. + addSyntheticSession(db, { id: 2, text: 'harbor tool output line', role: 'tool' }) + const rowid = Number( + (db.prepare('SELECT max(id) AS id FROM messages').get() as { id: number }).id + ) + const plan = planSessionSearchQuery('harbor') + expect(sessionSearchSnippet(db, 'conversation', rowid, plan)).toEqual(EMPTY_SNIPPET) + expect(sessionSearchSnippet(db, 'all', rowid, plan).text).toContain('output') + }) +}) + +describe('a session is one hit, however many of its rows matched', () => { + it.each(['relevance', 'newest'] as const)( + 'keeps a short session on the %s page beside a 650-row session', + async (sort) => { + const { db, engine } = await open('ss-engine-aggregate', { sessionCandidateLimit: 600 }) + addSyntheticSession(db, { id: 1, rows: 650, updatedAt: '2026-09-06T00:00:00.000Z' }) + addSyntheticSession(db, { + id: 2, + text: 'needle padding', + updatedAt: '2026-09-05T00:00:00.000Z' + }) + // Collapsing to one row per session happens before the candidate limit, + // so the 650-row session cannot crowd the one-row session off the page on + // either order; which of them ranks first is the sort's business. + expect(ids(engine.search({ query: 'needle', filters: { sort } })).sort()).toEqual(['1', '2']) + } + ) + + it('folds forks the same way for an operator-only page as for a text page', async () => { + const { db, engine } = await open('ss-engine-forks') + for (const id of [1, 2, 3, 4]) { + addSyntheticSession(db, { id, updatedAt: `2026-09-0${id}T00:00:00.000Z` }) + } + markFork(db, [1, 2, 3, 4], 'shared-fork-prefix') + const operatorOnly = engine.search({ query: 'repo:app' }) + const withText = engine.search({ query: 'needle repo:app' }) + expect(ids(operatorOnly)).toEqual(['4']) + expect(operatorOnly.hits[0]?.duplicateCount).toBe(4) + expect(ids(withText)).toEqual(ids(operatorOnly)) + expect(withText.hits[0]?.duplicateCount).toBe(4) + }) + + it('answers an operator-only query with the newest sessions and no evidence', async () => { + const { db, engine } = await open('ss-engine-operator-only') + addSyntheticSession(db, { id: 1, updatedAt: '2026-09-01T00:00:00.000Z' }) + addSyntheticSession(db, { id: 2, updatedAt: '2026-09-09T00:00:00.000Z' }) + const result = engine.search({ query: 'repo:app' }) + expect(ids(result)).toEqual(['2', '1']) + expect(result.hits[0]?.evidence).toBeNull() + }) + + it('has no hits for a query with neither text nor operators', async () => { + const { db, engine } = await open('ss-engine-empty') + addSyntheticSession(db, { id: 1 }) + expect(engine.search({ query: ' ' }).hits).toEqual([]) + }) +}) + +describe('filters narrow retrieval, not just the page', () => { + it('finds a scoped match behind 600 out-of-scope rows', async () => { + const { db, engine } = await open('ss-engine-scoped') + addSyntheticSession(db, { id: 1, cwd: '/unrelated', rows: 600 }) + addSyntheticSession(db, { id: 2, cwd: '/target', text: 'needle padding' }) + expect(ids(engine.search({ query: 'needle', filters: { scopePaths: ['/target'] } }))).toEqual([ + '2' + ]) + }) + + it('falls back to a later rung when the exact hit is out of scope', async () => { + const { db, engine } = await open('ss-engine-scoped-route') + addSyntheticSession(db, { id: 1, cwd: '/unrelated', text: 'resolveTerminalPath' }) + addSyntheticSession(db, { id: 2, cwd: '/target', text: 'resolve terminal path' }) + expect( + ids(engine.search({ query: 'resolveTerminalPath', filters: { scopePaths: ['/target'] } })) + ).toEqual(['2']) + }) +}) + +describe('evidence', () => { + it('takes each snippet from that hit’s own best message', async () => { + const { db, engine } = await open('ss-engine-snippet') + // Written first, so its row owns the lowest rowid: the row a dropped rowid + // constraint would hand back for every hit. + addSyntheticSession(db, { + id: 1, + text: 'hydration marmoset appears once in a long paragraph about routing and caching', + updatedAt: '2026-09-01T00:00:00.000Z' + }) + addSyntheticSession(db, { + id: 2, + text: 'hydration capybara', + updatedAt: '2026-09-09T00:00:00.000Z' + }) + const hits = engine.search({ query: 'hydration' }).hits + expect(hits[0]?.evidence?.snippet).toContain('capybara') + expect(hits[0]?.evidence?.snippet).not.toContain('marmoset') + expect(hits.find((hit) => hit.sessionId === '1')?.evidence?.snippet).toContain('marmoset') + }) + + it('shows the prose column rather than the identifier shadow when both match', async () => { + const { db, engine } = await open('ss-engine-snippet-shadow') + addSyntheticSession(db, { + id: 1, + text: 'resolveTerminalPath is broken and the terminal never comes up for a pane, which is odd because every other pane on this host resolves its path' + }) + const snippet = engine.search({ query: 'terminal path' }).hits[0]?.evidence?.snippet ?? '' + expect(snippet).toContain('[[') + expect(snippet).not.toContain('resolve [[terminal]] [[path]]') + }) + + it('flags a snippet it had to cut, and counts it on the result', async () => { + const { db, engine } = await open('ss-engine-snippet-truncated') + // The window is twelve tokens wide, and one of them is 4000 characters, so + // the token count is no bound at all on what a hit carries. + addSyntheticSession(db, { id: 1, text: `needle ${'x'.repeat(4000)}` }) + const result = engine.search({ query: 'needle' }) + expect(result.hits[0]?.evidence?.snippetTruncated).toBe(true) + expect(result.hits[0]?.evidence?.snippet.length).toBeLessThan(600) + expect(result.truncated.snippets).toBe(1) + }) + + it('leaves an ordinary snippet unflagged', async () => { + const { db, engine } = await open('ss-engine-snippet-whole') + addSyntheticSession(db, { id: 1, text: 'needle in a short line' }) + const result = engine.search({ query: 'needle' }) + expect(result.hits[0]?.evidence?.snippetTruncated).toBeUndefined() + expect(result.truncated.snippets).toBe(0) + }) +}) + +describe('source presence comes from the files table, never a stat', () => { + it('calls a session with a live file record present', async () => { + const { db, engine } = await open('ss-engine-presence') + addSyntheticSession(db, { id: 1 }) + expect(engine.search({ query: 'needle' }).hits[0]?.source).toBe('present') + }) + + it('calls a session with no file record unverifiable, and still returns it', async () => { + // Loss of contact is never evidence of absence: the hit stays on the page. + const { db, engine } = await open('ss-engine-presence-unknown') + addSyntheticSession(db, { id: 1, filePath: null }) + const hits = engine.search({ query: 'needle' }).hits + expect(hits).toHaveLength(1) + expect(hits[0]?.source).toBe('unverifiable') + }) +}) + +describe('the engine carries its own schema and puts it back', () => { + it('installs the vocabulary and the log over an index a writer built alone', async () => { + // The store creates none of these: PR 3's indexer can fill a whole index + // before anything opens an engine over it. + const { db, engine } = await open('ss-engine-installs') + addSyntheticSession(db, { id: 1, text: 'the coalesces path is slow' }) + addSyntheticSession(db, { id: 2, text: 'coalesces again here' }) + const result = engine.search({ query: 'coalescs' }) + expect(result.unavailable).toEqual([]) + expect(result.planner.route).toBe('typo+or') + expect(ids(result).sort()).toEqual(['1', '2']) + }) + + it('re-creates a vocabulary that vanished under a live engine', async () => { + const { db, engine } = await open('ss-engine-vocab-vanishes') + addSyntheticSession(db, { id: 1, text: 'coalesces here now' }) + addSyntheticSession(db, { id: 2, text: 'coalesces again here' }) + expect(engine.search({ query: 'coalescs' }).planner.route).toBe('typo+or') + + db.exec('DROP TABLE messages_vocab') + const after = engine.search({ query: 'coalescs' }) + expect(after.unavailable).toEqual([]) + expect(after.planner.route).toBe('typo+or') + }) + + it('names the feature it cannot serve when the vocabulary has no source left', async () => { + // What an index being rebuilt by another handle looks like from here. The + // vocabulary can be created over a missing `messages_fts` and every query + // against it then fails, so the probe reads the source, not the view. + // + // With one FTS table there is no scope left to answer from, so this is now + // the boundary of the degrade: the engine names the feature and the search + // fails loudly on the table it cannot read, rather than returning an empty + // page that looks like an answer. + const { db, engine } = await open('ss-engine-vocab-source-gone') + addSyntheticSession(db, { id: 1, text: 'coalesces here now', role: 'user' }) + db.exec('DROP TABLE messages_vocab; DROP TABLE messages_fts') + + expect(ensureSessionSearchQuerySchema(db)).toEqual(['typo-repair']) + for (const scope of ['all', 'conversation'] as const) { + expect(() => engine.search({ query: 'coalesces', scope })).toThrow(/no such (fts5 )?table/i) + } + }) + + it('picks the feature back up when the source comes back', async () => { + const { db, engine } = await open('ss-engine-vocab-returns') + addSyntheticSession(db, { id: 1, text: 'coalesces here now' }) + addSyntheticSession(db, { id: 2, text: 'coalesces again here' }) + const fts = ( + db.prepare("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").get() as { + sql: string + } + ).sql + db.exec('DROP TABLE messages_vocab; DROP TABLE messages_fts') + expect(ensureSessionSearchQuerySchema(db)).toEqual(['typo-repair']) + + db.exec(fts) + // Two, because the vocabulary only offers a term at least two rows carry. + addSyntheticSession(db, { id: 3, text: 'coalesces one more time' }) + addSyntheticSession(db, { id: 4, text: 'coalesces once again' }) + // Nothing throws on the way back up, so the recovery cannot come from the + // error path; it comes from the probe running per search. + const restored = engine.search({ query: 'coalescs' }) + expect(restored.unavailable).toEqual([]) + expect(restored.planner.route).toBe('typo+or') + }) +}) + +describe('a query the engine had to cut says so', () => { + it('answers a query whose cap falls inside an astral character', async () => { + // The cut is on a whole code point rather than a code unit, so nothing + // downstream is handed half a surrogate pair. That is hygiene rather than a + // behaviour: the planner's tokenizer does not treat a lone surrogate as a + // token character, so it drops out of the terms either way. What this pins + // is that the boundary is answerable at all. + const { db, engine } = await open('ss-engine-surrogate-cap') + const kept = 'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH - 2) + addSyntheticSession(db, { id: 1, text: kept }) + const result = engine.search({ query: `${kept} 😀 tail` }) + expect(result.truncated.query).toBe(true) + expect(result.hits.map((hit) => hit.sessionId)).toEqual(['1']) + }) + + it('loads a candidate set larger than one batch of bound ids', async () => { + // The id list is as long as the candidate limit and every id is a bound + // parameter. No SQLite this stack can run refuses 1,100 of them, so this + // pins that batching returns the same answer, not that it rescues one. + const { db, engine } = await open('ss-engine-id-batching', { + sessionCandidateLimit: 1200 + }) + for (let id = 1; id <= 1100; id++) { + addSyntheticSession(db, { id, text: 'needle' }) + } + const result = engine.search({ query: 'needle', limit: 5 }) + expect(result.hits).toHaveLength(5) + expect(result.truncated.candidates).toBe(false) + }) + + it('reports truncation when the planner drops terms past its cap', async () => { + // The 56th term is the only one that matches. Without the flag this is a + // confident empty answer to a query the engine never finished reading. + const { db, engine } = await open('ss-engine-term-cap') + addSyntheticSession(db, { id: 1, text: 'onlyattheend' }) + const query = `${Array.from({ length: 55 }, (_unused, n) => `term${n}`).join(' ')} onlyattheend` + const result = engine.search({ query }) + expect(result.hits).toEqual([]) + expect(result.truncated.query).toBe(true) + }) + + it('reports truncation when the query is longer than the engine will plan', async () => { + const { db, engine } = await open('ss-engine-length-cap') + addSyntheticSession(db, { id: 1, text: 'needle' }) + const result = engine.search({ query: `needle ${'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH)}` }) + expect(result.truncated.query).toBe(true) + }) + + it('claims no truncation for a query that fit', async () => { + const { db, engine } = await open('ss-engine-no-cap') + addSyntheticSession(db, { id: 1, text: 'needle' }) + expect(engine.search({ query: 'needle' }).truncated.query).toBe(false) + }) +}) + +describe('a query longer than the engine will plan is cut, not refused', () => { + it('cuts one enormous token down to the cap before FTS5 ever sees it', async () => { + const { db, engine } = await open('ss-engine-long-query') + // The planner already caps how many terms it will plan, so a long query of + // ordinary words is bounded without this. What is not bounded is a single + // token: one 100 kB word is one term, and FTS5 would carry the whole thing + // into the MATCH expression. The cut is observable because the indexed + // token is exactly the capped length. + addSyntheticSession(db, { id: 1, text: 'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH) }) + expect(ids(engine.search({ query: 'x'.repeat(4000) }))).toEqual(['1']) + }) +}) + +describe('unicode terms survive the round trip', () => { + it.each(['café', 'C', 'R', 'x', '修復', '안녕하세요'])('searches %s', async (text) => { + const { db, engine } = await open('ss-engine-unicode') + addSyntheticSession(db, { id: 1, text }) + expect(engine.search({ query: text }).hits).toHaveLength(1) + }) +}) diff --git a/src/main/ai-vault-search/session-search-engine.ts b/src/main/ai-vault-search/session-search-engine.ts new file mode 100644 index 00000000000..4b6e11e407d --- /dev/null +++ b/src/main/ai-vault-search/session-search-engine.ts @@ -0,0 +1,349 @@ +import type SyncDatabase from '../sqlite/sync-database' +import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers' +import { sliceAtCodeUnitLimit } from '../ai-vault/session-scanner-text-normalization' +import { + hasAiVaultSearchQueryOperators, + splitAiVaultSearchQuery, + type AiVaultSearchQuerySplit +} from '../../shared/ai-vault-search-query-operators' +import { matchesAiVaultQueryOperators } from '../../shared/ai-vault-session-filters' +import { + resolveSessionSearchLimit, + SESSION_SEARCH_QUERY_MAX_LENGTH, + type SessionSearchHit, + type SessionSearchRequest, + type SessionSearchResponse, + type SessionSearchScope, + type SessionSearchSourcePresence +} from './session-search-engine-types' +import { readIndexGeneration } from './session-search-index-generation' +import { + rankSessionHits, + type MessageRow, + type RankedSession, + type SessionRow +} from './session-search-hit-ranking' +import { + decodeSessionSearchCursor, + encodeSessionSearchCursor, + sessionSearchPageKey +} from './session-search-page-cursor' +import { planSessionSearchQuery } from './session-search-query-planner' +import { logSessionSearchQuery } from './session-search-query-log' +import { + SessionSearchRetrieval, + type RetrievalScope, + type Retrieved +} from './session-search-retrieval' +import { sessionRowFilter } from './session-search-row-filter' +import { + ensureSessionSearchQuerySchema, + type SessionSearchUnavailableFeature +} from './session-search-query-schema' +import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet' +import { sessionSourcePresence } from './session-search-source-presence' + +/** + * Sessions retrieved before ranking cuts the page. + * + * Not a fixed constant (the reviewer's F13): it is the knob that trades page + * completeness for retrieval cost, and the right value depends on index size. + * Measurements behind this default, and what changing it costs, are in + * docs/reference/agent-session-search-query-tuning.md. + */ +export const SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT = 600 + +/** One ranked list plus what produced it; a page is a slice of `ranked`. */ +type RankedPage = { + ranked: RankedSession[] + /** Null when no text was searched, so there is nothing to snippet from. */ + retrieved: Retrieved | null + /** + * Retrieval may have missed a session: a cap ended it, not the data. True + * whether the candidate limit filled or the operator walk gave up scanning. + */ + incomplete: boolean +} + +export type SessionSearchEngineOptions = { + sessionCandidateLimit?: number + /** Oldest transcript mtime a hit may come from; PR 3 derives it from retention. */ + retentionCutoffMs?: number | null + /** Write each query to `search_log`. Off unless a caller asks (see query-log). */ + logQueries?: boolean +} + +/** + * Ranked session search over the PR 2 index. + * + * A library: it holds no timers, reads no settings, and knows nothing about + * Electron, IPC or a panel. It is handed a connection rather than opening one, + * because which process may open, rebuild or unlink the index file is PR 3b's + * decision and not a query engine's. + * + * **Every read here is a single statement, and no read transaction is ever + * open across an `await`.** There is no `BEGIN` on this path, no `.iterate()` + * outliving its statement, and `search` is synchronous end to end. That is a + * constraint PR 2 measured rather than a style: a reader that pins a WAL + * snapshot holds off every checkpoint behind it, and the same 47 MB of writes + * that leave a 9.9 MB WAL grew to 266 MB with one `BEGIN` + `SELECT` held open. + * + * One search is one synchronous pass, and every page of it is a slice of the + * same ranked list. That list is rebuilt per page rather than streamed, which + * is what makes a page repeatable: within one index generation the same request + * ranks the same way, and a cursor from any other generation is refused. + * + * That fence is strict on purpose, and the cost is worth stating plainly: any + * committed read moves the generation, so while a backfill is running an + * outstanding cursor will be refused, often within a second. Pagination is + * usable against a settled index and unreliable against one still filling. The + * rejection carries both generations, so a caller that sees `stale-generation` + * knows the index moved rather than that it holds a bad cursor, and can quietly + * re-issue page one instead of showing anyone an error. + */ +export class SessionSearchEngine { + private retrieval: SessionSearchRetrieval + private readonly candidateLimit: number + /** Re-probed whenever a query proves it stale; see `withCapabilityRetry`. */ + private unavailable: readonly SessionSearchUnavailableFeature[] + + constructor( + private readonly db: SyncDatabase, + private readonly options: SessionSearchEngineOptions = {} + ) { + this.candidateLimit = options.sessionCandidateLimit ?? SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT + // Installed here and not on the first search, so the generation triggers are + // watching before anything this engine will be asked to page over is + // written, and so retrieval below prepares against tables that exist. + this.unavailable = ensureSessionSearchQuerySchema(this.db) + this.retrieval = new SessionSearchRetrieval(this.db, !this.unavailable.includes('typo-repair')) + } + + search(request: SessionSearchRequest): SessionSearchResponse { + const startedAt = performance.now() + this.probeCapabilities() + const generation = readIndexGeneration(this.db) + const scope = request.scope ?? 'all' + const sort = request.filters?.sort ?? 'relevance' + // Not a bare `slice`: cutting between a surrogate pair leaves a lone half + // that no tokenizer can match and that a caller cannot echo back. + const capped = sliceAtCodeUnitLimit(request.query, SESSION_SEARCH_QUERY_MAX_LENGTH) + const split = splitAiVaultSearchQuery(capped) + const retrievalScope: RetrievalScope = { + scope, + sort, + filter: sessionRowFilter(request.filters ?? {}, this.options.retentionCutoffMs ?? null), + matchesOperators: operatorPredicate(split), + candidateLimit: this.candidateLimit + } + // Decoded before any retrieval: a cursor the engine will refuse must not + // cost a query, and the caller has to hear about it either way. + const pageKey = sessionSearchPageKey(request) + const offset = request.cursor + ? decodeSessionSearchCursor(request.cursor, generation, pageKey) + : 0 + + const plan = planSessionSearchQuery(split.text) + const { ranked, retrieved, incomplete } = this.withCapabilityRetry(() => + plan.terms.length === 0 + ? this.operatorOnly(split, retrievalScope) + : this.text(plan, retrievalScope, sort) + ) + + const limit = resolveSessionSearchLimit(request.limit) + const page = ranked.slice(offset, offset + limit) + const hits = this.hits(page, scope, retrieved) + const hasMore = ranked.length > offset + limit + const response: SessionSearchResponse = { + hits, + unavailable: this.unavailable, + planner: { + route: retrieved?.route ?? 'or', + tier: scope, + ...(retrieved?.repairedTerms ? { repairedTerms: retrieved.repairedTerms } : {}) + }, + page: { + hasMore, + cursor: hasMore ? encodeSessionSearchCursor(generation, offset + limit, pageKey) : null + }, + truncated: { + // Decided by retrieval, which is the only layer that knows whether a cap + // ended it. Deriving it from the hits cannot work: an operator walk that + // gave up at its scan ceiling returns no hits, and so does a search that + // genuinely matched nothing. + candidates: incomplete, + snippets: hits.filter((hit) => hit.evidence?.snippetTruncated).length, + query: capped.length < request.query.length || plan.truncated + }, + generation, + durationMs: performance.now() - startedAt + } + if (this.options.logQueries) { + logSessionSearchQuery(this.db, { + query: request.query, + route: response.planner.route, + hits: hits.length, + durationMs: response.durationMs + }) + } + return response + } + + /** + * Where the engine's own schema is created and checked, once per search. + * + * A capability is a fact about the file, not about this object: another handle + * can rebuild the index under a live connection, so a verdict cached in the + * constructor is wrong for the rest of the engine's life in both directions — + * it would keep reaching for a table that went away, and never pick one back + * up when it returned. Retrieval is only rebuilt when the answer changes, so + * the steady-state cost is one indexed lookup and nothing else. + */ + private probeCapabilities(): void { + const unavailable = ensureSessionSearchQuerySchema(this.db) + if (unavailable.join() === this.unavailable.join()) { + return + } + this.unavailable = unavailable + this.retrieval = new SessionSearchRetrieval(this.db, !unavailable.includes('typo-repair')) + } + + /** + * Runs a retrieval, and re-probes once if it turns out the index no longer + * has what an earlier probe found. + * + * `probeCapabilities` already runs per search, so this only covers the window + * between that probe and the statement that reaches for the table. Losing a + * table there is a thrown error rather than a wrong verdict, so it re-probes + * and runs the search again. + */ + private withCapabilityRetry(run: () => RankedPage): RankedPage { + try { + return run() + } catch (error) { + if (!isMissingTableError(error)) { + throw error + } + this.probeCapabilities() + return run() + } + } + + /** + * Operators with no free text still name a scope, so the answer is the newest + * sessions inside it. Ranked through the same path as a text query, because + * forks must fold here exactly as they do there or the same sessions answer + * `repo:x` and `word repo:x` differently. There is no relevance signal + * without text, so the order is always newest. + */ + private operatorOnly(split: AiVaultSearchQuerySplit, scope: RetrievalScope): RankedPage { + if (!hasAiVaultSearchQueryOperators(split)) { + return { ranked: [], retrieved: null, incomplete: false } + } + const { sessions, incomplete } = this.retrieval.recent(scope) + return { ranked: rankSessionHits(sessions, new Map(), 'newest'), retrieved: null, incomplete } + } + + private text( + plan: ReturnType, + scope: RetrievalScope, + sort: 'relevance' | 'newest' + ): RankedPage { + const retrieved = this.retrieval.run(plan, scope) + // `match` already grouped to one best row per session. + const best = new Map(retrieved.rows.map((row) => [row.session_row_id, row])) + // Operators cut here, after retrieval, so the candidate count still reports + // what the SQL limit saw: that is what tells a caller the limit was binding. + const sessions = this.retrieval.loadSessions([...best.keys()], scope) + // Counted before the operator predicate and before fork folding: the SQL + // LIMIT is what could have hidden a session, and it saw the unfiltered set. + return { + ranked: rankSessionHits(sessions, best, sort), + retrieved, + incomplete: best.size >= this.candidateLimit + } + } + + /** Snippets and source presence are paid for by the page, never by the list. */ + private hits( + page: readonly RankedSession[], + scope: SessionSearchScope, + retrieved: Retrieved | null + ): SessionSearchHit[] { + const presence = sessionSourcePresence( + this.db, + page.map((entry) => entry.session.id) + ) + return page.map((entry) => this.hit(entry, scope, retrieved, presence)) + } + + private hit( + entry: RankedSession, + scope: SessionSearchScope, + retrieved: Retrieved | null, + presence: ReadonlyMap + ): SessionSearchHit { + const { session, message } = entry + const snippet = + message && retrieved + ? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan) + : EMPTY_SNIPPET + return { + ...sessionFields(session), + score: entry.score, + ...(entry.duplicateCount > 1 ? { duplicateCount: entry.duplicateCount } : {}), + source: presence.get(session.id) ?? 'unverifiable', + evidence: message + ? { + role: message.role as TranscriptMessageRole, + timestamp: message.ts, + snippet: snippet.text, + ...(snippet.truncated ? { snippetTruncated: true } : {}) + } + : null + } + } +} + +// SQLite reports a table that went away at the statement that reaches for it. +// `fts5` is in the message when the table is the vocabulary's target, which is +// the one an index rebuilt under a live connection loses first. +const MISSING_TABLE = /no such (fts5 )?table/i + +function isMissingTableError(error: unknown): boolean { + return error instanceof Error && MISSING_TABLE.test(error.message) +} + +/** + * The one reading of `repo:` / `path:`: the sessions panel's own predicate, over + * the columns the index stores. The engine has no project map, so a session's + * repo label falls back to its folder label, which is what the panel does for + * every session it cannot resolve a project for. + */ +function operatorPredicate(split: AiVaultSearchQuerySplit): (session: SessionRow) => boolean { + if (!hasAiVaultSearchQueryOperators(split)) { + return () => true + } + return (session) => + matchesAiVaultQueryOperators( + { cwd: session.cwd, filePath: session.file_path }, + { repoTerms: split.repoTerms, pathTerms: split.pathTerms } + ) +} + +function sessionFields( + session: SessionRow +): Omit { + return { + agent: session.agent, + sessionId: session.session_id, + filePath: session.file_path, + codexHome: session.codex_home, + title: session.title, + cwd: session.cwd, + branch: session.branch, + updatedAt: session.updated_at, + messageCount: session.message_count, + resumeCommand: session.resume_command + } +} diff --git a/src/main/ai-vault-search/session-search-fts5-contract.test.ts b/src/main/ai-vault-search/session-search-fts5-contract.test.ts new file mode 100644 index 00000000000..be815623ce7 --- /dev/null +++ b/src/main/ai-vault-search/session-search-fts5-contract.test.ts @@ -0,0 +1,172 @@ +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { removeTree } from '../../shared/windows-transient-lock-removal' +import type SyncDatabase from '../sqlite/sync-database' +import { indexTokens } from './session-search-query-planner' +import { ensureSessionSearchQuerySchema } from './session-search-query-schema' +import { openSessionSearchDatabase } from './session-search-schema' + +// SQLite/FTS5 behaviours the query layer depends on. Each one cost a live +// debugging session; a refactor that reintroduces the trap fails here. + +const FIRST_ROWID = 101 +const SECOND_ROWID = 202 + +let tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.map((root) => removeTree(root))) + tempRoots = [] +}) + +async function openDatabase(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-fts5-contract-')) + tempRoots.push(root) + return openSessionSearchDatabase(join(root, 'index.sqlite')) +} + +function insertMessageRow(db: SyncDatabase, rowid: number, text: string): void { + db.prepare( + `INSERT INTO messages_fts(rowid, user_text, assistant_text, tool_text, identifiers) + VALUES (?, ?, '', '', '')` + ).run(rowid, text) +} + +describe('FTS5 aux functions take the table name, never an alias', () => { + it('rejects bm25 over an aliased table and accepts the table-name form', async () => { + const db = await openDatabase() + insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one') + + expect(() => + db.prepare('SELECT bm25(f) AS score FROM messages_fts f WHERE f MATCH ?').all('alpha') + ).toThrow(/no such column: f/) + + const scored = db + .prepare('SELECT bm25(messages_fts) AS score FROM messages_fts WHERE messages_fts MATCH ?') + .all('alpha') as { score: number }[] + expect(scored).toHaveLength(1) + expect(Number.isFinite(scored[0]?.score)).toBe(true) + db.close() + }) + + it('rejects snippet over an aliased table too', async () => { + const db = await openDatabase() + insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one') + + expect(() => + db + .prepare( + "SELECT snippet(f, -1, '[', ']', '…', 12) AS s FROM messages_fts f WHERE f MATCH ?" + ) + .all('alpha') + ).toThrow(/no such column: f/) + db.close() + }) +}) + +describe('a rowid constraint beside MATCH is honoured only as a subselect', () => { + it('ignores `rowid = ?` and returns every match, first row first', async () => { + const db = await openDatabase() + insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one') + insertMessageRow(db, SECOND_ROWID, 'alpha capybara two') + + const rows = db + .prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?') + .all('alpha', SECOND_ROWID) as { rowid: number }[] + // The planner drops the constraint entirely: both rows come back. + expect(rows.map((row) => row.rowid)).toEqual([FIRST_ROWID, SECOND_ROWID]) + // A caller reading one row therefore gets the first match, not the one asked for. + const single = db + .prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?') + .get('alpha', SECOND_ROWID) as { rowid: number } | undefined + expect(single?.rowid).toBe(FIRST_ROWID) + db.close() + }) + + it('ignores `rowid IN (?)` the same way', async () => { + const db = await openDatabase() + insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one') + insertMessageRow(db, SECOND_ROWID, 'alpha capybara two') + + const rows = db + .prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid IN (?)') + .all('alpha', SECOND_ROWID) as { rowid: number }[] + expect(rows.map((row) => row.rowid)).toEqual([FIRST_ROWID, SECOND_ROWID]) + db.close() + }) + + it('honours `rowid IN (SELECT ?)` even with the session join on', async () => { + const db = await openDatabase() + db.prepare( + `INSERT INTO sessions(id,agent,session_id,file_path,title,resume_command) + VALUES (1,'claude','1','/synthetic/1','fixture','')` + ).run() + for (const rowid of [FIRST_ROWID, SECOND_ROWID]) { + db.prepare("INSERT INTO messages(id,session_row_id,role) VALUES (?,1,'user')").run(rowid) + } + insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one') + insertMessageRow(db, SECOND_ROWID, 'alpha capybara two') + + // The shape the snippet read uses: the joins are what subtract a row whose + // session a purge cut loose, and they must not cost the rowid constraint + // its effect. + const snippet = db + .prepare( + `SELECT snippet(messages_fts, -1, '[', ']', '…', 12) AS s + FROM messages_fts + JOIN messages m ON m.id = messages_fts.rowid + JOIN sessions s ON s.id = m.session_row_id + WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)` + ) + .get('alpha', SECOND_ROWID) as { s: string } | undefined + expect(snippet?.s).toContain('capybara') + expect(snippet?.s).not.toContain('marmoset') + db.close() + }) +}) + +describe('sessions.file_path is deliberately not unique', () => { + it('accepts two sessions sharing one store path', async () => { + const db = await openDatabase() + const insert = db.prepare( + `INSERT INTO sessions(agent, session_id, file_path, title, resume_command) + VALUES (?, ?, ?, ?, ?)` + ) + // OpenCode and Cursor keep every session in one SQLite store; files.path is the key. + const storePath = '/home/user/.local/share/opencode/storage.db' + insert.run('opencode', 'ses_one', storePath, 'first', 'opencode --session ses_one') + expect(() => + insert.run('opencode', 'ses_two', storePath, 'second', 'opencode --session ses_two') + ).not.toThrow() + + const rows = db + .prepare('SELECT session_id FROM sessions WHERE file_path = ? ORDER BY session_id') + .all(storePath) as { session_id: string }[] + expect(rows.map((row) => row.session_id)).toEqual(['ses_one', 'ses_two']) + db.close() + }) +}) + +describe('the planner tokenizer draws the same boundaries as unicode61', () => { + // unicode61 folds case and strips Latin diacritics on both index and query side. + function asIndexed(token: string): string { + return token.toLowerCase().normalize('NFD').replaceAll(/\p{M}/gu, '') + } + + it('produces exactly the terms fts5vocab reports for the same text', async () => { + const db = await openDatabase() + // The vocabulary is the engine's own object, not the store's. + ensureSessionSearchQuerySchema(db) + const corpus = + 'resolveTerminalPath src/main/foo-bar.ts a.b C++ #123 修复 café naïve MAX_TOKEN x' + insertMessageRow(db, FIRST_ROWID, corpus) + const indexed = ( + db.prepare('SELECT term FROM messages_vocab ORDER BY term').all() as { term: string }[] + ).map((row) => row.term) + + expect([...new Set(indexTokens(corpus).map(asIndexed))].sort()).toEqual(indexed) + db.close() + }) +}) diff --git a/src/main/ai-vault-search/session-search-hit-ranking.test.ts b/src/main/ai-vault-search/session-search-hit-ranking.test.ts new file mode 100644 index 00000000000..54919bace0f --- /dev/null +++ b/src/main/ai-vault-search/session-search-hit-ranking.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' +import { rankSessionHits, type MessageRow, type SessionRow } from './session-search-hit-ranking' + +function session(id: number, overrides: Partial = {}): SessionRow { + return { + id, + agent: 'claude', + session_id: String(id), + file_path: `/synthetic/${id}.jsonl`, + codex_home: null, + title: 'fixture', + cwd: '/repo/app', + branch: null, + updated_at: '2026-09-01T00:00:00.000Z', + message_count: 1, + resume_command: 'resume', + content_hash: null, + content_hash_count: 0, + ...overrides + } +} + +function match(id: number, score: number): MessageRow { + return { rowid: id, score, session_row_id: id, role: 'user', ts: null } +} + +function matches(...rows: MessageRow[]): Map { + return new Map(rows.map((row) => [row.session_row_id, row])) +} + +describe('order', () => { + it('ranks by score under relevance and by recency under newest', () => { + const sessions = [ + session(1, { updated_at: '2026-09-01T00:00:00.000Z' }), + session(2, { updated_at: '2026-09-09T00:00:00.000Z' }) + ] + const scores = matches(match(1, 10), match(2, 1)) + expect(rankSessionHits(sessions, scores, 'relevance').map((e) => e.session.id)).toEqual([1, 2]) + expect(rankSessionHits(sessions, scores, 'newest').map((e) => e.session.id)).toEqual([2, 1]) + }) + + it.each(['relevance', 'newest'] as const)( + 'breaks a %s tie by session, whatever order retrieval handed them over in', + (sort) => { + // A cursor is an offset into this list, so two entries that tie must not + // be free to swap between pages. Retrieval hands sessions over in + // whatever order the `IN (...)` lookup produced, which SQL does not + // promise, so the order below is deliberately reversed. + const sessions = [6, 5, 4, 3, 2, 1].map((id) => session(id)) + const scores = matches(...sessions.map((entry) => match(entry.id, 5))) + expect(rankSessionHits(sessions, scores, sort).map((entry) => entry.session.id)).toEqual([ + 1, 2, 3, 4, 5, 6 + ]) + } + ) + + it('prefers the shorter session when two match equally well', () => { + // The length prior: `0.02 · ln(1 + messages)`, subtracted per session. + const sessions = [session(1, { message_count: 5000 }), session(2, { message_count: 2 })] + const ranked = rankSessionHits(sessions, matches(match(1, 5), match(2, 5)), 'relevance') + expect(ranked.map((entry) => entry.session.id)).toEqual([2, 1]) + expect(ranked[0]!.score).toBeGreaterThan(ranked[1]!.score) + }) +}) + +describe('forks fold into one answer', () => { + const fork = (id: number, updatedAt: string): SessionRow => + session(id, { + updated_at: updatedAt, + content_hash: 'shared-opening-prefix', + content_hash_count: 8 + }) + + it('keeps the newest copy and counts the rest', () => { + const sessions = [ + fork(1, '2026-09-01T00:00:00.000Z'), + fork(2, '2026-09-09T00:00:00.000Z'), + fork(3, '2026-09-05T00:00:00.000Z') + ] + const ranked = rankSessionHits( + sessions, + matches(match(1, 9), match(2, 1), match(3, 5)), + 'relevance' + ) + expect(ranked).toHaveLength(1) + expect(ranked[0]!.session.id).toBe(2) + expect(ranked[0]!.duplicateCount).toBe(3) + }) + + it('leaves sessions with no shared prefix alone', () => { + const sessions = [session(1), session(2)] + const ranked = rankSessionHits(sessions, matches(match(1, 9), match(2, 5)), 'relevance') + expect(ranked.map((entry) => entry.duplicateCount)).toEqual([1, 1]) + }) +}) + +it('scores a session that matched no text at zero, less its length prior', () => { + // The operator-only page: there is no relevance signal, only an order. + const ranked = rankSessionHits([session(1, { message_count: 9 })], new Map(), 'newest') + expect(ranked[0]!.message).toBeNull() + expect(ranked[0]!.score).toBeLessThan(0) +}) diff --git a/src/main/ai-vault-search/session-search-hit-ranking.ts b/src/main/ai-vault-search/session-search-hit-ranking.ts new file mode 100644 index 00000000000..364858ea650 --- /dev/null +++ b/src/main/ai-vault-search/session-search-hit-ranking.ts @@ -0,0 +1,109 @@ +import type { AiVaultAgent } from '../../shared/ai-vault-types' +import { isCollapsibleContentHash } from './session-search-content-hash' +import type { SessionSearchSort } from './session-search-engine-types' + +// Subtracted per session: `0.02 · ln(1 + messages)`; slightly positive on both eval sets. +const LENGTH_PRIOR = 0.02 + +export type SessionRow = { + id: number + agent: AiVaultAgent + session_id: string + file_path: string + codex_home: string | null + title: string + cwd: string | null + branch: string | null + updated_at: string | null + message_count: number + resume_command: string + content_hash: string | null + content_hash_count: number +} + +/** The one message that stands for a session: its best-scoring match. */ +export type MessageRow = { + rowid: number + score: number + session_row_id: number + role: string + ts: string | null +} + +export type RankedSession = { + session: SessionRow + /** Null on an operator-only page: the session matched no text at all. */ + message: MessageRow | null + score: number + duplicateCount: number +} + +/** + * Everything between "these sessions matched" and "this is the ranked list": + * the length prior, fork folding and the caller's order. Retrieval stays in SQL + * and nothing here touches the database. + * + * The whole list is returned, not a page: a cursor indexes into it, and slicing + * here would make page two a different ranking from page one. The engine cuts + * the page and only then pays for a snippet. + */ +export function rankSessionHits( + sessions: readonly SessionRow[], + matches: ReadonlyMap, + sort: SessionSearchSort +): RankedSession[] { + const scored = collapseForks( + sessions.map((session) => { + const message = matches.get(session.id) ?? null + return { + session, + message, + score: (message?.score ?? 0) - LENGTH_PRIOR * Math.log(1 + session.message_count), + duplicateCount: 1 + } + }) + ) + // Why a total order and not just the key: a cursor is an offset into this + // list, so two entries that tie must not be free to swap between pages. + scored.sort( + (left, right) => + (sort === 'newest' + ? (right.session.updated_at ?? '').localeCompare(left.session.updated_at ?? '') + : right.score - left.score) || left.session.id - right.session.id + ) + return scored +} + +/** + * Folds forked copies of one conversation into a single entry: same opening + * prefix, newest `updated_at` wins, the rest become `duplicateCount`. Done here + * and not at write time so index rows stay per file (cursors and deletes). + */ +function collapseForks(scored: RankedSession[]): RankedSession[] { + const groups = new Map() + for (const entry of scored) { + const { content_hash: hash, content_hash_count: count, id } = entry.session + const key = isCollapsibleContentHash(hash, count) ? `hash:${hash}` : `session:${id}` + const group = groups.get(key) + if (group) { + group.push(entry) + } else { + groups.set(key, [entry]) + } + } + const collapsed: RankedSession[] = [] + for (const group of groups.values()) { + if (group.length === 1) { + collapsed.push(group[0]!) + continue + } + const winner = group.reduce((best, entry) => (isNewer(entry, best) ? entry : best)) + collapsed.push({ ...winner, duplicateCount: group.length }) + } + return collapsed +} + +function isNewer(entry: RankedSession, best: RankedSession): boolean { + const order = (entry.session.updated_at ?? '').localeCompare(best.session.updated_at ?? '') + return order === 0 ? entry.score > best.score : order > 0 +} diff --git a/src/main/ai-vault-search/session-search-index-generation.test.ts b/src/main/ai-vault-search/session-search-index-generation.test.ts new file mode 100644 index 00000000000..a3fffd2648f --- /dev/null +++ b/src/main/ai-vault-search/session-search-index-generation.test.ts @@ -0,0 +1,320 @@ +import { appendFile, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { removeTree } from '../../shared/windows-transient-lock-removal' +import type SyncDatabase from '../sqlite/sync-database' +import { SessionSearchEngine } from './session-search-engine' +import { readIndexGeneration } from './session-search-index-generation' +import { registerSessionSearchIndexConsumer } from './session-search-index-consumer' +import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache' +import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers' +import type { SessionSearchCursorError } from './session-search-page-cursor' +import { openSessionSearchDatabase } from './session-search-schema' +import { SessionSearchStore } from './session-search-store' +import { parseTranscript, userRecord } from './session-search-transcript-fixtures' + +let roots: string[] = [] +let handles: SyncDatabase[] = [] + +afterEach(async () => { + resetTranscriptConsumersForTests() + resetSessionParseCacheForTests() + for (const handle of handles) { + handle.close() + } + handles = [] + await Promise.all(roots.map((root) => removeTree(root))) + roots = [] +}) + +async function tempRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-search-generation-')) + roots.push(root) + return root +} + +/** + * A reader's own handle on the index, with the engine's schema installed. + * + * PR 2's store keeps its connection private, so a reader opens its own — which + * is what the fence has to survive: nothing this handle does moves the + * generation, and it must still see every writer's move. + */ +function reader(path: string): SyncDatabase { + const db = openSessionSearchDatabase(path) + handles.push(db) + // Constructing an engine is what installs the triggers. + new SessionSearchEngine(db) + return db +} + +/** Indexes one transcript through the real consumer and returns its path. */ +async function indexOneTranscript(root: string, store: SessionSearchStore): Promise { + resetSessionParseCacheForTests() + const sessionId = `aaaaaaaa-0000-4000-8000-${String(roots.length).padStart(12, '0')}` + const path = join(root, `${Math.random().toString(36).slice(2)}.jsonl`) + await writeFile(path, `${userRecord(0, 'generation fixture needle', sessionId)}\n`) + const unregister = registerSessionSearchIndexConsumer(store) + try { + await parseTranscript(path) + } finally { + unregister() + } + return path +} + +it('moves the generation forward when a committed read changes what a read returns', async () => { + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + const before = readIndexGeneration(db) + await indexOneTranscript(root, store) + expect(readIndexGeneration(db)).toBeGreaterThan(before) + } finally { + store.close() + } +}) + +it('moves the generation forward when an append adds rows to a live session', async () => { + // The first read of a file inserts its `files` row; every read after that + // updates it. An append changes a session's rank and its message count, so a + // cursor minted before it indexes into a list that no longer exists. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + const transcript = await indexOneTranscript(root, store) + const indexed = readIndexGeneration(db) + const unregister = registerSessionSearchIndexConsumer(store) + try { + resetSessionParseCacheForTests() + await appendFile(transcript, `${userRecord(1, 'a second needle turn')}\n`) + await parseTranscript(transcript) + } finally { + unregister() + } + expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).toEqual({ c: 2 }) + expect(readIndexGeneration(db)).toBeGreaterThan(indexed) + } finally { + store.close() + } +}) + +it('moves the generation forward when a proven deletion hides a session', async () => { + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + const transcript = await indexOneTranscript(root, store) + const indexed = readIndexGeneration(db) + store.removeFile(transcript) + expect(readIndexGeneration(db)).toBeGreaterThan(indexed) + } finally { + store.close() + } +}) + +it('moves the generation forward when retention cuts a session loose', async () => { + // Retention deletes the session row and the file row in one transaction, then + // reclaims the messages over many. It is the first half that changes what a + // search returns, and the first half that has to move the generation. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + await indexOneTranscript(root, store) + const indexed = readIndexGeneration(db) + await store.purgeOlderThan(Date.now() + 60_000) + expect(db.prepare('SELECT COUNT(*) AS c FROM sessions').get()).toEqual({ c: 0 }) + expect(readIndexGeneration(db)).toBeGreaterThan(indexed) + } finally { + store.close() + } +}) + +it('moves the generation when a purge reclaims rows nothing can reach', async () => { + // The drain writes only `messages`, and for a while that was argued to change + // no answer. Retrieval never saw those rows; the typo repair's dictionary + // did, because `messages_vocab` is a view over the FTS b-tree and lists a + // term whether or not a reader can reach it. See + // `session-search-orphan-rows.test.ts` for the answer that moved. The price + // of fencing it is a cursor refused once per batch while a purge runs. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + await indexOneTranscript(root, store) + // The shape an interrupted purge leaves: rows with no session row. + db.prepare('DELETE FROM sessions').run() + const orphaned = readIndexGeneration(db) + expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).not.toEqual({ c: 0 }) + await store.purgeOlderThan(null) + expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).toEqual({ c: 0 }) + expect(readIndexGeneration(db)).toBeGreaterThan(orphaned) + } finally { + store.close() + } +}) + +it("leaves the generation alone when a replace swaps a session's own rows", async () => { + // The same trigger must not fire here, or every re-read of a large transcript + // would move the generation once per deleted row on top of the one bump its + // file record already makes. A replace deletes rows whose session row still + // stands, which is what the trigger's `WHEN` clause tests. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + await indexOneTranscript(root, store) + const rows = db.prepare('SELECT COUNT(*) AS c FROM messages').get() as { c: number } + const indexed = readIndexGeneration(db) + db.prepare('DELETE FROM messages WHERE session_row_id IN (SELECT id FROM sessions)').run() + expect(rows.c).toBeGreaterThan(0) + expect(readIndexGeneration(db)).toBe(indexed) + } finally { + store.close() + } +}) + +it('leaves the generation alone when a removal hides nothing', async () => { + // A backfill retires paths it never held; if that moved the generation, every + // cursor would be refused for as long as indexing ran. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + await indexOneTranscript(root, store) + const before = readIndexGeneration(db) + store.removeFile('/synthetic/never-indexed.jsonl') + expect(readIndexGeneration(db)).toBe(before) + } finally { + store.close() + } +}) + +it('keeps the generation across a reopen, because the bump rides its own commit', async () => { + // The bump is inside the transaction that changes visibility, so nothing can + // be lost to a crash and reopening need not invalidate anyone's cursor. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + reader(path) + const first = new SessionSearchStore(path, (error) => { + throw error + }) + await indexOneTranscript(root, first) + const indexed = readIndexGeneration(reader(path)) + first.close() + + const second = new SessionSearchStore(path) + try { + expect(readIndexGeneration(reader(path))).toBe(indexed) + } finally { + second.close() + } +}) + +it('fences a reader against a writer it does not share a process with', async () => { + // The shape PR 3 creates: the indexer writes from the scanner child while an + // engine reads elsewhere. A generation cached in the reader's memory tracks + // only that reader's own writes, so it would stand still through the + // writer's deletion, honour the stale cursor, and skip a session. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const writer = new SessionSearchStore(path, (error) => { + throw error + }) + try { + const transcripts: string[] = [] + for (let n = 0; n < 3; n++) { + transcripts.push(await indexOneTranscript(root, writer)) + } + const engine = new SessionSearchEngine(db) + const page = engine.search({ query: 'needle', limit: 1 }) + expect(page.page.cursor).not.toBeNull() + + writer.removeFile(transcripts[0]!) + + // The reader never wrote anything, and must still refuse. + try { + engine.search({ query: 'needle', limit: 1, cursor: page.page.cursor! }) + expect.unreachable('a page cursor must not survive another writer moving the index') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('stale-generation') + } + } finally { + writer.close() + } +}) + +it('re-creates a fence something dropped, on the next search', async () => { + // An index whose triggers are gone cannot move its generation, so every stale + // cursor would compare equal and be honoured against a list the caller never + // saw. The engine owns those triggers, so it puts them back. + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const store = new SessionSearchStore(path, (error) => { + throw error + }) + try { + await indexOneTranscript(root, store) + const engine = new SessionSearchEngine(db) + db.exec('DROP TRIGGER search_generation_file_update') + engine.search({ query: 'needle' }) + + const restored = readIndexGeneration(db) + await indexOneTranscript(root, store) + expect(readIndexGeneration(db)).toBeGreaterThan(restored) + } finally { + store.close() + } +}) + +it('mints a distinct generation per change even when two handles write', async () => { + const root = await tempRoot() + const path = join(root, 'index.sqlite') + const db = reader(path) + const first = new SessionSearchStore(path, (error) => { + throw error + }) + const second = new SessionSearchStore(path, (error) => { + throw error + }) + try { + const seen: number[] = [readIndexGeneration(db)] + for (const store of [first, second, first, second]) { + await indexOneTranscript(root, store) + seen.push(readIndexGeneration(db)) + } + // Read-then-write from two connections would hand out one value twice. + expect(new Set(seen).size).toBe(seen.length) + expect([...seen].sort((left, right) => left - right)).toEqual(seen) + } finally { + second.close() + first.close() + } +}) diff --git a/src/main/ai-vault-search/session-search-index-generation.ts b/src/main/ai-vault-search/session-search-index-generation.ts new file mode 100644 index 00000000000..891608ed2e7 --- /dev/null +++ b/src/main/ai-vault-search/session-search-index-generation.ts @@ -0,0 +1,97 @@ +import type SyncDatabase from '../sqlite/sync-database' + +const GENERATION_KEY = 'index_generation' + +/** + * Names of the triggers that move the generation. Exported so the query schema + * can check they are all still there before an engine trusts a cursor. + */ +export const SESSION_SEARCH_GENERATION_TRIGGERS = [ + 'search_generation_file_insert', + 'search_generation_file_update', + 'search_generation_file_delete', + 'search_generation_orphan_reclaim' +] as const + +const BUMP = `INSERT INTO meta(key, value) VALUES ('${GENERATION_KEY}', '1') + ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1;` + +/** + * The fence, as three triggers on `files`. + * + * Why `files`. Every transaction the store opens that can change what a search + * returns writes this table: a committed read upserts the file's cursor beside + * its rows, a chunk of a long read upserts the partial sentinel beside its + * prefix, `removeFile` deletes the row with the session, and retention deletes + * the file row in the same transaction as the session row. + * + * And why `messages` as well, for orphans only. Retention's second half + * reclaims rows whose session row is already gone, and touches neither table + * above. It was left unfenced on the argument that those rows answer nothing, + * which is true of retrieval and was not true of the whole engine: the typo + * repair's dictionary is `messages_vocab`, a view over the FTS b-tree that + * lists a term whether or not a reader can reach the rows carrying it, and + * reclaiming them moved which word a query was repaired to. The repair now + * counts live rows instead, so the common case is fixed at its source; this + * trigger is what makes the fence true rather than nearly true, because the + * vocabulary still decides which candidates survive its scan limit. + * + * The `WHEN` clause is what keeps it free. `removeFile` deletes a session's + * rows while its `sessions` row still stands, so it does not fire here; a + * replace cuts the old `sessions` row loose and leaves its messages to the + * drain (PR 2 round 10). Both already bump through `files`; the drain is the + * only path that deletes a row whose session is gone, and it fires here. The cost of the fence is real and worth naming: a + * cursor outstanding while a purge runs is refused once per batch, which + * `SessionSearchCursorError` reports as `stale-generation` so a caller + * re-issues page one rather than showing anyone an error. + * + * A trigger rather than a call the writer makes, for two reasons. PR 4 does not + * own the writer, and more importantly the fence has to hold for writers this + * process cannot see: the triggers live in the file, so PR 3's indexer in the + * scanner child moves the generation without knowing a reader exists. + * + * Correctness comes from where the increment runs, not from what it counts. It + * is one statement inside the writer's own `BEGIN IMMEDIATE`, so it commits + * with the change it describes and two connections cannot mint one value twice. + * It over-counts in one harmless direction: a read that decoded no session from + * a file the index also held no session for advances a cursor and bumps + * anyway. That refuses a cursor early; it never honours one late. + */ +export const SESSION_SEARCH_GENERATION_SQL = ` +CREATE TRIGGER IF NOT EXISTS search_generation_file_insert AFTER INSERT ON files BEGIN + ${BUMP} +END; +CREATE TRIGGER IF NOT EXISTS search_generation_file_update AFTER UPDATE ON files BEGIN + ${BUMP} +END; +CREATE TRIGGER IF NOT EXISTS search_generation_file_delete AFTER DELETE ON files BEGIN + ${BUMP} +END; +CREATE TRIGGER IF NOT EXISTS search_generation_orphan_reclaim AFTER DELETE ON messages +WHEN NOT EXISTS (SELECT 1 FROM sessions WHERE id = OLD.session_row_id) BEGIN + ${BUMP} +END; +` + +/** + * A monotone id for what the index currently publishes. + * + * A search page is a slice of one ranked list, so a cursor only means anything + * against the snapshot that produced it. Every change to what a read can return + * moves this on, and a cursor minted under an older value is refused rather + * than silently re-run against a list it no longer indexes into. + * + * Read from the database on every call, never cached in a process. The writer + * and the reader need not be the same one: PR 3's indexer runs in the scanner + * child while an engine reads elsewhere, and any number of handles may be open + * on one file. A generation cached in memory only ever tracks that process's + * own writes, so a reader would see another writer's deletions while its + * generation stood still, honour a stale cursor, and skip a session. + */ +export function readIndexGeneration(db: SyncDatabase): number { + const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(GENERATION_KEY) as + | { value: string } + | undefined + const parsed = row ? Number(row.value) : Number.NaN + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0 +} diff --git a/src/main/ai-vault-search/session-search-orphan-rows.test.ts b/src/main/ai-vault-search/session-search-orphan-rows.test.ts new file mode 100644 index 00000000000..dfda2303104 --- /dev/null +++ b/src/main/ai-vault-search/session-search-orphan-rows.test.ts @@ -0,0 +1,186 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type SyncDatabase from '../sqlite/sync-database' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' +import { identifierShadowText } from './session-search-identifier-split' +import { readIndexGeneration } from './session-search-index-generation' +import { planSessionSearchQuery } from './session-search-query-planner' +import { sessionSearchSnippet } from './session-search-snippet' +import type { SessionSearchCursorError } from './session-search-page-cursor' +import { SessionSearchTypoRepair } from './session-search-typo-repair' + +// Retention deletes a session row in one small transaction and reclaims its +// message rows in batches afterwards, so a `messages` row with no `sessions` row +// is a state every purge, every removed source and every interrupted drain +// passes through. Those rows are still in both FTS tables and still in the +// vocabulary, and nothing here may return one. +// +// A hit is a session row, and the ranked list is loaded `FROM sessions`, so the +// route ladder below cannot surface an orphan even if a join were loosened — +// those cases are a ratchet over the shape, not the proof. The two reads that +// can leak one are pinned separately and each is a real oracle: the snippet, +// which is handed a rowid and asked for its text, and the typo repair, whose +// dictionary is the FTS b-tree and lists an orphan's terms like any other. + +const ORPHAN_SESSION_ROW = 99 +const ORPHAN_TEXT = 'orphaned marmoset secret' + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +/** Two rows in the FTS table and the vocabulary, and no session row for them. */ +function plantOrphans(db: SyncDatabase, text: string = ORPHAN_TEXT): number[] { + const rowids: number[] = [] + for (let n = 0; n < 2; n++) { + const rowid = Number( + db + .prepare("INSERT INTO messages(session_row_id,role,ts) VALUES (?,'user',?)") + .run(ORPHAN_SESSION_ROW, '2026-09-10T00:00:00.000Z').lastInsertRowid + ) + db.prepare( + 'INSERT INTO messages_fts(rowid,user_text,assistant_text,tool_text,identifiers) VALUES (?,?,?,?,?)' + ).run(rowid, text, '', '', identifierShadowText(text)) + rowids.push(rowid) + } + return rowids +} + +async function withOrphans(): Promise<{ harness: SessionSearchHarness; rowids: number[] }> { + harness = await openSessionSearchHarness('ss-orphan-rows') + addSyntheticSession(harness.db, { id: 1, text: 'the haystack line here' }) + const rowids = plantOrphans(harness.db) + // The oracle only means anything if the rows are really there to be found. + expect( + harness.db + .prepare("SELECT count(*) AS c FROM messages_fts WHERE messages_fts MATCH 'marmoset'") + .get() + ).toEqual({ c: 2 }) + expect( + harness.db.prepare("SELECT doc FROM messages_vocab WHERE term = 'marmoset'").get() + ).toEqual({ doc: 2 }) + return { harness, rowids } +} + +it.each([ + ['phrase', '"orphaned marmoset"'], + ['and', 'orphaned secret'], + ['single-token literal', 'marmoset'], + ['or', 'marmoset haystack orphaned'], + ['typo repair', 'marmosett'], + ['operator only', 'repo:app'] +])('returns no orphaned row on the %s route', async (_route, query) => { + const { harness: open } = await withOrphans() + for (const scope of ['all', 'conversation'] as const) { + const hits = open.engine.search({ query, scope }).hits + expect(hits.map((hit) => hit.sessionId)).not.toContain(String(ORPHAN_SESSION_ROW)) + expect(hits.filter((hit) => hit.evidence?.snippet.includes('marmoset'))).toEqual([]) + } +}) + +it('never repairs a term onto a spelling only orphaned rows carry', async () => { + const { harness: open } = await withOrphans() + // `marmoset` is in the vocabulary twice, which is what would make it the + // repair for `marmosett` if the repair trusted the vocabulary alone. + expect(new SessionSearchTypoRepair(open.db).correct('marmosett', 'all')).toBeNull() + expect(open.engine.search({ query: 'marmosett' }).planner.repairedTerms).toBeUndefined() +}) + +it('snippets nothing for an orphaned row, even asked for it by rowid', async () => { + const { harness: open, rowids } = await withOrphans() + const plan = planSessionSearchQuery('marmoset') + for (const scope of ['all', 'conversation'] as const) { + expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan)).toEqual({ + text: '', + truncated: false + }) + } +}) + +it('still answers for the live session beside them', async () => { + const { harness: open } = await withOrphans() + expect(open.engine.search({ query: 'haystack' }).hits.map((hit) => hit.sessionId)).toEqual(['1']) +}) + +// Reclaiming those rows is the other half. The drain deletes only from +// `messages`, so for a long time it was argued to change no answer and left +// outside the generation fence. Retrieval never saw them, but the typo repair's +// dictionary is `messages_vocab`, a view over the FTS b-tree that lists a term +// whether or not a reader can reach the rows carrying it — so the drain moved +// which word a query was repaired to, under a cursor that was still honoured. +describe('a purge reclaiming rows nothing can reach', () => { + /** A live session and a purged one that both carry `text`. */ + async function withReclaimable(): Promise { + harness = await openSessionSearchHarness('ss-orphan-drain') + // Two live rows, which is what makes `marmoset` eligible as a repair at all. + addSyntheticSession(harness.db, { id: 1, text: 'the marmoset lives here', rows: 2 }) + plantOrphans(harness.db) + return harness + } + + it('answers the same before and after, because the repair counts live rows', async () => { + const open = await withReclaimable() + const before = open.engine.search({ query: 'marmosett' }) + expect(before.planner.repairedTerms).toEqual(['marmoset']) + expect(before.hits.map((hit) => hit.sessionId)).toEqual(['1']) + + await open.store.purgeOlderThan(null) + expect(open.db.prepare('SELECT count(*) AS c FROM messages').get()).toEqual({ c: 2 }) + + const after = open.engine.search({ query: 'marmosett' }) + expect(after.planner.repairedTerms).toEqual(before.planner.repairedTerms) + expect(after.hits.map((hit) => hit.sessionId)).toEqual(before.hits.map((hit) => hit.sessionId)) + }) + + it('moves the generation anyway, so no cursor spans it', async () => { + // The repair counting live rows fixes the common case. It does not make the + // drain provably inert: `messages_vocab` still decides which candidates + // survive its scan limit, and reclaiming a term's last row changes where + // that limit cuts. The fence is what covers the rest, at the price of + // refusing a cursor once per batch while a purge runs. + const open = await withReclaimable() + // A second live session, so page one has a page two to be refused. + addSyntheticSession(open.db, { id: 2, text: 'the marmoset again', rows: 2 }) + const page = open.engine.search({ query: 'marmoset', limit: 1 }) + expect(page.page.cursor).not.toBeNull() + const before = readIndexGeneration(open.db) + + await open.store.purgeOlderThan(null) + + expect(readIndexGeneration(open.db)).toBeGreaterThan(before) + try { + open.engine.search({ query: 'marmoset', limit: 1, cursor: page.page.cursor! }) + expect.unreachable('a cursor must not span a purge') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('stale-generation') + } + }) + + it('picks the same repair when an unreachable spelling was the more common one', async () => { + // Two candidates equally close to the query. `marmosetx` led on the old + // ranking only because two of its rows belonged to a session retention had + // already cut loose, so the drain swapped the repair under a live cursor. + harness = await openSessionSearchHarness('ss-orphan-drain-tie') + const db = harness.db + for (let id = 1; id <= 4; id++) { + addSyntheticSession(db, { id, text: `marmosetx session${id}` }) + } + for (let id = 5; id <= 9; id++) { + addSyntheticSession(db, { id, text: `marmosetq session${id}` }) + } + plantOrphans(db, 'marmosetx') + + const before = harness.engine.search({ query: 'marmosett' }) + expect(before.planner.repairedTerms).toEqual(['marmosetq']) + await harness.store.purgeOlderThan(null) + expect(harness.engine.search({ query: 'marmosett' }).planner.repairedTerms).toEqual( + before.planner.repairedTerms + ) + }) +}) diff --git a/src/main/ai-vault-search/session-search-page-cursor.ts b/src/main/ai-vault-search/session-search-page-cursor.ts new file mode 100644 index 00000000000..838e5e1e3ab --- /dev/null +++ b/src/main/ai-vault-search/session-search-page-cursor.ts @@ -0,0 +1,108 @@ +import { createHash } from 'node:crypto' +import type { SessionSearchRequest } from './session-search-engine-types' + +export type SessionSearchCursorRejection = 'stale-generation' | 'different-query' | 'malformed' + +/** + * A cursor the engine refuses to honour. Typed, and thrown rather than + * swallowed: silently restarting at page one hands the caller a page it has + * already shown as if it were the next one, and silently re-running against a + * newer index hands it a slice of a list it never saw. + */ +export class SessionSearchCursorError extends Error { + constructor( + readonly rejection: SessionSearchCursorRejection, + /** + * The generation the index is at now. Always present: the engine knows it + * before it looks at the cursor at all. + */ + readonly actualGeneration: number, + /** + * The generation the cursor claims it was minted in. Absent only when the + * cursor could not be decoded far enough to carry a number, which is one of + * the `malformed` cases. + */ + readonly expectedGeneration?: number + ) { + super(`Search cursor rejected: ${rejection}`) + this.name = 'SessionSearchCursorError' + } +} + +type CursorPayload = { + /** Index generation. */ + g: number + /** + * Offset into the ranked list, not a session id. Ids are not in a cursor at + * all, so nothing here depends on `sessions.id` being unique over time — + * though it is, because PR 2 made the column AUTOINCREMENT so a purged + * session's id is never reissued to a live one. + */ + o: number + /** Query identity; see `sessionSearchPageKey`. */ + k: string +} + +/** + * Everything a page's ranking depends on except the limit. Two requests with + * the same key produce the same ranked list within one generation, so a cursor + * minted by one is meaningful to the other; the limit is left out on purpose so + * a caller may change its page size mid-pagination. + */ +export function sessionSearchPageKey(request: SessionSearchRequest): string { + const filters = request.filters ?? {} + const identity = JSON.stringify([ + request.query, + request.scope ?? 'all', + filters.sort ?? 'relevance', + filters.since ?? null, + [...(filters.agents ?? [])].sort(), + [...(filters.scopePaths ?? [])].sort() + ]) + return createHash('sha256').update(identity).digest('base64url').slice(0, 16) +} + +export function encodeSessionSearchCursor(generation: number, offset: number, key: string): string { + const payload: CursorPayload = { g: generation, o: offset, k: key } + return Buffer.from(JSON.stringify(payload), 'utf-8').toString('base64url') +} + +/** + * The offset this cursor points at, or a typed rejection. + * + * Every rejection carries `actualGeneration`, and every one that could read a + * generation out of the cursor carries `expectedGeneration` too, so a caller + * can tell "the index moved under you, ask for page one" from "this cursor is + * not ours" and act on the first without showing anyone an error. + */ +export function decodeSessionSearchCursor(cursor: string, generation: number, key: string): number { + let payload: CursorPayload + try { + payload = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf-8')) as CursorPayload + } catch { + throw new SessionSearchCursorError('malformed', generation) + } + // A generation that survived parsing is worth reporting even when the rest of + // the payload is unusable: it is what tells the caller which snapshot the + // cursor thought it was walking. + const claimed = + typeof payload?.g === 'number' && Number.isFinite(payload.g) ? payload.g : undefined + if ( + claimed === undefined || + !Number.isInteger(payload?.o) || + payload.o < 0 || + typeof payload?.k !== 'string' + ) { + throw new SessionSearchCursorError('malformed', generation, claimed) + } + // Generation first: a caller who changed the query AND waited through a + // publish should hear about the index moving, which is the condition it + // cannot fix by paging again. + if (claimed !== generation) { + throw new SessionSearchCursorError('stale-generation', generation, claimed) + } + if (payload.k !== key) { + throw new SessionSearchCursorError('different-query', generation, claimed) + } + return payload.o +} diff --git a/src/main/ai-vault-search/session-search-paging.test.ts b/src/main/ai-vault-search/session-search-paging.test.ts new file mode 100644 index 00000000000..319c5b0dba4 --- /dev/null +++ b/src/main/ai-vault-search/session-search-paging.test.ts @@ -0,0 +1,309 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type { SessionSearchRequest } from './session-search-engine-types' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' +import { readIndexGeneration } from './session-search-index-generation' +import { + decodeSessionSearchCursor, + encodeSessionSearchCursor, + SessionSearchCursorError, + sessionSearchPageKey +} from './session-search-page-cursor' + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +async function open(name: string, options = {}): Promise { + harness = await openSessionSearchHarness(name, options) + return harness +} + +async function withSessions(count: number, options = {}): Promise { + harness = await openSessionSearchHarness('ss-engine-paging', options) + for (let id = 1; id <= count; id++) { + addSyntheticSession(harness.db, { + id, + text: `needle padding ${'word '.repeat(id % 5)}`, + updatedAt: `2026-09-${String(id).padStart(2, '0')}T00:00:00.000Z` + }) + } + return harness +} + +describe('a cursor walks one ranked list', () => { + it('pages through every session exactly once, in one stable order', async () => { + const { engine } = await withSessions(25) + const request: SessionSearchRequest = { query: 'needle', limit: 10 } + const seen: string[] = [] + let cursor: string | null = null + let pages = 0 + do { + const page = engine.search(cursor ? { ...request, cursor } : request) + seen.push(...page.hits.map((hit) => hit.sessionId)) + cursor = page.page.cursor + pages++ + expect(pages).toBeLessThan(10) + } while (cursor !== null) + + expect(pages).toBe(3) + expect(seen).toHaveLength(25) + expect(new Set(seen).size).toBe(25) + // The same walk, run again against the same generation, is the same walk. + expect(engine.search(request).hits.map((hit) => hit.sessionId)).toEqual(seen.slice(0, 10)) + }) + + it('closes the page when the last hit has been handed out', async () => { + const { engine } = await withSessions(3) + const page = engine.search({ query: 'needle', limit: 10 }) + expect(page.hits).toHaveLength(3) + expect(page.page.hasMore).toBe(false) + expect(page.page.cursor).toBeNull() + }) + + it('lets a caller change page size mid-walk', async () => { + const { engine } = await withSessions(12) + const first = engine.search({ query: 'needle', limit: 5 }) + const rest = engine.search({ query: 'needle', limit: 20, cursor: first.page.cursor! }) + expect(rest.hits).toHaveLength(7) + expect(rest.page.hasMore).toBe(false) + }) + + it('breaks a tie by session, so two entries cannot swap between pages', async () => { + // Same text, same timestamp: every ranking key is equal, which is exactly + // where an unstable sort would hand one session out twice and lose another. + harness = await openSessionSearchHarness('ss-engine-ties') + for (let id = 1; id <= 6; id++) { + addSyntheticSession(harness.db, { id, text: 'needle', updatedAt: '2026-09-01T00:00:00.000Z' }) + } + const first = harness.engine.search({ query: 'needle', limit: 3 }) + const second = harness.engine.search({ query: 'needle', limit: 3, cursor: first.page.cursor! }) + const seen = [...first.hits, ...second.hits].map((hit) => hit.sessionId) + expect(seen).toEqual(['1', '2', '3', '4', '5', '6']) + }) +}) + +describe('a cursor is refused rather than reinterpreted', () => { + it('rejects a cursor minted before the index moved', async () => { + const { engine, store } = await withSessions(25) + const first = engine.search({ query: 'needle', limit: 10 }) + // A proven deletion of a path this index really held hides a session, which + // is exactly the change a cursor must not be allowed to page across. + store.removeFile('/synthetic/1.jsonl') + + expect(() => engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! })).toThrow( + SessionSearchCursorError + ) + try { + engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! }) + expect.unreachable('a stale cursor must not be silently re-run') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('stale-generation') + } + }) + + it('names both generations, so a caller can tell a moved index from a bad cursor', async () => { + // What a caller does about it differs: a moved index means quietly ask for + // page one again, a bad cursor means something is wrong with the caller. + const { engine, store } = await withSessions(25) + const first = engine.search({ query: 'needle', limit: 10 }) + const minted = readIndexGeneration(harness!.db) + // Any published read moves the generation, including one for a file this + // page never mentioned. That is the fence working, not a defect. + store.removeFile('/synthetic/9.jsonl') + + try { + engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! }) + expect.unreachable('the index moved') + } catch (error) { + const rejected = error as SessionSearchCursorError + expect(rejected.rejection).toBe('stale-generation') + expect(rejected.expectedGeneration).toBe(minted) + expect(rejected.actualGeneration).toBe(readIndexGeneration(harness!.db)) + expect(rejected.actualGeneration).toBeGreaterThan(rejected.expectedGeneration!) + } + }) + + it('rejects a cursor carried over to a different query', async () => { + const { engine } = await withSessions(25) + const first = engine.search({ query: 'needle', limit: 10 }) + try { + engine.search({ query: 'padding', limit: 10, cursor: first.page.cursor! }) + expect.unreachable('a cursor indexes into one ranked list, not any list') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('different-query') + } + }) + + it('rejects a cursor whose filters changed, which reranks the list', async () => { + const { engine } = await withSessions(25) + const first = engine.search({ query: 'needle', limit: 10 }) + try { + engine.search({ + query: 'needle', + limit: 10, + cursor: first.page.cursor!, + filters: { sort: 'newest' } + }) + expect.unreachable('a different sort is a different ranked list') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('different-query') + } + }) + + // Every field the ranked list depends on has to be in the key, and a field + // that is in the key but never pinned is a field a refactor can drop while + // the suite stays green. One case each, through the engine, so the assertion + // is about a refused page and not about a hash. + it.each([ + ['scope', { scope: 'conversation' as const }], + ['sort', { filters: { sort: 'newest' as const } }], + ['agents', { filters: { agents: ['codex' as const] } }], + ['scopePaths', { filters: { scopePaths: ['/repo/app'] } }], + ['since', { filters: { since: '2026-09-01T00:00:00.000Z' } }] + ])('rejects a cursor presented with a different %s', async (_field, changed) => { + const { engine } = await withSessions(25) + const request: SessionSearchRequest = { + query: 'needle', + limit: 10, + scope: 'all', + filters: { sort: 'relevance', agents: ['claude'], scopePaths: ['/'], since: undefined } + } + const first = engine.search(request) + expect(first.page.cursor).not.toBeNull() + try { + engine.search({ + ...request, + ...changed, + filters: { ...request.filters, ...('filters' in changed ? changed.filters : {}) }, + cursor: first.page.cursor! + }) + expect.unreachable('a narrowing the ranked list depends on must invalidate the cursor') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('different-query') + } + }) + + it('rejects a cursor that is not one of ours', async () => { + const { engine } = await withSessions(3) + try { + engine.search({ query: 'needle', cursor: 'not-a-cursor' }) + expect.unreachable('a malformed cursor is not an empty one') + } catch (error) { + expect((error as SessionSearchCursorError).rejection).toBe('malformed') + } + }) +}) + +describe('cursor encoding', () => { + const request: SessionSearchRequest = { query: 'needle', filters: { scopePaths: ['/a'] } } + + it('round-trips an offset within its own generation and query', () => { + const key = sessionSearchPageKey(request) + expect(decodeSessionSearchCursor(encodeSessionSearchCursor(7, 40, key), 7, key)).toBe(40) + }) + + it('keys a request by what changes its ranking, and not by its page size', () => { + expect(sessionSearchPageKey({ ...request, limit: 5 })).toBe( + sessionSearchPageKey({ ...request, limit: 50 }) + ) + expect(sessionSearchPageKey({ ...request, scope: 'conversation' })).not.toBe( + sessionSearchPageKey(request) + ) + }) + + it('reads a filter list in any order as the same request', () => { + expect(sessionSearchPageKey({ query: 'a', filters: { agents: ['claude', 'codex'] } })).toBe( + sessionSearchPageKey({ query: 'a', filters: { agents: ['codex', 'claude'] } }) + ) + }) + + it.each([ + ['a negative offset', encodeSessionSearchCursor(1, -1, 'k'), 1], + ['a non-integer offset', Buffer.from('{"g":1,"o":1.5,"k":"k"}').toString('base64url'), 1], + ['a payload that is not an object', Buffer.from('"nope"').toString('base64url'), undefined], + ['text that is not base64url JSON', 'zzz!!', undefined] + ])('rejects %s as malformed, still naming the index generation', (_name, cursor, claimed) => { + // The caller has to know which snapshot it was refused against whatever was + // wrong with the cursor, and the generation it claimed whenever that + // survived parsing. + try { + decodeSessionSearchCursor(cursor, 7, 'k') + expect.unreachable('a malformed cursor is not an empty one') + } catch (error) { + const rejected = error as SessionSearchCursorError + expect(rejected.rejection).toBe('malformed') + expect(rejected.actualGeneration).toBe(7) + expect(rejected.expectedGeneration).toBe(claimed) + } + }) +}) + +describe('the candidate limit is a tunable default, and says when it cut', () => { + it('does not claim truncation when every session fits', async () => { + const { engine } = await withSessions(5, { sessionCandidateLimit: 600 }) + expect(engine.search({ query: 'needle' }).truncated.candidates).toBe(false) + }) + + it('claims truncation, and ranks only what it retrieved, at the limit', async () => { + const { engine } = await withSessions(10, { sessionCandidateLimit: 4 }) + const result = engine.search({ query: 'needle', limit: 100 }) + expect(result.truncated.candidates).toBe(true) + expect(result.hits).toHaveLength(4) + }) + + it('applies the same limit to an operator-only page', async () => { + const { engine } = await withSessions(10, { sessionCandidateLimit: 4 }) + const result = engine.search({ query: 'repo:app', limit: 100 }) + expect(result.truncated.candidates).toBe(true) + expect(result.hits).toHaveLength(4) + }) + + it('says it gave up when the operator walk stopped scanning, not that it is done', async () => { + // The shape that reads as a confident empty answer: the only match sits + // past the walk's ceiling, so the walk stops having found nothing. Zero + // hits and `truncated.candidates` false would tell a caller there is + // nothing to find, which is a different claim from "I stopped looking". + // The walk reads a page at a time and gives up past a ceiling of + // `candidateLimit` x 20, so the corpus has to be deeper than one page for + // the ceiling to be what ends it. The only match is the oldest session. + const deep = 600 + const { db, engine } = await open('ss-engine-sparse-deep', { sessionCandidateLimit: 2 }) + for (let id = 1; id <= deep; id++) { + addSyntheticSession(db, { + id, + cwd: id === deep ? '/repo/needleonly' : '/repo/app', + updatedAt: new Date(Date.UTC(2026, 8, 9) - id * 60_000).toISOString() + }) + } + const result = engine.search({ query: 'repo:needleonly' }) + expect(result.hits).toHaveLength(0) + expect(result.truncated.candidates).toBe(true) + }) + + it('does not claim it gave up when the walk really did read everything', async () => { + const { db, engine } = await open('ss-engine-sparse-shallow', { sessionCandidateLimit: 600 }) + addSyntheticSession(db, { id: 1, cwd: '/repo/app' }) + const result = engine.search({ query: 'repo:nothing-here' }) + expect(result.hits).toHaveLength(0) + expect(result.truncated.candidates).toBe(false) + }) +}) + +describe('the response carries the snapshot it was built from', () => { + it('reports the index generation on every result', async () => { + const { db, engine, store } = await withSessions(3) + const before = engine.search({ query: 'needle' }).generation + expect(before).toBe(readIndexGeneration(db)) + store.removeFile('/synthetic/1.jsonl') + const after = engine.search({ query: 'needle' }).generation + expect(after).toBe(readIndexGeneration(db)) + expect(after).toBeGreaterThan(before) + }) +}) diff --git a/src/main/ai-vault-search/session-search-query-log.test.ts b/src/main/ai-vault-search/session-search-query-log.test.ts new file mode 100644 index 00000000000..7b88e628a83 --- /dev/null +++ b/src/main/ai-vault-search/session-search-query-log.test.ts @@ -0,0 +1,61 @@ +import { afterEach, expect, it } from 'vitest' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' +import { logSessionSearchQuery } from './session-search-query-log' + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +async function open(options = {}): Promise { + harness = await openSessionSearchHarness('ss-query-log', options) + addSyntheticSession(harness.db, { id: 1, text: 'needle' }) + return harness +} + +function loggedQueries(harness: SessionSearchHarness): string[] { + return ( + harness.db.prepare('SELECT query FROM search_log ORDER BY id').all() as { query: string }[] + ).map((row) => row.query) +} + +it('writes nothing on the query path unless the caller asked for a log', async () => { + const opened = await open() + opened.engine.search({ query: 'needle' }) + expect(loggedQueries(opened)).toEqual([]) +}) + +it('records the query and its route when logging is on', async () => { + const opened = await open({ logQueries: true }) + opened.engine.search({ query: 'needle' }) + const rows = opened.db.prepare('SELECT query, route, hits FROM search_log').all() as { + query: string + route: string + hits: number + }[] + expect(rows).toEqual([{ query: 'needle', route: 'or', hits: 1 }]) +}) + +it('stores the query as typed, the way the index stores content as written', async () => { + // PR 2 decided the index does not redact: it is a second copy of plaintext + // the user already holds under their own home directory. The same holds for + // what they typed into the search box. + const opened = await open({ logQueries: true }) + opened.engine.search({ query: 'Bearer abcdefghijklmnopqrstuvwxyz012345' }) + expect(loggedQueries(opened)[0]).toBe('Bearer abcdefghijklmnopqrstuvwxyz012345') +}) + +it('keeps the newest N and drops the rest, so the log cannot grow with use', async () => { + const opened = await open() + // The real ceiling is 5,000; the trim is the same statement at any size. + for (let n = 0; n < 12; n++) { + logSessionSearchQuery(opened.db, { query: `q${n}`, route: 'or', hits: 0, durationMs: 1 }, 5) + } + expect(loggedQueries(opened)).toEqual(['q7', 'q8', 'q9', 'q10', 'q11']) +}) diff --git a/src/main/ai-vault-search/session-search-query-log.ts b/src/main/ai-vault-search/session-search-query-log.ts new file mode 100644 index 00000000000..cdc5ce54425 --- /dev/null +++ b/src/main/ai-vault-search/session-search-query-log.ts @@ -0,0 +1,30 @@ +import type SyncDatabase from '../sqlite/sync-database' + +export const SEARCH_LOG_LIMIT = 5000 + +/** + * Local-only telemetry the eval set is rebuilt from. + * + * The query is stored as typed, for the reason PR 2 gives for not redacting + * transcript content: this file sits beside an index that already holds the + * user's own plaintext, so a second copy of what they typed is not a new + * exposure. What may leave the machine is a transport policy and belongs where + * the wire is. + * + * Nothing enables this by default: the engine writes a row only when its caller + * asked for it, because a log write on the query path is a write on what is + * otherwise a read-only lane. Who turns it on is PR 3b's settings decision. + */ +export function logSessionSearchQuery( + db: SyncDatabase, + entry: { query: string; route: string; hits: number; durationMs: number }, + limit: number = SEARCH_LOG_LIMIT +): void { + db.prepare( + 'INSERT INTO search_log(ts, query, route, hits, duration_ms) VALUES (?, ?, ?, ?, ?)' + ).run(new Date().toISOString(), entry.query, entry.route, entry.hits, entry.durationMs) + db.prepare( + `DELETE FROM search_log WHERE id <= ( + SELECT id FROM search_log ORDER BY id DESC LIMIT 1 OFFSET ?)` + ).run(limit) +} diff --git a/src/main/ai-vault-search/session-search-query-planner.test.ts b/src/main/ai-vault-search/session-search-query-planner.test.ts new file mode 100644 index 00000000000..ac4874b1d10 --- /dev/null +++ b/src/main/ai-vault-search/session-search-query-planner.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { + andExpression, + isLiteralQuery, + orExpression, + phraseExpression, + planSessionSearchQuery, + quoteFtsTerm +} from './session-search-query-planner' + +describe('literal shape decides whether the phrase route is even tried', () => { + it.each([ + 'resolveTerminalPath', + 'src/main/foo-bar.ts', + 'MAX_RETRY_COUNT', + 'kern.tty.ptmx_max', + '#19687', + 'STA-4850', + '"exact words here"', + 'TypeError: undefined', + 'foo() {' + ])('treats %s as quoting something from a transcript', (query) => { + expect(isLiteralQuery(query)).toBe(true) + }) + + it.each(['why is the terminal slow', 'how do I resume a session', 'relay capacity'])( + 'treats %s as prose', + (query) => { + expect(isLiteralQuery(query)).toBe(false) + } + ) +}) + +describe('the body is what the phrase and AND routes see', () => { + it('drops stop words from prose so the AND route is not defeated by "the"', () => { + expect(planSessionSearchQuery('why is the relay dropping frames').body).toEqual([ + 'relay', + 'dropping', + 'frames' + ]) + }) + + it('keeps stop words inside a literal, where they are part of what was quoted', () => { + // The literal shape is `foo.ts`; dropping `the` would change what was typed. + expect(planSessionSearchQuery('the foo.ts file').body).toEqual(['the', 'foo.ts', 'file']) + }) + + it('keeps a query that is nothing but stop words rather than answering nothing', () => { + expect(planSessionSearchQuery('how do I').body).toEqual(['how', 'do', 'I']) + }) + + it('has no terms for a query with no searchable token', () => { + expect(planSessionSearchQuery(' ... ').terms).toEqual([]) + }) +}) + +describe('the OR fallback fans an identifier out into its pieces', () => { + it('adds the split pieces after the whole term, never in place of it', () => { + const plan = planSessionSearchQuery('resolveTerminalPath') + expect(plan.terms[0]).toBe('resolveTerminalPath') + expect(plan.terms).toContain('terminal') + expect(plan.terms).toContain('path') + // `resolve` is not a stop word, so the whole identifier is reachable by piece. + expect(plan.terms).toContain('resolve') + }) + + it('leaves an ordinary word alone', () => { + expect(planSessionSearchQuery('relay').terms).toEqual(['relay']) + }) +}) + +describe('FTS5 expressions quote every term', () => { + it('quotes punctuation that would otherwise be syntax', () => { + expect(quoteFtsTerm('cli.mjs')).toBe('"cli.mjs"') + expect(quoteFtsTerm('C++')).toBe('"C++"') + expect(quoteFtsTerm('say "hi"')).toBe('"say ""hi"""') + }) + + it('builds one phrase, an AND chain, and an OR chain from the same terms', () => { + expect(phraseExpression(['alpha', 'beta'])).toBe('"alpha beta"') + expect(andExpression(['alpha', 'beta'])).toBe('"alpha" AND "beta"') + expect(orExpression(['alpha', 'beta'])).toBe('"alpha" OR "beta"') + }) +}) diff --git a/src/main/ai-vault-search/session-search-query-planner.ts b/src/main/ai-vault-search/session-search-query-planner.ts new file mode 100644 index 00000000000..6c2c2f3b91c --- /dev/null +++ b/src/main/ai-vault-search/session-search-query-planner.ts @@ -0,0 +1,140 @@ +import type { SessionSearchScope } from './session-search-engine-types' +import { identifierShadowTerms } from './session-search-identifier-split' + +// Tokens exactly as the unicode61 tokenizer with `_ . - / +` tokenchars emits them. +const INDEX_TOKEN = /[\p{L}\p{N}\p{M}\p{Co}_./+-]+/gu +const STOP_WORDS = new Set( + ( + 'a an and are as at be but by for from how i if in into is it its of on or that the this to ' + + 'was were what when where which who why with you your we my me do does did not no can could ' + + 'should would about our us they them there their has have had been being so such then than ' + + "these those there's im ive dont" + ).split(' ') +) +const MAX_BODY_TERMS = 48 +const MAX_TERMS = 64 + +// A query that quotes something from a transcript: camelCase, SCREAMING_SNAKE, +// a dotted or snake_case name, a path, a filename, a PR number, a ticket, code +// punctuation, or an error word. +const LITERAL_SHAPE = + /[A-Za-z0-9_]*[a-z][A-Z][A-Za-z0-9_]*|\b[A-Z][A-Z0-9]{2,}(_[A-Z0-9]+)+\b|\b\w{2,}[._]\w{2,}\b|\b[\w.-]+\/[\w/.-]+\b|\b\w+\.(ts|tsx|js|jsx|py|rs|go|json|md|sh|yml|yaml|toml|c|cc|h|java|sql)\b|#\d{3,}|\b[A-Z]{2,6}-\d{2,}\b|[(){};=]|::|->|--\w|\b(Error|Exception|Traceback|error:|warning:)\b/ +const QUOTED = /"[^"]{3,}"|'[^']{3,}'/ + +export type SessionSearchQueryPlan = { + literal: boolean + /** + * The query had more terms than the planner will search. What is dropped is + * the tail, so a match that only the last term would have found is missed; + * the caller is told rather than handed a confident empty answer. + */ + truncated: boolean + /** Deduplicated index-faithful terms for the OR fallback, incl. identifier pieces. */ + terms: string[] + /** Query-order tokens minus stop words: the phrase / AND candidate. */ + body: string[] +} + +export function isLiteralQuery(query: string): boolean { + return QUOTED.test(query) || LITERAL_SHAPE.test(query) +} + +/** + * The tokenizer contract, unfolded: the same boundaries FTS5 draws for + * `unicode61 tokenchars '_.-/+'`. Pinned against real `fts5vocab` output in + * session-search-fts5-contract.test.ts, which is what makes it safe to plan a + * query without asking SQLite. + */ +export function indexTokens(query: string, limit = Number.POSITIVE_INFINITY): string[] { + const out: string[] = [] + for (const match of query.matchAll(INDEX_TOKEN)) { + const token = match[0] + // Separators alone (`--`, `...`) are a token to FTS5 but never a search term. + if (/[\p{L}\p{N}\p{Co}]/u.test(token)) { + out.push(token) + if (out.length >= limit) { + break + } + } + } + return out +} + +/** + * `literal` overrides the shape test. Typo repair re-plans the query it + * corrected, and a corrected spelling can look like ordinary prose even though + * what was typed was a literal: `parseJsonn(the, data)` has the punctuation that + * makes it literal, `parsejson the data` does not. Without the override the + * re-plan would drop `the` as a stop word, so the repaired query would search + * for less than the original asked for and `repairedTerms` would report a body + * the user never typed. + */ +export function planSessionSearchQuery( + query: string, + literal = isLiteralQuery(query) +): SessionSearchQueryPlan { + // One past the cap, so the plan can tell a query that just fits from one that + // was cut. `indexTokens` stops at its limit, so it cannot be asked afterwards. + const overCap = indexTokens(query, MAX_BODY_TERMS + 1) + const truncated = overCap.length > MAX_BODY_TERMS + const raw = overCap.slice(0, MAX_BODY_TERMS) + let body = literal ? raw : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase())) + if (body.length < 2) { + body = raw + } + const terms = [...new Set(body)] + const extra: string[] = [] + for (const term of terms) { + for (const piece of identifierShadowTerms(term, 12)) { + if (!terms.includes(piece) && !STOP_WORDS.has(piece) && !extra.includes(piece)) { + extra.push(piece) + } + } + } + return { + literal, + truncated, + terms: [...terms, ...extra].slice(0, MAX_TERMS), + body: body.slice(0, MAX_BODY_TERMS) + } +} + +// Why: `cli.mjs`, `foo-bar`, and `C++` are all FTS5 syntax errors unquoted. +export function quoteFtsTerm(term: string): string { + return `"${term.replaceAll('"', '""')}"` +} + +export function phraseExpression(terms: readonly string[]): string { + return quoteFtsTerm(terms.join(' ')) +} + +export function andExpression(terms: readonly string[]): string { + return terms.map(quoteFtsTerm).join(' AND ') +} + +export function orExpression(terms: readonly string[]): string { + return terms.map(quoteFtsTerm).join(' OR ') +} + +/** + * What a scope is, now that there is one FTS table. + * + * `conversation` used to be a second table holding a copy of the two prose + * columns. It is a column filter instead: PR 2 measured the filter at + * 1.16-1.36x the p95 of the dedicated table on a 105 MB corpus, against a 2x + * bar, and the table cost a tenth of the index to maintain. + * + * It lives beside the other expression builders, and not with the retrieval + * that uses it, because the typo repair has to ask the same question of the + * same scope and importing it from there is a cycle. + * + * The filter binds to the whole expression, so it is applied here and nowhere + * else — `{cols}: (a AND b)` filters both terms, while a prefix pasted in front + * of a bare `a AND b` would filter only `a` and quietly search tool output for + * the rest. + */ +const CONVERSATION_COLUMNS = '{user_text assistant_text}' + +export function scopedExpression(scope: SessionSearchScope, expression: string): string { + return scope === 'all' ? expression : `${CONVERSATION_COLUMNS}: (${expression})` +} diff --git a/src/main/ai-vault-search/session-search-query-schema.ts b/src/main/ai-vault-search/session-search-query-schema.ts new file mode 100644 index 00000000000..f17b290e530 --- /dev/null +++ b/src/main/ai-vault-search/session-search-query-schema.ts @@ -0,0 +1,79 @@ +import type SyncDatabase from '../sqlite/sync-database' +import { + SESSION_SEARCH_GENERATION_SQL, + SESSION_SEARCH_GENERATION_TRIGGERS +} from './session-search-index-generation' + +/** An engine feature the index on disk cannot serve. */ +export type SessionSearchUnavailableFeature = 'typo-repair' + +const QUERY_SCHEMA_SQL = ` +-- The typo repair's whole dictionary. Why the index's own vocabulary and not a +-- word list: it can never suggest a term this index does not hold, and it needs +-- no model. fts5vocab is a view over the FTS5 b-tree, so it costs no extra rows. +CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row'); +-- Locally logged queries, stored as typed, bounded. Nothing writes here unless a +-- caller opts in; the eval set is rebuilt from it (see session-search-query-log). +CREATE TABLE IF NOT EXISTS search_log( + id INTEGER PRIMARY KEY, + ts TEXT NOT NULL, + query TEXT NOT NULL, + route TEXT NOT NULL, + hits INTEGER NOT NULL, + duration_ms REAL NOT NULL +); +${SESSION_SEARCH_GENERATION_SQL}` + +/** Everything the SQL above creates, so a missing one is what triggers a re-run. */ +const OWNED = ['messages_vocab', 'search_log', ...SESSION_SEARCH_GENERATION_TRIGGERS] + +/** + * The vocabulary's target. Creating a fts5vocab table over a missing FTS table + * succeeds and every query against it then fails, so the feature's health is + * this name's presence rather than the vocabulary's own. + */ +const VOCABULARY_SOURCE = 'messages_fts' + +const PROBED = [...OWNED, VOCABULARY_SOURCE] + +/** + * Creates whatever of the engine's own schema is missing, and reports what it + * still cannot serve. + * + * These objects are the query engine's, not the store's. Nothing on the write + * path reads any of them, so under the stack's YAGNI rule they do not belong in + * PR 2's schema, and an index built by a process that never opens an engine + * carries none of their cost. None of them needs a schema version either: every + * one is derived from what PR 2 already holds, so re-creating them over any of + * its files is correct, while a version bump would throw a whole index away to + * add a view over its own b-tree. + * + * Run per search, not once per engine. A capability is a fact about the file + * rather than about this object: another handle can rebuild the index under a + * live connection, so a verdict taken in a constructor is wrong for the rest of + * the engine's life in both directions — it would keep reaching for a table + * that went away and never pick one back up when it returned. The steady-state + * cost is the single indexed `sqlite_master` lookup below. + * + * A create that throws is not caught. The only way to reach one is an index + * whose `files` table is gone, which is a rebuild in flight — and an engine + * over that cannot report a hit's source either, so there is nothing to degrade + * to. Losing only the vocabulary's source is the case worth surviving, and that + * one is reported rather than thrown. + */ +export function ensureSessionSearchQuerySchema( + db: SyncDatabase +): readonly SessionSearchUnavailableFeature[] { + const present = presentNames(db) + if (OWNED.some((name) => !present.has(name))) { + db.exec(QUERY_SCHEMA_SQL) + } + return present.has(VOCABULARY_SOURCE) ? [] : ['typo-repair'] +} + +function presentNames(db: SyncDatabase): Set { + const rows = db + .prepare(`SELECT name FROM sqlite_master WHERE name IN (${PROBED.map(() => '?').join(',')})`) + .all(...PROBED) as { name: string }[] + return new Set(rows.map((row) => row.name)) +} diff --git a/src/main/ai-vault-search/session-search-retrieval.ts b/src/main/ai-vault-search/session-search-retrieval.ts new file mode 100644 index 00000000000..2bbac5d3e4e --- /dev/null +++ b/src/main/ai-vault-search/session-search-retrieval.ts @@ -0,0 +1,251 @@ +import type SyncDatabase from '../sqlite/sync-database' +import type { SessionSearchRoute, SessionSearchScope } from './session-search-engine-types' +import type { MessageRow, SessionRow } from './session-search-hit-ranking' +import { + andExpression, + orExpression, + phraseExpression, + planSessionSearchQuery, + scopedExpression, + type SessionSearchQueryPlan +} from './session-search-query-planner' +import type { SessionRowFilter } from './session-search-row-filter' +import { SessionSearchTypoRepair } from './session-search-typo-repair' + +// The operator-only walk: rows per page, and how far past a full candidate set +// it will read before giving up on finding more matches. +const RECENT_PAGE_ROWS = 512 +// Ids per `loadSessions` statement, with room to spare for the filter's own +// bound values beside them. +const SESSION_ID_BATCH = 500 +const RECENT_SCAN_FACTOR = 20 + +// Measured: user 3 / assistant 2 / tool 1 / identifiers 1 (MRR 0.503 vs 0.475 flat). +const FULL_WEIGHTS = '3.0, 2.0, 1.0, 1.0' +// The conversation scope zeroes the two columns its filter already excludes. +// Measured, and stated because it is easy to over-read: these zeros change no +// score. FTS5's bm25 sums over the columns the query matched, and the filter +// has already kept the match out of those two, so the same rows come back with +// `1.0, 1.0` here. They are a statement of what the scope means, not the fence +// that enforces it — `scopedExpression` is the fence. +const CONVERSATION_WEIGHTS = '3.0, 2.0, 0.0, 0.0' + +export type RetrievalScope = { + scope: SessionSearchScope + sort: 'relevance' | 'newest' + filter: SessionRowFilter + /** + * `repo:` / `path:`, which SQL cannot express. Applied over retrieved rows; + * see session-search-row-filter for why it cannot be pushed down. + */ + matchesOperators: (session: SessionRow) => boolean + /** + * Sessions retrieved before ranking cuts the page. See + * docs/reference/agent-session-search-query-tuning.md for the measurements + * behind the default; it is an option because the right value depends on how + * large an index is and no single number is right for every host. + */ + candidateLimit: number +} + +export type Retrieved = { + rows: MessageRow[] + route: SessionSearchRoute + /** The plan the rows were actually retrieved by; snippets highlight from it. */ + plan: SessionSearchQueryPlan + repairedTerms?: string[] +} + +/** + * The bm25 weights a scope ranks with. The conversation pair stays here rather + * than beside `scopedExpression`, because weights are a property of this SQL + * and nothing else asks for them. + */ +export function scopedWeights(scope: SessionSearchScope): string { + return scope === 'all' ? FULL_WEIGHTS : CONVERSATION_WEIGHTS +} + +/** The FTS half of a search: the route ladder and the SQL each rung runs. */ +export class SessionSearchRetrieval { + /** Null when this index has no vocabulary to repair against; the rung is skipped. */ + private readonly typoRepair: SessionSearchTypoRepair | null + + constructor( + private readonly db: SyncDatabase, + canRepairTypos = true + ) { + this.typoRepair = canRepairTypos ? new SessionSearchTypoRepair(db) : null + } + + /** + * The route ladder: phrase, then AND for a literal-looking query, then typo + * repair, then OR. + * + * Repair runs before the OR fallback rather than after it fails. A typo next + * to a common word would otherwise be masked: the common word alone retrieves + * plenty of rows over OR, so nothing would ever look like a miss worth + * repairing. + */ + run(plan: SessionSearchQueryPlan, scope: RetrievalScope): Retrieved { + const exact = this.literal(plan, scope) + if (exact) { + return { ...exact, plan } + } + const repaired = this.repair(plan, scope.scope) + const effective = repaired ?? plan + const literal = repaired ? this.literal(repaired, scope) : null + const found = literal ?? { + rows: this.match(orExpression(effective.terms), scope), + route: 'or' as const + } + return { + rows: found.rows, + route: repaired ? (`typo+${found.route}` as SessionSearchRoute) : found.route, + plan: effective, + ...(repaired ? { repairedTerms: repaired.body } : {}) + } + } + + /** + * Newest sessions the constraints allow: what an operator-only query names. + * + * Walked in pages rather than taken in one `LIMIT`, because the operators are + * applied in JS. A single cut of the newest N would hand ranking whatever + * happened to be recent and then throw most of it away, so `repo:x` on a busy + * index could answer with nothing while plenty matched. The walk is bounded + * both ways: it stops at a full candidate set, and at a ceiling on rows read. + */ + recent(scope: RetrievalScope): { sessions: SessionRow[]; incomplete: boolean } { + const { conditions, values } = scope.filter + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' + const page = this.db.prepare( + `SELECT * FROM sessions ${where} + ORDER BY updated_at DESC, id DESC LIMIT ? OFFSET ?` + ) + const ceiling = scope.candidateLimit * RECENT_SCAN_FACTOR + const sessions: SessionRow[] = [] + let scanned = 0 + // Why the flag and not a count: both caps mean the same thing to a caller — + // a session it never saw may have matched — and only the loop knows which + // of them ended it. Reporting rows read instead let the engine infer + // completeness from a full candidate set alone, so giving up at the ceiling + // with nothing found looked exactly like a search that found nothing. + let incomplete = false + while (sessions.length < scope.candidateLimit) { + if (scanned >= ceiling) { + incomplete = true + break + } + const rows = page.all(...values, RECENT_PAGE_ROWS, scanned) as SessionRow[] + if (rows.length === 0) { + break + } + scanned += rows.length + for (const row of rows) { + if (sessions.length < scope.candidateLimit && scope.matchesOperators(row)) { + sessions.push(row) + } + } + } + return { sessions, incomplete: incomplete || sessions.length >= scope.candidateLimit } + } + + /** + * Read in batches, because the id list is as long as the candidate limit and + * every id is a bound parameter, so a single statement scales with a knob the + * tuning doc invites a host to raise. + * + * Not a fix for a reachable failure, and worth saying so: SQLite has bound + * `SQLITE_MAX_VARIABLE_NUMBER` at 32,766 since 3.32, every runtime this stack + * supports is past that, and the measured limit on this one is higher still. + * A candidate limit that large is not a configuration anyone would choose. + * The batch is here so the ceiling belongs to this file rather than to + * whichever SQLite the process happened to link. + */ + loadSessions(ids: readonly number[], scope: RetrievalScope): SessionRow[] { + const rows: SessionRow[] = [] + for (let start = 0; start < ids.length; start += SESSION_ID_BATCH) { + const batch = ids.slice(start, start + SESSION_ID_BATCH) + const conditions = [`id IN (${batch.map(() => '?').join(',')})`, ...scope.filter.conditions] + rows.push( + ...(this.db + .prepare(`SELECT * FROM sessions WHERE ${conditions.join(' AND ')}`) + .all(...batch, ...scope.filter.values) as SessionRow[]) + ) + } + return rows.filter((row) => scope.matchesOperators(row)) + } + + private repair( + plan: SessionSearchQueryPlan, + scope: SessionSearchScope + ): SessionSearchQueryPlan | null { + if (!this.typoRepair) { + return null + } + const typoRepair = this.typoRepair + let changed = false + const body = plan.body.map((term) => { + // Repaired inside the scope the search will run in, so a spelling only + // tool output carries neither suppresses a repair nor becomes one. + const fix = typoRepair.correct(term, scope) + if (fix && fix !== term.toLowerCase()) { + changed = true + return fix + } + return term + }) + // The repair changes spellings, not the query's character: the re-plan is + // told what the original decided so a corrected literal keeps every term it + // was typed with. + return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null + } + + /** Phrase, then AND, for literal-looking queries; null when neither matches. */ + private literal( + plan: SessionSearchQueryPlan, + scope: RetrievalScope + ): { rows: MessageRow[]; route: 'phrase' | 'and' } | null { + if (!plan.literal || plan.body.length === 0) { + return null + } + // A one-token literal (`resolveTerminalPath`, `src/a/b.ts`) is its own + // phrase: the tokenizer keeps it whole, so the exact token is the cheap, + // precise first try before the identifier pieces fan out over OR. + const phrase = this.match(phraseExpression(plan.body), scope) + if (phrase.length > 0) { + return { rows: phrase, route: 'phrase' } + } + if (plan.body.length < 2) { + return null + } + const and = this.match(andExpression(plan.body), scope) + return and.length > 0 ? { rows: and, route: 'and' } : null + } + + private match(expression: string, scope: RetrievalScope): MessageRow[] { + const { filter, sort, candidateLimit } = scope + const eligible = filter.conditions.length + ? ` AND m.session_row_id IN (SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')})` + : '' + const matched = `SELECT messages_fts.rowid AS rowid, + -bm25(messages_fts, ${scopedWeights(scope.scope)}) AS score, + m.session_row_id, m.role, m.ts, s.updated_at + FROM messages_fts JOIN messages m ON m.id = messages_fts.rowid + JOIN sessions s ON s.id = m.session_row_id WHERE messages_fts MATCH ?${eligible}` + // Why: collapse to one row per session BEFORE the candidate limit, on both + // sort orders, so a single long session cannot occupy the whole page. + // `max(score)` makes SQLite pick that session's best row for the bare columns. + // Cost of grouping instead of a bounded top-N sorter, measured: ~1.75x + // (49.6 vs 28.6 ms at 80k matching rows, 183.6 vs 104.1 ms at 240k) and a + // temp b-tree over every match. No inner LIMIT can bound it: the CTE has no + // order, so any cut drops whole sessions rather than their surplus rows. + const order = sort === 'newest' ? 'updated_at DESC, score DESC' : 'score DESC' + const sql = `WITH matched AS MATERIALIZED (${matched}) + SELECT rowid, max(score) AS score, session_row_id, role, ts FROM matched + GROUP BY session_row_id ORDER BY ${order} LIMIT ${candidateLimit}` + return this.db + .prepare(sql) + .all(scopedExpression(scope.scope, expression), ...filter.values) as MessageRow[] + } +} diff --git a/src/main/ai-vault-search/session-search-row-filter.test.ts b/src/main/ai-vault-search/session-search-row-filter.test.ts new file mode 100644 index 00000000000..dd7ca0d1e5c --- /dev/null +++ b/src/main/ai-vault-search/session-search-row-filter.test.ts @@ -0,0 +1,137 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type SyncDatabase from '../sqlite/sync-database' +import type { SessionSearchFilters } from './session-search-engine-types' +import { cwdKey } from './session-search-file-records' +import { sessionRowFilter } from './session-search-row-filter' +import { + openSessionSearchIndexFile, + type SessionSearchIndexFile +} from './session-search-index-test-fixture' + +let index: SessionSearchIndexFile | null = null + +afterEach(async () => { + await index?.close() + index = null +}) + +async function openIndex(): Promise { + index = await openSessionSearchIndexFile('ss-row-filter') + return index.db +} + +function addSession( + db: SyncDatabase, + id: number, + cwd: string | null, + overrides: { agent?: string; updatedAt?: string } = {} +): void { + db.prepare( + `INSERT INTO sessions(id,agent,session_id,file_path,title,cwd,cwd_key,updated_at,resume_command) + VALUES (?,?,?,?,'fixture',?,?,?,'')` + ).run( + id, + overrides.agent ?? 'claude', + String(id), + `/synthetic/${id}`, + cwd, + cwdKey(cwd), + overrides.updatedAt ?? '2026-09-01T00:00:00.000Z' + ) +} + +function selected(db: SyncDatabase, filters: SessionSearchFilters = {}): number[] { + const filter = sessionRowFilter(filters) + const where = filter.conditions.length > 0 ? `WHERE ${filter.conditions.join(' AND ')}` : '' + return ( + db.prepare(`SELECT id FROM sessions ${where} ORDER BY id`).all(...filter.values) as { + id: number + }[] + ).map((row) => row.id) +} + +describe('a cwd scope is the sidebar key, or anything below it', () => { + it.each([ + ['C:\\Work\\App', 'c:/work/app', true], + ['C:\\Work\\App\\src', 'c:/work/app', true], + ['/work/APP/src', '/work/app', false], + ['/work/caf\u00e9', '/work/cafe\u0301', true], + ['/work/app-other', '/work/app', false], + ['/work/a_b/src', '/work/a_b', true], + ['/work/axb/src', '/work/a_b', false], + // Roots: `/` is the one key that is already a separator, which is where a + // range bound is easiest to get wrong. A Windows key is not under POSIX `/`. + ['/', '/', true], + ['/work/app', '/', true], + ['C:\\Work\\App', '/', false], + ['C:\\', 'C:\\', true], + ['C:\\Work\\App', 'C:\\', true] + ])('scopes %s under %s: %s', async (cwd, scope, expected) => { + const db = await openIndex() + addSession(db, 1, cwd) + expect(selected(db, { scopePaths: [scope] })).toEqual(expected ? [1] : []) + }) + + it('never matches a session whose transcript recorded no cwd', async () => { + const db = await openIndex() + addSession(db, 1, null) + expect(selected(db, { scopePaths: ['/work'] })).toEqual([]) + expect(selected(db)).toEqual([1]) + }) + + it('keeps a WSL UNC workspace distinct from the bare Linux spelling', async () => { + // PR 2 decided cwd_key does not qualify a Linux path with its distro: the + // collision is real but every SSH host has it too, and the fix is a column + // naming the execution host, not a key only some hosts spell differently. + const db = await openIndex() + addSession(db, 1, '\\\\wsl.localhost\\Ubuntu\\home\\ada\\app') + addSession(db, 2, '/home/ada/app') + expect(selected(db, { scopePaths: ['\\\\wsl$\\Ubuntu\\home\\ada'] })).toEqual([1]) + expect(selected(db, { scopePaths: ['/home/ada/app'] })).toEqual([2]) + expect(selected(db, { scopePaths: ['\\\\wsl$\\Debian\\home\\ada\\app'] })).toEqual([]) + }) +}) + +describe('caller filters', () => { + it('narrows by agent, and by updated-at floor', async () => { + const db = await openIndex() + addSession(db, 1, '/work/app', { agent: 'claude', updatedAt: '2026-09-01T00:00:00.000Z' }) + addSession(db, 2, '/work/app', { agent: 'codex', updatedAt: '2026-09-05T00:00:00.000Z' }) + expect(selected(db, { agents: ['codex'] })).toEqual([2]) + expect(selected(db, { since: '2026-09-03T00:00:00.000Z' })).toEqual([2]) + expect(selected(db, { agents: ['claude'], since: '2026-09-03T00:00:00.000Z' })).toEqual([]) + }) + + it('applies the retention cutoff through the files table', async () => { + const db = await openIndex() + addSession(db, 1, '/work/app') + addSession(db, 2, '/work/app') + db.prepare( + "INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES ('a',0,100,1)" + ).run() + db.prepare( + "INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES ('b',0,500,2)" + ).run() + const filter = sessionRowFilter({}, 300) + const rows = db + .prepare(`SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')}`) + .all(...filter.values) as { id: number }[] + expect(rows.map((row) => row.id)).toEqual([2]) + }) +}) + +it('plans a cwd scope as a seek on sessions_cwd_key, never a scan', async () => { + const db = await openIndex() + const filter = sessionRowFilter({ scopePaths: ['/work/app'] }) + const plan = ( + db + .prepare( + `EXPLAIN QUERY PLAN SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')}` + ) + .all(...filter.values) as { detail: string }[] + ).map((row) => row.detail) + + expect(plan.join(' | ')).toContain('sessions_cwd_key') + expect(plan.some((detail) => detail.startsWith('SEARCH'))).toBe(true) + expect(plan.some((detail) => detail.startsWith('SCAN sessions'))).toBe(false) +}) diff --git a/src/main/ai-vault-search/session-search-row-filter.ts b/src/main/ai-vault-search/session-search-row-filter.ts new file mode 100644 index 00000000000..5015fa460e6 --- /dev/null +++ b/src/main/ai-vault-search/session-search-row-filter.ts @@ -0,0 +1,90 @@ +import { cwdKey } from './session-search-file-records' +import type { SessionSearchFilters } from './session-search-engine-types' + +/** SQL fragments for the `sessions` WHERE clause; every condition is ANDed. */ +export type SessionRowFilter = { + conditions: string[] + values: (string | number)[] +} + +// Stored identity: `cwdKey` is the sidebar's `folderGroupKey` without its prefix, +// so a scope term and an indexed session are keyed by one function, never two. +const CWD = 'cwd_key' + +/** + * The narrowings SQL can express exactly, in one place, so retrieval, the + * operator-only page and the session load cannot drift apart. These conditions + * run over `sessions` itself. Reachability is not here and is not a condition: + * it is the INNER JOIN to `sessions` that every retrieval carries, which is + * what makes a message row a purge has not reclaimed yet unreadable. + * + * `repo:` and `path:` are deliberately absent. What they mean is the predicate + * the sessions panel applies (`matchesAiVaultQueryOperators`), and SQL cannot + * express it: LIKE folds ASCII and nothing else, so `path:CAFÉ` would miss + * `café`; `path:` searches the transcript path as well as the working + * directory, so `path:jsonl` would miss every session; and `repo:` compares the + * last two path segments, not one. A second spelling that came close would be a + * query meaning different things in the list and in the index, so the engine + * applies the panel's own predicate over the rows it retrieves instead. + * + * `scopePaths` stays here because it is exact: a prefix range over the key + * `cwdKey` produces, which folds exactly where the execution host folds — + * Windows drives, never a POSIX directory name. + */ +export function sessionRowFilter( + filters: SessionSearchFilters, + cutoffMs: number | null = null +): SessionRowFilter { + const filter: SessionRowFilter = { conditions: [], values: [] } + if (cutoffMs !== null) { + filter.conditions.push('id IN (SELECT session_row_id FROM files WHERE mtime_ms >= ?)') + filter.values.push(cutoffMs) + } + if (filters.agents && filters.agents.length > 0) { + filter.conditions.push(`agent IN (${filters.agents.map(() => '?').join(',')})`) + filter.values.push(...filters.agents) + } + if (filters.since) { + filter.conditions.push('updated_at >= ?') + filter.values.push(filters.since) + } + if (filters.scopePaths && filters.scopePaths.length > 0) { + // Several scopes mean any of them; every other narrowing is ANDed on. + const present = filters.scopePaths + .map((scope) => scopeCondition(filter, scope)) + .filter((condition) => condition !== null) + if (present.length > 0) { + filter.conditions.push(`(${present.join(' OR ')})`) + } + } + return filter +} + +/** A scope the caller could not key is a scope nothing is inside of. */ +function scopeCondition(filter: SessionRowFilter, scope: string): string | null { + const key = cwdKey(scope) + return key === null ? null : insideCondition(filter, key) +} + +/** + * `key` itself, or anything below it. Why a half-open range and not + * `substr(key, 1, length(?)) = ?`: only `>=`/`<` can seek `sessions_cwd_key`; + * the substr form scans it. The bound is the child prefix with its last byte + * incremented, so it stops at the end of that prefix and nowhere else. The two + * arms cannot merge: one range over the bare key would also swallow a sibling + * like `/work/app-other`. No wildcards, so `%`/`_` in a folder name are literal. + * + * The filesystem root is the one key that already ends in a separator, and + * appending a second one would bound the range at `//`, which sorts below every + * real child; `cwdKey` keeps it as `/` for exactly this reason. + */ +function insideCondition(filter: SessionRowFilter, key: string): string { + const children = key.endsWith('/') ? key : `${key}/` + filter.values.push(key, children, nextAfterPrefix(children)) + return `(${CWD} = ? OR (${CWD} >= ? AND ${CWD} < ?))` +} + +/** The first string that sorts after every string starting with `prefix`. */ +function nextAfterPrefix(prefix: string): string { + return prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1) +} diff --git a/src/main/ai-vault-search/session-search-sidebar-parity.test.ts b/src/main/ai-vault-search/session-search-sidebar-parity.test.ts new file mode 100644 index 00000000000..081b16479ed --- /dev/null +++ b/src/main/ai-vault-search/session-search-sidebar-parity.test.ts @@ -0,0 +1,137 @@ +import { afterEach, expect, it } from 'vitest' +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { filterAiVaultSessions } from '../../shared/ai-vault-session-filters' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' + +// `repo:` and `path:` have to mean one thing. The sessions panel and the index +// answer from different stores by different mechanisms, so the only way to keep +// them equal is for both to run the same predicate; this asserts they do, over +// the shapes where a second SQL spelling went wrong. + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +type Fixture = { id: number; cwd: string; filePath: string; text: string } + +const SESSIONS: Fixture[] = [ + { + id: 1, + cwd: '/Users/Ada/orca/session-search', + filePath: '/Users/Ada/.claude/projects/a/one.jsonl', + text: 'harbor pilot manifest' + }, + { + id: 2, + cwd: '/Users/ada/work/café', + filePath: '/Users/ada/.codex/sessions/two.jsonl', + text: 'harbor dock crane' + }, + { + id: 3, + cwd: '/srv/other/service', + filePath: '/srv/.claude/projects/b/three.jsonl', + text: 'harbor manifest beta' + }, + { + id: 4, + cwd: 'C:\\Work\\Orca\\App', + filePath: 'C:\\Users\\Ada\\.claude\\four.jsonl', + text: 'harbor windows lane' + } +] + +// Each of these matched in the panel and missed in the index while the engine +// tried to say `repo:` / `path:` in SQL. +const QUERIES = [ + 'harbor path:jsonl', + 'harbor repo:orca/session-search', + 'harbor path:CAFÉ', + 'harbor path:/Users/Ada/orca', + 'harbor repo:app', + 'harbor repo:Orca/App', + 'harbor path:.codex', + 'harbor path:/srv repo:other/service', + 'harbor repo:session-search path:jsonl', + 'harbor path:"/Users/ada/work"', + 'harbor repo:nothing-here', + 'harbor path:one.jsonl path:two.jsonl', + 'harbor' +] + +function asSession(fixture: Fixture): AiVaultSession { + const at = '2026-09-01T00:00:00.000Z' + return { + id: String(fixture.id), + executionHostId: 'local', + agent: 'claude', + sessionId: String(fixture.id), + title: 'fixture', + cwd: fixture.cwd, + branch: null, + model: null, + filePath: fixture.filePath, + codexHome: null, + createdAt: at, + updatedAt: at, + modifiedAt: at, + messageCount: 1, + totalTokens: 0, + previewMessages: [{ role: 'user', text: fixture.text }], + queuedMessageCount: 0, + subagentTranscriptCount: 0, + resumeCommand: '', + subagent: null + } as AiVaultSession +} + +/** The panel's own answer, operators only: free text is FTS in the index. */ +function sidebarIds(query: string): string[] { + const operatorsOnly = query + .split(/\s+/) + .filter((token) => /^(repo|path):/i.test(token)) + .join(' ') + return filterAiVaultSessions(SESSIONS.map(asSession), { + query: operatorsOnly, + agents: ['claude'], + scope: 'all', + sort: 'updated', + activeWorktreePaths: [], + hideEmptySessions: false + }) + .map((session) => session.sessionId) + .sort() +} + +it.each(QUERIES)('answers %s the way the sessions panel does', async (query) => { + harness = await openSessionSearchHarness('ss-sidebar-parity') + for (const fixture of SESSIONS) { + addSyntheticSession(harness.db, { + id: fixture.id, + cwd: fixture.cwd, + text: fixture.text, + filePath: fixture.filePath, + sessionFilePath: fixture.filePath + }) + } + const engineIds = harness.engine + .search({ query, limit: 100 }) + .hits.map((hit) => hit.sessionId) + .sort() + expect(engineIds).toEqual(sidebarIds(query)) +}) + +it('is not vacuous: these queries do select, and reject, real sessions', () => { + // A parity suite where every query matched everything, or nothing, would pass + // against any predicate at all. + const answers = QUERIES.map((query) => sidebarIds(query).length) + expect(answers.some((count) => count > 0 && count < SESSIONS.length)).toBe(true) + expect(answers.some((count) => count === 0)).toBe(true) +}) diff --git a/src/main/ai-vault-search/session-search-snippet-marks.test.ts b/src/main/ai-vault-search/session-search-snippet-marks.test.ts new file mode 100644 index 00000000000..71704b78fee --- /dev/null +++ b/src/main/ai-vault-search/session-search-snippet-marks.test.ts @@ -0,0 +1,112 @@ +import { afterEach, expect, it } from 'vitest' +import { + SESSION_SEARCH_SNIPPET_MARK_CLOSE, + SESSION_SEARCH_SNIPPET_MARK_OPEN +} from './session-search-engine-types' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' + +// A snippet has to name which of a row's four columns matched, and the marks +// FTS5 wraps a match in are the only signal. Searching the marked text for the +// public `[[` reads a transcript's own brackets as a highlight — and transcripts +// are full of them, because a bash `[[ -f x ]]` and numpy's `[[1, 2]]` are +// exactly the sort of thing an agent session holds. Whether a column matched is +// the difference between two renderings of the same text instead. + +let harness: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + harness = null +}) + +const BASH = 'run this: if [[ -f /home/me/.aws/credentials ]]; then cat it; fi' +const TOOL = 'zebrafish appears only in the tool output here' + +it('shows the column that matched, not the one that happens to contain brackets', async () => { + harness = await openSessionSearchHarness('ss-snippet-marks') + // Session 1's match is in tool output while its user turn holds a bash test + // expression; session 2 is the same match with no brackets anywhere. + addSyntheticSession(harness.db, { id: 1, text: BASH, toolText: TOOL }) + addSyntheticSession(harness.db, { id: 2, text: 'run this script please', toolText: TOOL }) + + const hits = harness.engine.search({ query: 'zebrafish' }).hits + expect(hits).toHaveLength(2) + for (const hit of hits) { + expect(hit.evidence?.snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebrafish${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) + expect(hit.evidence?.snippet).not.toContain('credentials') + } +}) + +it('falls back to any column for an identifier-only match, brackets or not', async () => { + // `zebra` reaches this row only through the identifier shadow column, which is + // what column -1 exists for. The user turn holds numpy output, so a bracket + // scan would have stopped at it and shown a column with no match in it. + harness = await openSessionSearchHarness('ss-snippet-marks-fallback') + addSyntheticSession(harness.db, { + id: 1, + text: 'numpy printed [[1, 2], [3, 4]] before the call', + toolText: 'zebra-fish-count = 4' + }) + + const [hit] = harness.engine.search({ query: 'zebra' }).hits + expect(hit?.evidence?.snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebra${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) + expect(hit?.evidence?.snippet).not.toContain('numpy') +}) + +it('leaves a transcript’s own brackets in the text it shows', async () => { + // The marks are rewritten from private-use code points at the very end, so a + // row that both matches and contains `[[` keeps its own characters. + harness = await openSessionSearchHarness('ss-snippet-marks-literal') + addSyntheticSession(harness.db, { id: 1, text: `zebrafish ${BASH}` }) + + const [hit] = harness.engine.search({ query: 'zebrafish' }).hits + expect(hit?.evidence?.snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebrafish${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) + expect(hit?.evidence?.snippet).toContain('[[ -f') +}) + +it('picks by comparison, so a private-use code point in content cannot pose as a mark', async () => { + // The marks are private-use code points, and a transcript may hold one: + // agent output carries Nerd Font glyphs, which live in the same block. So the + // column is chosen by comparing a marked rendering against an unmarked one, + // not by looking for a mark in the text. + harness = await openSessionSearchHarness('ss-snippet-marks-private-use') + addSyntheticSession(harness.db, { + id: 1, + text: 'the \uE000 glyph a font printed here', + toolText: TOOL + }) + + const [hit] = harness.engine.search({ query: 'zebrafish' }).hits + expect(hit?.evidence?.snippet).toContain('zebrafish') + expect(hit?.evidence?.snippet).not.toContain('glyph') +}) + +it('truncates on the last real mark, not on a bracket the transcript wrote', async () => { + // Over the character ceiling the snippet is cut, and it must not cut between + // an open mark and its close. Finding that open mark by searching for `[[` + // stops at the transcript's own bracket instead and throws away everything + // after it. + harness = await openSessionSearchHarness('ss-snippet-marks-truncation') + const long = (letter: string): string => + Array.from({ length: 5 }, () => `${letter.repeat(55)}/tail`).join(' ') + addSyntheticSession(harness.db, { + id: 1, + text: `zebrafish ${long('p')} [[ ${long('q')}` + }) + + const snippet = harness.engine.search({ query: 'zebrafish' }).hits[0]?.evidence?.snippet ?? '' + expect(snippet).toContain('[[zebrafish]]') + // The cut is the character ceiling, so the text after the transcript's own + // bracket survives up to it. + expect(snippet).toContain('qqqqq') +}) diff --git a/src/main/ai-vault-search/session-search-snippet.ts b/src/main/ai-vault-search/session-search-snippet.ts new file mode 100644 index 00000000000..2e23e707dd0 --- /dev/null +++ b/src/main/ai-vault-search/session-search-snippet.ts @@ -0,0 +1,126 @@ +import type SyncDatabase from '../sqlite/sync-database' +import { + SESSION_SEARCH_SNIPPET_MARK_CLOSE, + SESSION_SEARCH_SNIPPET_MARK_OPEN +} from './session-search-engine-types' +import { + orExpression, + scopedExpression, + type SessionSearchQueryPlan +} from './session-search-query-planner' +import type { SessionSearchScope } from './session-search-engine-types' + +// What FTS5 wraps a match in before this module rewrites it to the public +// marks. Private-use code points, and not `[[`, because two different jobs here +// have to tell a mark from content: choosing the column to show, and refusing +// to cut a snippet between an open mark and its close. Transcripts contain +// `[[` — a bash `[[ -f x ]]`, numpy's `[[1, 2]]` — and a mark the content can +// forge makes both of those decisions wrong on real text. +const MARK_OPEN = '\uE000' +const MARK_CLOSE = '\uE001' + +const SNIPPET_TOKENS = 12 +// Why a ceiling on top of the token count: a transcript chunk can be 8000 +// characters with no separator in it, which FTS5 reports as one token, so +// "twelve tokens" is not by itself a bound on what a hit carries. +const SNIPPET_MAX_CHARS = 512 + +export type SessionSearchSnippet = { + text: string + truncated: boolean +} + +export const EMPTY_SNIPPET: SessionSearchSnippet = { text: '', truncated: false } + +/** + * The window of one message that shows why it matched. + * + * The expression is the plan's OR form rather than the route's, so a hit found + * through typo repair is marked with the repaired terms it was actually + * retrieved by, and a phrase hit still marks each of its words. + */ +export function sessionSearchSnippet( + db: SyncDatabase, + scope: SessionSearchScope, + rowid: number, + plan: SessionSearchQueryPlan +): SessionSearchSnippet { + // Why: the identifier shadow column is word soup; a hit that also matches in a + // prose column should be shown from there. Column -1 (any column) is the + // fallback for rows that only matched through the shadow column. + // + // The same four for every scope, because the scope is already in the + // expression below. A conversation snippet cannot come out of `tool_text` for + // the reason the search could not: the row has to match + // `{user_text assistant_text}: …` before any of these columns is read, and a + // row that matches under that filter carries its mark in column 0 or 1. A + // second list here would be a guard with nothing left to guard, and the two + // would mask each other's mistakes. + const columns = [0, 1, 2, -1] + // Each column twice: once marked, once with empty marks. Whether a column + // matched is then the difference between two renderings of the same text, + // which content cannot forge — searching the marked one for a mark reads a + // transcript's own `[[` as a highlight and shows a column that matched + // nothing. + const select = columns + .flatMap((column, index) => [ + `snippet(messages_fts, ${column}, '${MARK_OPEN}', '${MARK_CLOSE}', '…', ${SNIPPET_TOKENS}) AS c${index}`, + `snippet(messages_fts, ${column}, '', '', '…', ${SNIPPET_TOKENS}) AS p${index}` + ]) + .join(', ') + try { + // Why the subselect: a bound `rowid = ?` or `rowid IN (?)` next to MATCH is + // silently ignored by the FTS5 planner, which then returns the first match + // in the table. Why the join to `sessions`: retrieval proved this rowid + // belonged to a live session, but a purge can commit between that statement + // and this one, and a message row outlives its session row until the drain + // reaches it. INNER, never LEFT — this is the last read before content is + // returned to a caller. + const row = db + .prepare( + `SELECT ${select} FROM messages_fts + JOIN messages m ON m.id = messages_fts.rowid + JOIN sessions s ON s.id = m.session_row_id + WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)` + ) + .get(scopedExpression(scope, orExpression(plan.terms)), rowid) as + | Record + | undefined + if (!row) { + return EMPTY_SNIPPET + } + // A snippet with nothing highlighted tells the user nothing; omit it. + const marked = columns + .map((_column, index) => row[`c${index}`]) + .find((text, index) => text !== undefined && text !== row[`p${index}`]) + return marked === undefined ? EMPTY_SNIPPET : publicMarks(truncateSnippet(marked)) + } catch { + return EMPTY_SNIPPET + } +} + +/** The internal marks, swapped for the ones a caller sees, once and at the end. */ +function publicMarks(snippet: SessionSearchSnippet): SessionSearchSnippet { + return { + ...snippet, + text: snippet.text + .replaceAll(MARK_OPEN, SESSION_SEARCH_SNIPPET_MARK_OPEN) + .replaceAll(MARK_CLOSE, SESSION_SEARCH_SNIPPET_MARK_CLOSE) + } +} + +/** Cut on a code-point boundary, and never between a mark and its close. */ +export function truncateSnippet(text: string): SessionSearchSnippet { + if (text.length <= SNIPPET_MAX_CHARS) { + return { text, truncated: false } + } + const points = [...text] + if (points.length <= SNIPPET_MAX_CHARS) { + return { text, truncated: false } + } + const cut = points.slice(0, SNIPPET_MAX_CHARS).join('') + const opened = cut.lastIndexOf(MARK_OPEN) + // An open mark with no close hands the renderer something it can never close. + const balanced = opened !== -1 && !cut.includes(MARK_CLOSE, opened) ? cut.slice(0, opened) : cut + return { text: balanced, truncated: true } +} diff --git a/src/main/ai-vault-search/session-search-source-presence.ts b/src/main/ai-vault-search/session-search-source-presence.ts new file mode 100644 index 00000000000..cc7155fccc8 --- /dev/null +++ b/src/main/ai-vault-search/session-search-source-presence.ts @@ -0,0 +1,40 @@ +import type SyncDatabase from '../sqlite/sync-database' +import type { SessionSearchSourcePresence } from './session-search-engine-types' + +/** + * Where each session's source stands, read from the index's own `files` table. + * + * Why not a stat: a search page of 20 hits would be 20 filesystem round trips + * on the query path, and on an SSH or WSL host each one can block for as long + * as the connection takes to answer — the reviewer's F11. The index already + * records what discovery last proved about every file it read, so the query + * path reads that instead of asking the disk again. + * + * The vocabulary is deliberately short of `missing`. A row here means the index + * holds a live file record for the session, which is `present`. No row means + * this read cannot tell whether the source is gone or merely unrecorded, and + * loss of contact is never evidence of absence + * (docs/reference/ssh-execution-boundary.md), so it is `unverifiable`. Proving + * a deletion is the indexer's job and it retires the session's rows outright. + */ +export function sessionSourcePresence( + db: SyncDatabase, + sessionRowIds: readonly number[] +): Map { + const presence = new Map( + sessionRowIds.map((id) => [id, 'unverifiable' as const]) + ) + if (sessionRowIds.length === 0) { + return presence + } + const rows = db + .prepare( + `SELECT DISTINCT session_row_id FROM files + WHERE session_row_id IN (${sessionRowIds.map(() => '?').join(',')})` + ) + .all(...sessionRowIds) as { session_row_id: number }[] + for (const row of rows) { + presence.set(row.session_row_id, 'present') + } + return presence +} diff --git a/src/main/ai-vault-search/session-search-typo-policy.test.ts b/src/main/ai-vault-search/session-search-typo-policy.test.ts new file mode 100644 index 00000000000..938c1e1fc3e --- /dev/null +++ b/src/main/ai-vault-search/session-search-typo-policy.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' +import type SyncDatabase from '../sqlite/sync-database' +import { openSessionSearchIndexFile } from './session-search-index-test-fixture' +import { ensureSessionSearchQuerySchema } from './session-search-query-schema' +import { SessionSearchTypoRepair } from './session-search-typo-repair' + +/** A session row the planted messages below hang off, so a repair can see them. */ +function addSession(db: SyncDatabase, id: number): void { + db.prepare( + `INSERT INTO sessions(id,agent,session_id,file_path,title,resume_command) + VALUES (?, 'claude', ?, '/synthetic/fixture', 'typo fixture', '')` + ).run(id, String(id)) +} + +function addTerm(db: SyncDatabase, sessionRowId: number, term: string): void { + const rowid = db + .prepare("INSERT INTO messages(session_row_id, role) VALUES (?, 'user')") + .run(sessionRowId).lastInsertRowid + db.prepare('INSERT INTO messages_fts(rowid, user_text) VALUES (?, ?)').run(Number(rowid), term) +} + +describe('typo repair policy', () => { + it.each([ + { input: 'coalesces', candidate: 'coalesced', copies: 2, exact: true, expected: null }, + { input: 'coalescs', candidate: 'coalesces', copies: 1, exact: false, expected: null }, + { input: 'coalescs', candidate: 'coalesces', copies: 2, exact: false, expected: 'coalesces' }, + { input: 'café', candidate: 'cafe', copies: 1, exact: false, expected: null }, + { input: 'car', candidate: 'cars', copies: 2, exact: false, expected: null }, + { input: 'calm', candidate: 'clam', copies: 2, exact: false, expected: null } + ])( + 'repairs $input to $expected with $copies postings (exact=$exact)', + async ({ input, candidate, copies, exact, expected }) => { + const index = await openSessionSearchIndexFile('ss-typo-policy') + try { + ensureSessionSearchQuerySchema(index.db) + addSession(index.db, 1) + for (let i = 0; i < copies; i++) { + addTerm(index.db, 1, candidate) + } + if (exact) { + addTerm(index.db, 1, input) + } + expect(new SessionSearchTypoRepair(index.db).correct(input, 'all')).toBe(expected) + } finally { + await index.close() + } + } + ) + + // A purge cuts a session loose in one transaction and reclaims its rows over + // many, so the vocabulary can still list a term whose only rows nothing can + // reach. Abandoning the prefix at that term would lose a repair the rest of + // the index can already serve. + it('falls through to the best candidate a reader can still reach', async () => { + const index = await openSessionSearchIndexFile('ss-typo-orphaned') + try { + const { db } = index + ensureSessionSearchQuerySchema(db) + addSession(db, 1) + // `coalesces` scores higher against `coalescs` than `coalesced` does, and + // shares its prefix, so only the fall-through can reach the reachable one. + // Session 2 is never created: these rows are what an unfinished purge + // leaves behind, and the vocabulary counts them all the same. + for (const [term, session] of [ + ['coalesces', 2], + ['coalesces', 2], + ['coalesced', 1], + ['coalesced', 1] + ] as const) { + addTerm(db, session, term) + } + expect(db.prepare("SELECT doc FROM messages_vocab WHERE term='coalesces'").get()).toEqual({ + doc: 2 + }) + expect(new SessionSearchTypoRepair(db).correct('coalescs', 'all')).toBe('coalesced') + } finally { + await index.close() + } + }) +}) diff --git a/src/main/ai-vault-search/session-search-typo-repair.ts b/src/main/ai-vault-search/session-search-typo-repair.ts new file mode 100644 index 00000000000..1aed90e2991 --- /dev/null +++ b/src/main/ai-vault-search/session-search-typo-repair.ts @@ -0,0 +1,163 @@ +import type SyncDatabase from '../sqlite/sync-database' +import type { SessionSearchScope } from './session-search-engine-types' +import { quoteFtsTerm, scopedExpression } from './session-search-query-planner' + +// Why: a query term with zero postings is usually a typo. The index's own +// vocabulary (fts5vocab) is the dictionary, so repair needs no model and can +// never suggest a word the index does not contain. Measured MRR 0.553 → 0.566. +const MIN_TERM_LENGTH = 4 +const MAX_TERM_LENGTH = 40 +const LENGTH_SLACK = 2 +const MIN_DOC_FREQUENCY = 2 +const MIN_SIMILARITY = 0.82 +const MAX_CANDIDATES = 4000 +// Candidates counted against live rows per prefix before giving up on it. Only +// reached for a term the scope has no posting for, which is the rare case. +const MAX_VISIBILITY_PROBES = 8 +// How far a live count walks before it stops caring. It exists to break ties +// between candidates of equal similarity, and the difference between a term in +// sixty-four rows and one in six thousand does not change which is the better +// repair — but reading either in full would. +const MAX_COUNTED_ROWS = 64 + +// Longest common subsequence length; the indel distance is len(a)+len(b)-2·LCS. +function commonSubsequenceLength(a: string, b: string): number { + let previous = Array.from({ length: b.length + 1 }).fill(0) + let current = Array.from({ length: b.length + 1 }).fill(0) + for (let i = 1; i <= a.length; i += 1) { + for (let j = 1; j <= b.length; j += 1) { + current[j] = + a.charCodeAt(i - 1) === b.charCodeAt(j - 1) + ? previous[j - 1] + 1 + : Math.max(previous[j], current[j - 1]) + } + ;[previous, current] = [current, previous] + } + return previous[b.length] +} + +/** Normalized indel similarity in [0, 1], the scale rapidfuzz's `fuzz.ratio` uses. */ +function similarity(a: string, b: string): number { + const total = a.length + b.length + return total === 0 ? 1 : (2 * commonSubsequenceLength(a, b)) / total +} + +/** + * Spelling repair over the index's own vocabulary. + * + * The vocabulary proposes and a scoped count disposes. `messages_vocab` is a + * view over the whole FTS b-tree: it has no column filter, because fts5vocab is + * per table, and it counts rows whose session a purge already cut loose. So + * every decision that reaches the plan — whether a term is already spelled + * right, whether a candidate is eligible, and which of two equally close + * candidates wins — is taken from a `messages_fts MATCH` under the same column + * filter retrieval uses, joined to `sessions`. + * + * That is not tidiness. Reading the vocabulary directly made the repair depend + * on rows the search could never return: tool output suppressed a + * conversation-scope repair and supplied suggestions the scope would never + * show, and retention's orphan drain silently changed which word a query was + * repaired to. + * + * The cost is one bounded count per candidate examined, at most + * `MAX_VISIBILITY_PROBES` per prefix, and only for a term the scope has no + * posting for. See docs/reference/agent-session-search-query-tuning.md. + */ +export class SessionSearchTypoRepair { + private readonly liveRows: ReturnType + private readonly candidatesByPrefix: ReturnType + + constructor(db: SyncDatabase) { + this.liveRows = db.prepare( + `SELECT count(*) AS rows FROM ( + SELECT m.id FROM messages_fts + JOIN messages m ON m.id = messages_fts.rowid + JOIN sessions s ON s.id = m.session_row_id + WHERE messages_fts MATCH ? LIMIT ${MAX_COUNTED_ROWS})` + ) + // fts5vocab is ordered by term, so a prefix range plus a length band is a + // bounded scan and no sort. Ordered by term rather than by `doc`: the + // ordering decides which candidates survive the limit, and `doc` counts + // rows no reader can see, so the drain reclaiming them moved the cut. + this.candidatesByPrefix = db.prepare( + `SELECT term FROM messages_vocab + WHERE term >= ? AND term < ? AND length(term) BETWEEN ? AND ? + ORDER BY term LIMIT ?` + ) + } + + /** Live rows carrying this term inside `scope`, counted no further than it matters. */ + private countRows(term: string, scope: SessionSearchScope): number { + const row = this.liveRows.get(scopedExpression(scope, quoteFtsTerm(term))) as { rows: number } + return row.rows + } + + /** Whether a live row inside `scope` holds this term. */ + hasPostings(term: string, scope: SessionSearchScope): boolean { + return this.countRows(term, scope) > 0 + } + + /** Returns the closest indexed term, or null when `term` exists or nothing is close enough. */ + correct(term: string, scope: SessionSearchScope): string | null { + const lowered = term.toLowerCase() + if (lowered.length < MIN_TERM_LENGTH || lowered.length > MAX_TERM_LENGTH) { + return null + } + if (this.hasPostings(lowered, scope)) { + return null + } + // Two-letter prefix first (a typo rarely hits both), then the transposed + // pair, then the bare first letter as the wide fallback. + const prefixes = [lowered.slice(0, 2), lowered[1] + lowered[0], lowered[0]] + for (const prefix of prefixes) { + const best = this.bestVisible(lowered, prefix, scope) + if (best) { + return best + } + } + return null + } + + /** + * The closest candidate at `prefix` that this scope can actually answer with. + * + * Ranking is pure CPU, so the walk is bounded rather than the count: the + * closest term can be one the scope never shows, and abandoning the prefix + * there would lose a repair the rest of the index can serve. Ties on + * similarity go to the more common word, which is the same prior the + * vocabulary's `doc` used to supply — counted live here so the answer does + * not move when a purge reclaims rows nothing could reach. + */ + private bestVisible(lowered: string, prefix: string, scope: SessionSearchScope): string | null { + const counted = this.ranked(lowered, prefix) + .slice(0, MAX_VISIBILITY_PROBES) + .map((candidate) => ({ ...candidate, rows: this.countRows(candidate.term, scope) })) + .filter((candidate) => candidate.rows >= MIN_DOC_FREQUENCY) + if (counted.length === 0) { + return null + } + // Already sorted by similarity; a stable sort keeps that and orders the ties. + return counted.sort((left, right) => right.score - left.score || right.rows - left.rows)[0]! + .term + } + + /** Candidates similar enough to be a repair, closest first. */ + private ranked(lowered: string, prefix: string): { term: string; score: number }[] { + return this.candidates(prefix, lowered.length) + .map((row) => ({ term: row.term, score: similarity(lowered, row.term) })) + .filter((candidate) => candidate.score >= MIN_SIMILARITY) + .sort((left, right) => right.score - left.score || (left.term < right.term ? -1 : 1)) + } + + private candidates(prefix: string, length: number): { term: string }[] { + const last = prefix.charCodeAt(prefix.length - 1) + const upper = prefix.slice(0, -1) + String.fromCharCode(last + 1) + return this.candidatesByPrefix.all( + prefix, + upper, + Math.max(MIN_TERM_LENGTH - 1, length - LENGTH_SLACK), + length + LENGTH_SLACK, + MAX_CANDIDATES + ) as { term: string }[] + } +} diff --git a/src/main/ai-vault-search/session-search-typo-scope.test.ts b/src/main/ai-vault-search/session-search-typo-scope.test.ts new file mode 100644 index 00000000000..98e3938178e --- /dev/null +++ b/src/main/ai-vault-search/session-search-typo-scope.test.ts @@ -0,0 +1,71 @@ +import { afterEach, expect, it } from 'vitest' +import { + addSyntheticSession, + openSessionSearchHarness, + type SessionSearchHarness +} from './session-search-engine-test-fixture' + +// Typo repair used to read `messages_vocab` and probe `messages_fts` with no +// column filter, so tool output decided whether a conversation-scoped query was +// repaired — in both directions. A tool row carrying the misspelling made the +// query look correctly spelled and suppressed the repair; a tool row carrying a +// rare word offered it as the suggestion, naming in `repairedTerms` a string +// from a column the scope will never show. + +let harness: SessionSearchHarness | null = null +let control: SessionSearchHarness | null = null + +afterEach(async () => { + await harness?.close() + await control?.close() + harness = null + control = null +}) + +it('repairs a conversation query the same way with or without a tool row', async () => { + harness = await openSessionSearchHarness('ss-typo-scope-suppress') + addSyntheticSession(harness.db, { id: 1, text: 'we changed resolveTerminalPath today', rows: 2 }) + // A second session whose tool output happens to contain the misspelling. + addSyntheticSession(harness.db, { + id: 2, + text: 'ran the linter', + toolText: 'warning: unknown symbol resolveterminalpth in build log', + rows: 2, + role: 'assistant' + }) + + // The same index without that one tool row. + control = await openSessionSearchHarness('ss-typo-scope-control') + addSyntheticSession(control.db, { id: 1, text: 'we changed resolveTerminalPath today', rows: 2 }) + addSyntheticSession(control.db, { id: 2, text: 'ran the linter' }) + + const request = { query: 'resolveterminalpth', scope: 'conversation' } as const + const withTool = harness.engine.search(request) + const clean = control.engine.search(request) + + expect(clean.planner.repairedTerms).toEqual(['resolveterminalpath']) + expect(clean.hits.map((hit) => hit.sessionId)).toEqual(['1']) + expect(withTool.planner.repairedTerms).toEqual(clean.planner.repairedTerms) + expect(withTool.hits.map((hit) => hit.sessionId)).toEqual(clean.hits.map((hit) => hit.sessionId)) +}) + +it('never repairs a conversation query onto a word only tool output holds', async () => { + harness = await openSessionSearchHarness('ss-typo-scope-leak') + addSyntheticSession(harness.db, { + id: 1, + text: 'ran the deploy', + toolText: 'AWS_SESSION_TOKEN=quicksilverfox expired', + rows: 2, + role: 'assistant' + }) + addSyntheticSession(harness.db, { id: 2, text: 'ordinary prose about nothing' }) + + const narrowed = harness.engine.search({ query: 'quicksilverfx', scope: 'conversation' }) + expect(narrowed.planner.repairedTerms).toBeUndefined() + expect(narrowed.hits).toEqual([]) + // The same query over the whole corpus still finds it, which is the scope + // doing its job rather than the repair being broken. + const wide = harness.engine.search({ query: 'quicksilverfx', scope: 'all' }) + expect(wide.planner.repairedTerms).toEqual(['quicksilverfox']) + expect(wide.hits.map((hit) => hit.sessionId)).toEqual(['1']) +}) diff --git a/src/shared/ai-vault-search-query-operators.test.ts b/src/shared/ai-vault-search-query-operators.test.ts new file mode 100644 index 00000000000..0bb5cbc463b --- /dev/null +++ b/src/shared/ai-vault-search-query-operators.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from 'vitest' +import { parseVaultQuery } from './ai-vault-session-filters' +import { + hasAiVaultSearchQueryOperators, + splitAiVaultSearchQuery +} from './ai-vault-search-query-operators' + +describe('what counts as an operator', () => { + it('splits repo: and path: out of the free text', () => { + const split = splitAiVaultSearchQuery('relay capacity repo:orca path:/work/app') + expect(split.text).toBe('relay capacity') + expect(split.terms).toEqual(['relay', 'capacity']) + expect(split.repoTerms).toEqual(['orca']) + expect(split.pathTerms).toEqual(['/work/app']) + expect(hasAiVaultSearchQueryOperators(split)).toBe(true) + }) + + it('keeps a value that only looks like an operator as ordinary text', () => { + const split = splitAiVaultSearchQuery('myrepo:x https://host/path:y') + expect(split.repoTerms).toEqual([]) + expect(split.pathTerms).toEqual([]) + expect(split.text).toBe('myrepo:x https://host/path:y') + }) + + it('reads a quoted operator value whole, including its spaces', () => { + expect(splitAiVaultSearchQuery('path:"/Users/ada/My Project" needle').pathTerms).toEqual([ + '/Users/ada/My Project' + ]) + }) + + it('does not let an apostrophe in prose swallow the operator between quotes', () => { + const split = splitAiVaultSearchQuery("it's a repo:orca thing's") + expect(split.repoTerms).toEqual(['orca']) + }) + + it('preserves operator case, which the panel folds and the index must not', () => { + // cwd_key keeps execution-host case, so folding here would lose a POSIX + // directory whose name differs only in case. + expect(splitAiVaultSearchQuery('path:/Work/App').pathTerms).toEqual(['/Work/App']) + expect(parseVaultQuery('path:/Work/App').pathTerms).toEqual(['/work/app']) + }) + + it('has no operators when the query is plain text', () => { + expect(hasAiVaultSearchQueryOperators(splitAiVaultSearchQuery('relay capacity'))).toBe(false) + }) +}) + +// The panel parses through this module now, so the two cannot disagree by +// construction. What is worth pinning is the handful of shapes where the +// panel's old hand-rolled tokenizer answered differently, so the change of +// behaviour is a decision on the record rather than a surprise. +describe('the shapes where the panel parser used to answer differently', () => { + it.each([ + ['repo:"" x', 'repoTerms'], + ['path:"" x', 'pathTerms'] + ] as const)('drops the empty operator value in %s instead of filtering on `""`', (query, key) => { + // The old tokenizer kept the quote characters as the value, so `repo:""` + // filtered on a label no session has and silently emptied the list. An + // operator with nothing in it is not a narrowing. + expect(splitAiVaultSearchQuery(query)[key]).toEqual([]) + expect(parseVaultQuery(query)[key]).toEqual([]) + }) + + it.each([ + ['repo:" " x', 'repoTerms'], + ['path:" " x', 'pathTerms'] + ] as const)('drops the whitespace-only operator value in %s too', (query, key) => { + // Same defect as `repo:""` wearing a different hat: an untrimmed `" "` + // survives as a term, matches no label, and empties the list. + expect(splitAiVaultSearchQuery(query)[key]).toEqual([]) + expect(parseVaultQuery(query)[key]).toEqual([]) + }) + + it('trims a quoted operator value rather than searching for the spaces', () => { + expect(splitAiVaultSearchQuery('repo:" session-search "').repoTerms).toEqual(['session-search']) + }) + + it.each(['"" empty', "'' empty", '" " empty'])( + 'reads the empty quotes in %s as an empty term', + (query) => { + // Same reason one level up: the old parser searched for the two characters + // and found nothing, where an empty term matches everything and leaves the + // rest of the query to do the work. + expect(parseVaultQuery(query).terms).toEqual(['', 'empty']) + } + ) + + it.each([ + ['"foo"bar', { terms: ['foo', 'bar'], repoTerms: [], pathTerms: [] }], + ['"a b"c', { terms: ['a b', 'c'], repoTerms: [], pathTerms: [] }], + ['repo:"a"b', { terms: ['b'], repoTerms: ['a'], pathTerms: [] }], + ['path:"a"b', { terms: ['b'], repoTerms: [], pathTerms: ['a'] }], + ['repo:"a b"c d', { terms: ['c', 'd'], repoTerms: ['a b'], pathTerms: [] }] + ])('reads %s exactly as the panel always has', (query, expected) => { + // A closing quote does not have to end a word. Requiring it turned each of + // these into one term carrying its own quote characters, which matches + // nothing; the apostrophe case below is protected by the token start, not + // by that rule. + expect(parseVaultQuery(query)).toEqual(expected) + }) +}) + +describe('agrees with the sessions panel parser on operator recognition', () => { + it.each([ + 'relay capacity', + 'repo:orca needle', + 'path:/work/app needle', + 'myrepo:x', + 'needle repo:orca path:/work/app', + 'path:"/Users/ada/My Project"', + 'https://host/path:y' + ])('reads the same operators out of %s', (query) => { + const split = splitAiVaultSearchQuery(query) + const parsed = parseVaultQuery(query) + const fold = (values: readonly string[]): string[] => values.map((v) => v.toLowerCase()).sort() + expect(fold(split.repoTerms)).toEqual(fold(parsed.repoTerms)) + expect(fold(split.pathTerms)).toEqual(fold(parsed.pathTerms)) + }) +}) diff --git a/src/shared/ai-vault-search-query-operators.ts b/src/shared/ai-vault-search-query-operators.ts new file mode 100644 index 00000000000..a75da8c769e --- /dev/null +++ b/src/shared/ai-vault-search-query-operators.ts @@ -0,0 +1,90 @@ +/** Anchored at a token start only, so `myrepo:x` and `https://h/path:x` stay literal. */ +const OPERATOR = /(repo|path):/iy + +export type AiVaultSearchQuerySplit = { + /** Query minus the operator tokens, quoting intact; what FTS sees. */ + text: string + /** The same free text as tokens with quotes stripped; what a substring matcher wants. */ + terms: readonly string[] + /** Operator values as typed apart from surrounding space: the panel folds case, the index does not. */ + repoTerms: readonly string[] + pathTerms: readonly string[] +} + +/** + * The one reading of `repo:` / `path:` in the product: the sessions panel and the + * search index must agree on what is an operator and what is ordinary text. + */ +export function splitAiVaultSearchQuery(query: string): AiVaultSearchQuerySplit { + const spans: string[] = [] + const terms: string[] = [] + const repoTerms: string[] = [] + const pathTerms: string[] = [] + let index = 0 + while (index < query.length) { + if (isBoundary(query[index])) { + index += 1 + continue + } + OPERATOR.lastIndex = index + const operator = OPERATOR.exec(query) + if (operator) { + const at = index + operator[0].length + const quoted = readQuoted(query, at) + const value = quoted?.value ?? readBare(query, at) + index = quoted ? quoted.end : at + value.length + // Trimmed for the same reason an empty value is dropped: `repo:" "` is + // not a narrowing anyone typed on purpose, and an untrimmed one matches + // no label at all, which silently empties the list. + const operand = value.trim() + if (operand) { + ;(operator[1]!.toLowerCase() === 'repo' ? repoTerms : pathTerms).push(operand) + } + continue + } + const quoted = readQuoted(query, index) + const value = quoted?.value ?? readBare(query, index) + const end = quoted ? quoted.end : index + value.length + spans.push(query.slice(index, end)) + // The span keeps the query verbatim for FTS; only the substring matcher's + // copy is trimmed, so `" "` reads as the empty term `""` already does + // rather than as a term no session's text contains. + terms.push(value.trim()) + index = end + } + return { text: spans.join(' '), terms, repoTerms, pathTerms } +} + +export function hasAiVaultSearchQueryOperators(split: AiVaultSearchQuerySplit): boolean { + return split.repoTerms.length > 0 || split.pathTerms.length > 0 +} + +function isBoundary(char: string | undefined): boolean { + return char === undefined || /\s/.test(char) +} + +/** + * A quoted span, or null when this is not one. + * + * What keeps the apostrophes in `it's a repo:orca thing's` from opening a span + * that swallows the operator is the caller: this only ever runs at a token + * start, and the quote in `it's` is not at one. The closing quote is then just + * the next one, wherever it falls, so `"a b"c` reads as the panel has always + * read it — the span, then the rest as its own token. + */ +function readQuoted(query: string, at: number): { value: string; end: number } | null { + const quote = query[at] + if (quote !== '"' && quote !== "'") { + return null + } + const close = query.indexOf(quote, at + 1) + return close === -1 ? null : { value: query.slice(at + 1, close), end: close + 1 } +} + +function readBare(query: string, at: number): string { + let end = at + while (end < query.length && !isBoundary(query[end])) { + end += 1 + } + return query.slice(at, end) +} diff --git a/src/shared/ai-vault-session-filters.ts b/src/shared/ai-vault-session-filters.ts index 7a0708151ed..39aedaf4626 100644 --- a/src/shared/ai-vault-session-filters.ts +++ b/src/shared/ai-vault-session-filters.ts @@ -8,6 +8,7 @@ import { normalizeRuntimePathSeparators } from './cross-platform-path' import { isClipboardTextByteLengthOverLimit } from './clipboard-text' +import { splitAiVaultSearchQuery } from './ai-vault-search-query-operators' import { parseWslUncPath } from './wsl-paths' import type { AiVaultAgent, @@ -179,31 +180,61 @@ export function agentLabel(agent: AiVaultAgent): string { return aiVaultAgentLabel(agent) } +/** + * One reading of `repo:` / `path:` for the whole product. + * + * Delegates to `splitAiVaultSearchQuery`, which the search index also plans + * from, so a query cannot mean one thing in this list and another in the index. + * The values come back folded because everything this file compares is folded; + * the index keeps the unfolded form, which is why the split itself does not. + */ export function parseVaultQuery(query: string): ParsedQuery { - const terms: string[] = [] - const repoTerms: string[] = [] - const pathTerms: string[] = [] - - for (const rawToken of tokenizeQuery(query)) { - const token = rawToken.toLowerCase() - if (token.startsWith('repo:')) { - const value = token.slice('repo:'.length) - if (value) { - repoTerms.push(value) - } - continue - } - if (token.startsWith('path:')) { - const value = token.slice('path:'.length) - if (value) { - pathTerms.push(value) - } - continue - } - terms.push(token) + const split = splitAiVaultSearchQuery(query) + const fold = (values: readonly string[]): string[] => values.map((value) => value.toLowerCase()) + return { + terms: fold(split.terms), + repoTerms: fold(split.repoTerms), + pathTerms: fold(split.pathTerms) } +} - return { terms, repoTerms, pathTerms } +/** What `repo:` and `path:` are compared against for one session. */ +export type AiVaultQueryOperatorTarget = { + cwd: string | null + filePath: string + /** + * What `repo:` matches. The panel passes a resolved project label when it has + * one; everything else falls back to the last two path segments. + */ + repoLabel?: string +} + +/** + * Whether one session satisfies every `repo:` and `path:` term. + * + * The single definition of what those operators mean. The search index applies + * this over its retrieved rows rather than expressing it in SQL, because SQL + * cannot: LIKE folds ASCII and nothing else, and `path:` searches the transcript + * path as well as the working directory. Both keys are conjunctive, matching + * the qualifier semantics the panel has always had. + */ +export function matchesAiVaultQueryOperators( + target: AiVaultQueryOperatorTarget, + operators: { repoTerms: readonly string[]; pathTerms: readonly string[] } +): boolean { + if (operators.repoTerms.length > 0) { + const repoLabel = (target.repoLabel ?? folderLabel(target.cwd)).toLowerCase() + if (operators.repoTerms.some((term) => !repoLabel.includes(term.toLowerCase()))) { + return false + } + } + if (operators.pathTerms.length > 0) { + const pathSearch = `${target.cwd ?? ''} ${target.filePath}`.toLowerCase() + if (operators.pathTerms.some((term) => !pathSearch.includes(term.toLowerCase()))) { + return false + } + } + return true } function matchesQuery( @@ -229,25 +260,18 @@ function matchesQuery( return false } } - if (parsed.repoTerms.length > 0) { - const sessionProject = filters.sessionProjectById?.get(session.id) - const repoLabel = ( - sessionProject?.kind === 'repo' - ? (filters.projectLabelByKey?.get(sessionProject.key) ?? sessionProject.label) - : folderLabel(session.cwd) - ).toLowerCase() - if (parsed.repoTerms.some((term) => !repoLabel.includes(term))) { - return false - } - } - if (parsed.pathTerms.length > 0) { - const pathSearch = `${session.cwd ?? ''} ${session.filePath}`.toLowerCase() - if (parsed.pathTerms.some((term) => !pathSearch.includes(term))) { - return false - } - } - - return true + const sessionProject = filters.sessionProjectById?.get(session.id) + return matchesAiVaultQueryOperators( + { + cwd: session.cwd, + filePath: session.filePath, + repoLabel: + sessionProject?.kind === 'repo' + ? (filters.projectLabelByKey?.get(sessionProject.key) ?? sessionProject.label) + : undefined + }, + parsed + ) } function sessionSortTime(session: AiVaultSession, sort: AiVaultSort): number { @@ -291,25 +315,3 @@ function createAiVaultWorkspaceMatcher(workspacePath: string): (normalizedCwd: s const matchesLinux = createNormalizedPathInsideOrEqualMatcher(workspaceWslPath.linuxPath) return (cwd) => matches(cwd) || matchesLinux(cwd) } - -function tokenizeQuery(query: string): string[] { - const tokens: string[] = [] - // Why: keep quoted operator values (repo:/path:) intact so labels and paths - // containing spaces still match — e.g. path:"/Users/ada/My Project". - const pattern = /(repo|path):"([^"]+)"|(repo|path):'([^']+)'|"([^"]+)"|'([^']+)'|(\S+)/gi - let match: RegExpExecArray | null - while ((match = pattern.exec(query)) !== null) { - const operator = match[1] ?? match[3] - const operatorValue = match[2] ?? match[4] - if (operator && operatorValue?.trim()) { - tokens.push(`${operator.toLowerCase()}:${operatorValue.trim()}`) - continue - } - - const token = match[5] ?? match[6] ?? match[7] - if (token?.trim()) { - tokens.push(token.trim()) - } - } - return tokens -}