From b997fcc77a80ae82cfd08e9f876dcc4c673aef6c Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:27:05 -0400 Subject: [PATCH] fix(session-search): try phrase and AND routes for prose queries before OR (#20754) * fix(session-search): try phrase and AND routes for prose queries before OR An exact sentence pasted out of a transcript was not returned. The route ladder only ran the phrase and AND rungs for a literal-looking query, so prose fell straight to OR, where the sentence's common words filled the candidate limit with recent sessions and the old session holding the sentence never reached ranking. The planner now carries a `phrase` candidate: the query's tokens in order with stop words kept, which is what the sentence is actually indexed as. The ladder runs phrase then AND over those tokens for every query of two or more tokens. A one-token query still takes the rung only when it looked literal. `incomplete` is reported by the rung that answered rather than accumulated across every rung tried. * fix(session-search): mark a snippet with the route that retrieved it A phrase hit was highlighted with the OR expression over the stop-word stripped terms, so an exact sentence rendered as scattered bold words with its stop words plain. The snippet now uses the expression the route matched by: one run for a phrase, every typed word for AND, the terms for OR. * fix(session-search): repair a prose phrase without dropping its stop words Typo repair re-planned the query from `plan.body`, which prose has already had its stop words removed from. `relay is droppng frames` therefore came back as the plan for `relay dropping frames`, and the phrase rung searched for a sentence nobody wrote: the transcript holds `relay is dropping frames`, so the exact match fell through to AND. The repair now maps over `plan.phrase`, the tokens as typed, and re-plans from those. Only terms the body holds are offered to the corrector, so a stop word is still never repaired, and the re-plan recomputes the body from the corrected sentence exactly as before. --- .../agent-session-search-query-tuning.md | 14 +++- .../session-search-engine.test.ts | 72 ++++++++++++++++++- .../ai-vault-search/session-search-engine.ts | 2 +- .../session-search-orphan-rows.test.ts | 2 +- .../session-search-query-planner.test.ts | 25 +++++++ .../session-search-query-planner.ts | 12 +++- .../session-search-retrieval.ts | 58 ++++++++++----- .../session-search-snippet-marks.test.ts | 28 ++++++++ .../ai-vault-search/session-search-snippet.ts | 38 +++++++--- 9 files changed, 215 insertions(+), 36 deletions(-) diff --git a/docs/reference/agent-session-search-query-tuning.md b/docs/reference/agent-session-search-query-tuning.md index 9e18097cb9e..e7895e0382d 100644 --- a/docs/reference/agent-session-search-query-tuning.md +++ b/docs/reference/agent-session-search-query-tuning.md @@ -215,5 +215,15 @@ first page without a cursor. The caller can retry from page one. No long-lived read transaction is needed, and a mixed page is never returned as a valid snapshot. Repository/path operators are applied before a phrase or AND route is accepted. -Candidate truncation remains explicit, including when an earlier route reached -its cap but had no eligible sessions. +Candidate truncation is reported by the rung that answered, not by every rung +tried. Each rung of the ladder matches a superset of the one before it, so a +rung that reached its cap with no eligible sessions is always followed by one +that reaches it too: a full candidate set stays explicit either way. + +The phrase and AND rungs run for prose as well as for literal-looking input, +over the query's tokens as typed rather than the stop-word-stripped OR body. A +sentence pasted out of a transcript is ordinary words in order; over OR its +common words fill the candidate limit with recent sessions and the old session +holding the sentence never reaches ranking. The cost is two FTS queries that +usually miss, which on the corpus above sits inside this harness's run-to-run +noise. A one-token query still takes the rung only when it looked literal. diff --git a/src/main/ai-vault-search/session-search-engine.test.ts b/src/main/ai-vault-search/session-search-engine.test.ts index 6247144d4dc..f52213857c2 100644 --- a/src/main/ai-vault-search/session-search-engine.test.ts +++ b/src/main/ai-vault-search/session-search-engine.test.ts @@ -66,6 +66,18 @@ describe('the route ladder tries phrase, then AND, then repair, then OR', () => expect(ids(result).sort()).toEqual(['1', '2']) }) + it('keeps the stop words a repaired prose phrase was typed with', async () => { + const { db, engine } = await open('ss-engine-typo-phrase') + // Two copies, so the repair only suggests a term the index really holds. + addSyntheticSession(db, { id: 1, text: 'relay is dropping frames' }) + addSyntheticSession(db, { id: 2, text: 'dropping frames again here' }) + // Repairing the body alone would re-plan `relay dropping frames`, which no + // phrase in the index can match, and the answer would fall to AND. + const result = engine.search({ query: 'relay is droppng frames' }) + expect(result.planner.route).toBe('typo+phrase') + expect(ids(result)).toEqual(['1']) + }) + it('keeps every term a repaired literal was typed with', async () => { const { db, engine } = await open('ss-engine-typo-literal') addSyntheticSession(db, { id: 1, text: 'parseJson the data' }) @@ -184,8 +196,8 @@ describe('the conversation scope is a column filter, and it binds the whole quer (db.prepare('SELECT max(id) AS id FROM messages').get() as { id: number }).id ) const plan = planSessionSearchQuery('harbor') - expect(sessionSearchSnippet(db, 'conversation', rowid, plan)).toEqual(EMPTY_SNIPPET) - expect(sessionSearchSnippet(db, 'all', rowid, plan).text).toContain('output') + expect(sessionSearchSnippet(db, 'conversation', rowid, plan, 'or')).toEqual(EMPTY_SNIPPET) + expect(sessionSearchSnippet(db, 'all', rowid, plan, 'or').text).toContain('output') }) }) @@ -471,3 +483,59 @@ it.each(['repo:target', 'path:/work/target'])( expect(result.truncated.candidates).toBe(false) } ) + +describe('a sentence pasted out of a transcript is found behind a full candidate set', () => { + // The words of an ordinary sentence are common, so over OR the candidate + // limit fills with whatever is recent and the old session holding the + // sentence never reaches ranking. + const sentence = 'The sol review says the PR is not quite merge-ready yet' + + async function pasted(sessionCandidateLimit = 600): Promise { + const opened = await open('ss-engine-pasted-sentence', { sessionCandidateLimit }) + addSyntheticSession(opened.db, { + id: 1, + text: `${sentence}, but not because of the implementation.`, + updatedAt: '2026-08-01T00:00:00.000Z' + }) + for (let id = 2; id <= sessionCandidateLimit + 50; id++) { + addSyntheticSession(opened.db, { + id, + text: 'the review says the implementation is not quite there yet', + updatedAt: '2026-09-09T00:00:00.000Z' + }) + } + return opened + } + + it('returns the exact sentence first, over the phrase route', async () => { + const { engine } = await pasted() + const result = engine.search({ query: sentence }) + expect(result.planner.route).toBe('phrase') + expect(ids(result)).toEqual(['1']) + }) + + it('does not claim the results were limited when the phrase rung answered', async () => { + // The OR rung would have filled the candidate limit; the rung that answered + // did not, and it is the answering rung the notice describes. + const { engine } = await pasted() + expect(engine.search({ query: sentence }).truncated.candidates).toBe(false) + expect(engine.search({ query: 'the review says yet' }).truncated.candidates).toBe(true) + }) + + it('falls to AND for prose whose words are all present but not adjacent', async () => { + const { db, engine } = await open('ss-engine-prose-and') + addSyntheticSession(db, { + id: 1, + text: 'yet quite merge-ready the PR is not what sol says a review of it' + }) + const result = engine.search({ query: sentence }) + expect(result.planner.route).toBe('and') + expect(ids(result)).toEqual(['1']) + }) + + it('still sends a single prose word straight to OR', async () => { + const { db, engine } = await open('ss-engine-prose-one-word') + addSyntheticSession(db, { id: 1, text: 'relay' }) + expect(engine.search({ query: 'relay' }).planner.route).toBe('or') + }) +}) diff --git a/src/main/ai-vault-search/session-search-engine.ts b/src/main/ai-vault-search/session-search-engine.ts index f713d8d2a3f..499b7c71e9f 100644 --- a/src/main/ai-vault-search/session-search-engine.ts +++ b/src/main/ai-vault-search/session-search-engine.ts @@ -212,7 +212,7 @@ export class SessionSearchEngine { const { session, message } = entry const snippet = message && retrieved - ? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan) + ? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan, retrieved.route) : EMPTY_SNIPPET return { ...sessionFields(session), diff --git a/src/main/ai-vault-search/session-search-orphan-rows.test.ts b/src/main/ai-vault-search/session-search-orphan-rows.test.ts index dfda2303104..37a6c6a2f00 100644 --- a/src/main/ai-vault-search/session-search-orphan-rows.test.ts +++ b/src/main/ai-vault-search/session-search-orphan-rows.test.ts @@ -96,7 +96,7 @@ it('snippets nothing for an orphaned row, even asked for it by rowid', async () const { harness: open, rowids } = await withOrphans() const plan = planSessionSearchQuery('marmoset') for (const scope of ['all', 'conversation'] as const) { - expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan)).toEqual({ + expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan, 'or')).toEqual({ text: '', truncated: false }) diff --git a/src/main/ai-vault-search/session-search-query-planner.test.ts b/src/main/ai-vault-search/session-search-query-planner.test.ts index ac4874b1d10..3c1ab7f3848 100644 --- a/src/main/ai-vault-search/session-search-query-planner.test.ts +++ b/src/main/ai-vault-search/session-search-query-planner.test.ts @@ -82,3 +82,28 @@ describe('FTS5 expressions quote every term', () => { expect(orExpression(['alpha', 'beta'])).toBe('"alpha" OR "beta"') }) }) + +describe('the phrase candidate is the query as typed', () => { + const sentence = 'The sol review says the PR is not quite merge-ready yet' + + it('is prose, so nothing about its shape reaches the phrase route', () => { + expect(isLiteralQuery(sentence)).toBe(false) + }) + + it('keeps the stop words the OR body drops, because the index holds them', () => { + const plan = planSessionSearchQuery('why is the relay dropping frames') + expect(plan.phrase).toEqual(['why', 'is', 'the', 'relay', 'dropping', 'frames']) + expect(plan.body).toEqual(['relay', 'dropping', 'frames']) + }) + + it('is the same list as the body for a literal, which keeps every token', () => { + const plan = planSessionSearchQuery('the foo.ts file') + expect(plan.phrase).toEqual(plan.body) + }) + + it('quotes into one phrase a pasted sentence can actually match', () => { + expect(phraseExpression(planSessionSearchQuery(sentence).phrase)).toBe( + '"The sol review says the PR is not quite merge-ready yet"' + ) + }) +}) diff --git a/src/main/ai-vault-search/session-search-query-planner.ts b/src/main/ai-vault-search/session-search-query-planner.ts index 91e8711fbe3..5dbcbba5f37 100644 --- a/src/main/ai-vault-search/session-search-query-planner.ts +++ b/src/main/ai-vault-search/session-search-query-planner.ts @@ -31,8 +31,15 @@ export type SessionSearchQueryPlan = { truncated: boolean /** Deduplicated index-faithful terms for the OR fallback, incl. identifier pieces. */ terms: string[] - /** Query-order tokens minus stop words: the phrase / AND candidate. */ + /** Query-order tokens minus stop words for prose, all of them for a literal. */ body: string[] + /** + * Query-order tokens exactly as typed, stop words kept: the phrase / AND + * candidate. A sentence pasted out of a transcript is only adjacent in the + * index with its stop words in place, and `unicode61` indexes them, so the + * phrase rung has to search the words the user actually typed. + */ + phrase: string[] } export function isLiteralQuery(query: string): boolean { @@ -95,7 +102,8 @@ export function planSessionSearchQuery( literal, truncated, terms: [...terms, ...extra].slice(0, MAX_TERMS), - body: body.slice(0, MAX_BODY_TERMS) + body: body.slice(0, MAX_BODY_TERMS), + phrase: raw } } diff --git a/src/main/ai-vault-search/session-search-retrieval.ts b/src/main/ai-vault-search/session-search-retrieval.ts index 59fef23e903..f4a963d37b3 100644 --- a/src/main/ai-vault-search/session-search-retrieval.ts +++ b/src/main/ai-vault-search/session-search-retrieval.ts @@ -71,8 +71,7 @@ export class SessionSearchRetrieval { } /** - * The route ladder: phrase, then AND for a literal-looking query, then typo - * repair, then OR. + * The route ladder: phrase, then AND, then typo repair, then OR. * * Repair runs before the OR fallback rather than after it fails. A typo next * to a common word would otherwise be masked: the common word alone retrieves @@ -84,7 +83,10 @@ export class SessionSearchRetrieval { let sessions: SessionRow[] = [] const match = (expression: string): MessageRow[] => { const rows = this.match(expression, scope) - incomplete ||= rows.length >= scope.candidateLimit + // Assigned, not accumulated: only the rung whose rows are returned can + // say whether a cap hid anything. A phrase rung that filled the limit and + // was then discarded describes a row set the answering rung never used. + incomplete = rows.length >= scope.candidateLimit sessions = this.loadSessions( rows.map((row) => row.session_row_id), scope @@ -92,13 +94,13 @@ export class SessionSearchRetrieval { const eligible = new Set(sessions.map((row) => row.id)) return rows.filter((row) => eligible.has(row.session_row_id)) } - const exact = this.literal(plan, match) + const exact = this.phraseThenAnd(plan, match) if (exact) { return { ...exact, plan, incomplete, sessions } } const repaired = this.repair(plan, scope.scope) const effective = repaired ?? plan - const literal = repaired ? this.literal(repaired, match) : null + const literal = repaired ? this.phraseThenAnd(repaired, match) : null const found = literal ?? { rows: match(orExpression(effective.terms)), route: 'or' as const @@ -178,41 +180,59 @@ export class SessionSearchRetrieval { ): SessionSearchQueryPlan | null { const typoRepair = this.typoRepair let changed = false - const body = plan.body.map((term) => { + // Only the body is a candidate for a correction, but the re-plan is fed the + // tokens as typed: re-planning the body alone would hand the phrase rung a + // sentence with its stop words already gone, and `relay dropping frames` + // cannot match the `relay is dropping frames` that is in the transcript. + const repairable = new Set(plan.body.map((term) => term.toLowerCase())) + const phrase = plan.phrase.map((token) => { + if (!repairable.has(token.toLowerCase())) { + return token + } // Repaired inside the scope the search will run in, so a spelling only // tool output carries neither suppresses a repair nor becomes one. - const fix = typoRepair.correct(term, scope) - if (fix && fix !== term.toLowerCase()) { + const fix = typoRepair.correct(token, scope) + if (fix && fix !== token.toLowerCase()) { changed = true return fix } - return term + return token }) // The repair changes spellings, not the query's character: the re-plan is // told what the original decided so a corrected literal keeps every term it // was typed with. - return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null + return changed ? planSessionSearchQuery(phrase.join(' '), plan.literal) : null } - /** Phrase, then AND, for literal-looking queries; null when neither matches. */ - private literal( + /** + * Phrase, then AND, over the tokens as typed; null when neither matches. + * + * Prose runs it too, and not only a literal-looking query. A sentence pasted + * out of a transcript is ordinary words in order, and over OR its common + * words fill the candidate limit with recent sessions long before the old + * session that holds the sentence is reached, so the exact match a user can + * see in front of them comes back missing. + */ + private phraseThenAnd( plan: SessionSearchQueryPlan, match: (expression: string) => MessageRow[] ): { rows: MessageRow[]; route: 'phrase' | 'and' } | null { - if (!plan.literal || plan.body.length === 0) { - return null - } + const tokens = plan.phrase // A one-token literal (`resolveTerminalPath`, `src/a/b.ts`) is its own // phrase: the tokenizer keeps it whole, so the exact token is the cheap, - // precise first try before the identifier pieces fan out over OR. - const phrase = match(phraseExpression(plan.body)) + // precise first try before the identifier pieces fan out over OR. One word + // of prose is not quoting anything, so it goes straight to OR as before. + if (tokens.length === 0 || (tokens.length < 2 && !plan.literal)) { + return null + } + const phrase = match(phraseExpression(tokens)) if (phrase.length > 0) { return { rows: phrase, route: 'phrase' } } - if (plan.body.length < 2) { + if (tokens.length < 2) { return null } - const and = match(andExpression(plan.body)) + const and = match(andExpression(tokens)) return and.length > 0 ? { rows: and, route: 'and' } : null } diff --git a/src/main/ai-vault-search/session-search-snippet-marks.test.ts b/src/main/ai-vault-search/session-search-snippet-marks.test.ts index d4237ce8b6f..81b2f833981 100644 --- a/src/main/ai-vault-search/session-search-snippet-marks.test.ts +++ b/src/main/ai-vault-search/session-search-snippet-marks.test.ts @@ -142,3 +142,31 @@ it('does not cut a snippet at a private-use code point the transcript wrote', as ) expect(snippet).toContain('qqqqq') }) + +it('marks a phrase hit as one run, stop words included', async () => { + harness = await openSessionSearchHarness('ss-snippet-phrase-run') + addSyntheticSession(harness.db, { + id: 1, + text: 'Agent: the code already has several fixes for blank restores, including replaying' + }) + + const result = harness.engine.search({ query: 'the code already has several fixes' }) + expect(result.planner.route).toBe('phrase') + expect(result.hits[0]?.evidence?.snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}the code already has several fixes${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) +}) + +it('marks every typed word of an AND hit, stop words included', async () => { + harness = await openSessionSearchHarness('ss-snippet-and-words') + addSyntheticSession(harness.db, { id: 1, text: 'fixes for the restore path, several of them' }) + + const result = harness.engine.search({ query: 'several fixes for the restore' }) + expect(result.planner.route).toBe('and') + const snippet = result.hits[0]?.evidence?.snippet ?? '' + for (const word of ['several', 'fixes', 'for', 'the', 'restore']) { + expect(snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}${word}${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) + } +}) diff --git a/src/main/ai-vault-search/session-search-snippet.ts b/src/main/ai-vault-search/session-search-snippet.ts index 1204cdd10a8..e11dd9dca90 100644 --- a/src/main/ai-vault-search/session-search-snippet.ts +++ b/src/main/ai-vault-search/session-search-snippet.ts @@ -4,11 +4,13 @@ import { SESSION_SEARCH_SNIPPET_MARK_OPEN } from './session-search-engine-types' import { + andExpression, orExpression, + phraseExpression, scopedExpression, type SessionSearchQueryPlan } from './session-search-query-planner' -import type { SessionSearchScope } from './session-search-engine-types' +import type { SessionSearchRoute, SessionSearchScope } from './session-search-engine-types' // What FTS5 wraps a match in before this module rewrites it to the public // marks. Private-use code points, and not `[[`, because two different jobs here @@ -35,15 +37,17 @@ export const EMPTY_SNIPPET: SessionSearchSnippet = { text: '', truncated: false /** * The window of one message that shows why it matched. * - * The expression is the plan's OR form rather than the route's, so a hit found - * through typo repair is marked with the repaired terms it was actually - * retrieved by, and a phrase hit still marks each of its words. + * Marked with the expression the route retrieved by, so a phrase hit is one + * highlight over the words as typed, stop words included, and an OR hit marks + * each term it was found through. The plan is the effective one, so a hit + * found through typo repair is marked with the repaired terms. */ export function sessionSearchSnippet( db: SyncDatabase, scope: SessionSearchScope, rowid: number, - plan: SessionSearchQueryPlan + plan: SessionSearchQueryPlan, + route: SessionSearchRoute ): SessionSearchSnippet { // Why: the identifier shadow column is word soup; a hit that also matches in a // prose column should be shown from there. Column -1 (any column) is the @@ -83,10 +87,8 @@ export function sessionSearchSnippet( JOIN sessions s ON s.id = m.session_row_id WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)` ) - .get(scopedExpression(scope, orExpression(plan.terms)), rowid) as - | Record - | undefined - if (!row) { + .get(scopedExpression(scope, routeExpression(plan, route)), rowid) + if (!isSnippetRow(row)) { return EMPTY_SNIPPET } // A snippet with nothing highlighted tells the user nothing; omit it. @@ -103,6 +105,24 @@ export function sessionSearchSnippet( } } +function isSnippetRow(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + Object.values(value).every((column) => typeof column === 'string') + ) +} + +function routeExpression(plan: SessionSearchQueryPlan, route: SessionSearchRoute): string { + if (route.endsWith('phrase')) { + return phraseExpression(plan.phrase) + } + if (route.endsWith('and')) { + return andExpression(plan.phrase) + } + return orExpression(plan.terms) +} + /** One run of the snippet's own text, or one mark FTS5 put between two runs. */ type SnippetPiece = { kind: 'text'; value: string } | { kind: 'mark'; value: string }