diff --git a/docs/reference/agent-session-search-query-tuning.md b/docs/reference/agent-session-search-query-tuning.md index 9e18097cb9e..e7895e0382d 100644 --- a/docs/reference/agent-session-search-query-tuning.md +++ b/docs/reference/agent-session-search-query-tuning.md @@ -215,5 +215,15 @@ first page without a cursor. The caller can retry from page one. No long-lived read transaction is needed, and a mixed page is never returned as a valid snapshot. Repository/path operators are applied before a phrase or AND route is accepted. -Candidate truncation remains explicit, including when an earlier route reached -its cap but had no eligible sessions. +Candidate truncation is reported by the rung that answered, not by every rung +tried. Each rung of the ladder matches a superset of the one before it, so a +rung that reached its cap with no eligible sessions is always followed by one +that reaches it too: a full candidate set stays explicit either way. + +The phrase and AND rungs run for prose as well as for literal-looking input, +over the query's tokens as typed rather than the stop-word-stripped OR body. A +sentence pasted out of a transcript is ordinary words in order; over OR its +common words fill the candidate limit with recent sessions and the old session +holding the sentence never reaches ranking. The cost is two FTS queries that +usually miss, which on the corpus above sits inside this harness's run-to-run +noise. A one-token query still takes the rung only when it looked literal. diff --git a/src/main/ai-vault-search/session-search-engine.test.ts b/src/main/ai-vault-search/session-search-engine.test.ts index 6247144d4dc..f52213857c2 100644 --- a/src/main/ai-vault-search/session-search-engine.test.ts +++ b/src/main/ai-vault-search/session-search-engine.test.ts @@ -66,6 +66,18 @@ describe('the route ladder tries phrase, then AND, then repair, then OR', () => expect(ids(result).sort()).toEqual(['1', '2']) }) + it('keeps the stop words a repaired prose phrase was typed with', async () => { + const { db, engine } = await open('ss-engine-typo-phrase') + // Two copies, so the repair only suggests a term the index really holds. + addSyntheticSession(db, { id: 1, text: 'relay is dropping frames' }) + addSyntheticSession(db, { id: 2, text: 'dropping frames again here' }) + // Repairing the body alone would re-plan `relay dropping frames`, which no + // phrase in the index can match, and the answer would fall to AND. + const result = engine.search({ query: 'relay is droppng frames' }) + expect(result.planner.route).toBe('typo+phrase') + expect(ids(result)).toEqual(['1']) + }) + it('keeps every term a repaired literal was typed with', async () => { const { db, engine } = await open('ss-engine-typo-literal') addSyntheticSession(db, { id: 1, text: 'parseJson the data' }) @@ -184,8 +196,8 @@ describe('the conversation scope is a column filter, and it binds the whole quer (db.prepare('SELECT max(id) AS id FROM messages').get() as { id: number }).id ) const plan = planSessionSearchQuery('harbor') - expect(sessionSearchSnippet(db, 'conversation', rowid, plan)).toEqual(EMPTY_SNIPPET) - expect(sessionSearchSnippet(db, 'all', rowid, plan).text).toContain('output') + expect(sessionSearchSnippet(db, 'conversation', rowid, plan, 'or')).toEqual(EMPTY_SNIPPET) + expect(sessionSearchSnippet(db, 'all', rowid, plan, 'or').text).toContain('output') }) }) @@ -471,3 +483,59 @@ it.each(['repo:target', 'path:/work/target'])( expect(result.truncated.candidates).toBe(false) } ) + +describe('a sentence pasted out of a transcript is found behind a full candidate set', () => { + // The words of an ordinary sentence are common, so over OR the candidate + // limit fills with whatever is recent and the old session holding the + // sentence never reaches ranking. + const sentence = 'The sol review says the PR is not quite merge-ready yet' + + async function pasted(sessionCandidateLimit = 600): Promise { + const opened = await open('ss-engine-pasted-sentence', { sessionCandidateLimit }) + addSyntheticSession(opened.db, { + id: 1, + text: `${sentence}, but not because of the implementation.`, + updatedAt: '2026-08-01T00:00:00.000Z' + }) + for (let id = 2; id <= sessionCandidateLimit + 50; id++) { + addSyntheticSession(opened.db, { + id, + text: 'the review says the implementation is not quite there yet', + updatedAt: '2026-09-09T00:00:00.000Z' + }) + } + return opened + } + + it('returns the exact sentence first, over the phrase route', async () => { + const { engine } = await pasted() + const result = engine.search({ query: sentence }) + expect(result.planner.route).toBe('phrase') + expect(ids(result)).toEqual(['1']) + }) + + it('does not claim the results were limited when the phrase rung answered', async () => { + // The OR rung would have filled the candidate limit; the rung that answered + // did not, and it is the answering rung the notice describes. + const { engine } = await pasted() + expect(engine.search({ query: sentence }).truncated.candidates).toBe(false) + expect(engine.search({ query: 'the review says yet' }).truncated.candidates).toBe(true) + }) + + it('falls to AND for prose whose words are all present but not adjacent', async () => { + const { db, engine } = await open('ss-engine-prose-and') + addSyntheticSession(db, { + id: 1, + text: 'yet quite merge-ready the PR is not what sol says a review of it' + }) + const result = engine.search({ query: sentence }) + expect(result.planner.route).toBe('and') + expect(ids(result)).toEqual(['1']) + }) + + it('still sends a single prose word straight to OR', async () => { + const { db, engine } = await open('ss-engine-prose-one-word') + addSyntheticSession(db, { id: 1, text: 'relay' }) + expect(engine.search({ query: 'relay' }).planner.route).toBe('or') + }) +}) diff --git a/src/main/ai-vault-search/session-search-engine.ts b/src/main/ai-vault-search/session-search-engine.ts index f713d8d2a3f..499b7c71e9f 100644 --- a/src/main/ai-vault-search/session-search-engine.ts +++ b/src/main/ai-vault-search/session-search-engine.ts @@ -212,7 +212,7 @@ export class SessionSearchEngine { const { session, message } = entry const snippet = message && retrieved - ? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan) + ? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan, retrieved.route) : EMPTY_SNIPPET return { ...sessionFields(session), diff --git a/src/main/ai-vault-search/session-search-orphan-rows.test.ts b/src/main/ai-vault-search/session-search-orphan-rows.test.ts index dfda2303104..37a6c6a2f00 100644 --- a/src/main/ai-vault-search/session-search-orphan-rows.test.ts +++ b/src/main/ai-vault-search/session-search-orphan-rows.test.ts @@ -96,7 +96,7 @@ it('snippets nothing for an orphaned row, even asked for it by rowid', async () const { harness: open, rowids } = await withOrphans() const plan = planSessionSearchQuery('marmoset') for (const scope of ['all', 'conversation'] as const) { - expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan)).toEqual({ + expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan, 'or')).toEqual({ text: '', truncated: false }) diff --git a/src/main/ai-vault-search/session-search-query-planner.test.ts b/src/main/ai-vault-search/session-search-query-planner.test.ts index ac4874b1d10..3c1ab7f3848 100644 --- a/src/main/ai-vault-search/session-search-query-planner.test.ts +++ b/src/main/ai-vault-search/session-search-query-planner.test.ts @@ -82,3 +82,28 @@ describe('FTS5 expressions quote every term', () => { expect(orExpression(['alpha', 'beta'])).toBe('"alpha" OR "beta"') }) }) + +describe('the phrase candidate is the query as typed', () => { + const sentence = 'The sol review says the PR is not quite merge-ready yet' + + it('is prose, so nothing about its shape reaches the phrase route', () => { + expect(isLiteralQuery(sentence)).toBe(false) + }) + + it('keeps the stop words the OR body drops, because the index holds them', () => { + const plan = planSessionSearchQuery('why is the relay dropping frames') + expect(plan.phrase).toEqual(['why', 'is', 'the', 'relay', 'dropping', 'frames']) + expect(plan.body).toEqual(['relay', 'dropping', 'frames']) + }) + + it('is the same list as the body for a literal, which keeps every token', () => { + const plan = planSessionSearchQuery('the foo.ts file') + expect(plan.phrase).toEqual(plan.body) + }) + + it('quotes into one phrase a pasted sentence can actually match', () => { + expect(phraseExpression(planSessionSearchQuery(sentence).phrase)).toBe( + '"The sol review says the PR is not quite merge-ready yet"' + ) + }) +}) diff --git a/src/main/ai-vault-search/session-search-query-planner.ts b/src/main/ai-vault-search/session-search-query-planner.ts index 91e8711fbe3..5dbcbba5f37 100644 --- a/src/main/ai-vault-search/session-search-query-planner.ts +++ b/src/main/ai-vault-search/session-search-query-planner.ts @@ -31,8 +31,15 @@ export type SessionSearchQueryPlan = { truncated: boolean /** Deduplicated index-faithful terms for the OR fallback, incl. identifier pieces. */ terms: string[] - /** Query-order tokens minus stop words: the phrase / AND candidate. */ + /** Query-order tokens minus stop words for prose, all of them for a literal. */ body: string[] + /** + * Query-order tokens exactly as typed, stop words kept: the phrase / AND + * candidate. A sentence pasted out of a transcript is only adjacent in the + * index with its stop words in place, and `unicode61` indexes them, so the + * phrase rung has to search the words the user actually typed. + */ + phrase: string[] } export function isLiteralQuery(query: string): boolean { @@ -95,7 +102,8 @@ export function planSessionSearchQuery( literal, truncated, terms: [...terms, ...extra].slice(0, MAX_TERMS), - body: body.slice(0, MAX_BODY_TERMS) + body: body.slice(0, MAX_BODY_TERMS), + phrase: raw } } diff --git a/src/main/ai-vault-search/session-search-retrieval.ts b/src/main/ai-vault-search/session-search-retrieval.ts index 59fef23e903..f4a963d37b3 100644 --- a/src/main/ai-vault-search/session-search-retrieval.ts +++ b/src/main/ai-vault-search/session-search-retrieval.ts @@ -71,8 +71,7 @@ export class SessionSearchRetrieval { } /** - * The route ladder: phrase, then AND for a literal-looking query, then typo - * repair, then OR. + * The route ladder: phrase, then AND, then typo repair, then OR. * * Repair runs before the OR fallback rather than after it fails. A typo next * to a common word would otherwise be masked: the common word alone retrieves @@ -84,7 +83,10 @@ export class SessionSearchRetrieval { let sessions: SessionRow[] = [] const match = (expression: string): MessageRow[] => { const rows = this.match(expression, scope) - incomplete ||= rows.length >= scope.candidateLimit + // Assigned, not accumulated: only the rung whose rows are returned can + // say whether a cap hid anything. A phrase rung that filled the limit and + // was then discarded describes a row set the answering rung never used. + incomplete = rows.length >= scope.candidateLimit sessions = this.loadSessions( rows.map((row) => row.session_row_id), scope @@ -92,13 +94,13 @@ export class SessionSearchRetrieval { const eligible = new Set(sessions.map((row) => row.id)) return rows.filter((row) => eligible.has(row.session_row_id)) } - const exact = this.literal(plan, match) + const exact = this.phraseThenAnd(plan, match) if (exact) { return { ...exact, plan, incomplete, sessions } } const repaired = this.repair(plan, scope.scope) const effective = repaired ?? plan - const literal = repaired ? this.literal(repaired, match) : null + const literal = repaired ? this.phraseThenAnd(repaired, match) : null const found = literal ?? { rows: match(orExpression(effective.terms)), route: 'or' as const @@ -178,41 +180,59 @@ export class SessionSearchRetrieval { ): SessionSearchQueryPlan | null { const typoRepair = this.typoRepair let changed = false - const body = plan.body.map((term) => { + // Only the body is a candidate for a correction, but the re-plan is fed the + // tokens as typed: re-planning the body alone would hand the phrase rung a + // sentence with its stop words already gone, and `relay dropping frames` + // cannot match the `relay is dropping frames` that is in the transcript. + const repairable = new Set(plan.body.map((term) => term.toLowerCase())) + const phrase = plan.phrase.map((token) => { + if (!repairable.has(token.toLowerCase())) { + return token + } // Repaired inside the scope the search will run in, so a spelling only // tool output carries neither suppresses a repair nor becomes one. - const fix = typoRepair.correct(term, scope) - if (fix && fix !== term.toLowerCase()) { + const fix = typoRepair.correct(token, scope) + if (fix && fix !== token.toLowerCase()) { changed = true return fix } - return term + return token }) // The repair changes spellings, not the query's character: the re-plan is // told what the original decided so a corrected literal keeps every term it // was typed with. - return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null + return changed ? planSessionSearchQuery(phrase.join(' '), plan.literal) : null } - /** Phrase, then AND, for literal-looking queries; null when neither matches. */ - private literal( + /** + * Phrase, then AND, over the tokens as typed; null when neither matches. + * + * Prose runs it too, and not only a literal-looking query. A sentence pasted + * out of a transcript is ordinary words in order, and over OR its common + * words fill the candidate limit with recent sessions long before the old + * session that holds the sentence is reached, so the exact match a user can + * see in front of them comes back missing. + */ + private phraseThenAnd( plan: SessionSearchQueryPlan, match: (expression: string) => MessageRow[] ): { rows: MessageRow[]; route: 'phrase' | 'and' } | null { - if (!plan.literal || plan.body.length === 0) { - return null - } + const tokens = plan.phrase // A one-token literal (`resolveTerminalPath`, `src/a/b.ts`) is its own // phrase: the tokenizer keeps it whole, so the exact token is the cheap, - // precise first try before the identifier pieces fan out over OR. - const phrase = match(phraseExpression(plan.body)) + // precise first try before the identifier pieces fan out over OR. One word + // of prose is not quoting anything, so it goes straight to OR as before. + if (tokens.length === 0 || (tokens.length < 2 && !plan.literal)) { + return null + } + const phrase = match(phraseExpression(tokens)) if (phrase.length > 0) { return { rows: phrase, route: 'phrase' } } - if (plan.body.length < 2) { + if (tokens.length < 2) { return null } - const and = match(andExpression(plan.body)) + const and = match(andExpression(tokens)) return and.length > 0 ? { rows: and, route: 'and' } : null } diff --git a/src/main/ai-vault-search/session-search-snippet-marks.test.ts b/src/main/ai-vault-search/session-search-snippet-marks.test.ts index d4237ce8b6f..81b2f833981 100644 --- a/src/main/ai-vault-search/session-search-snippet-marks.test.ts +++ b/src/main/ai-vault-search/session-search-snippet-marks.test.ts @@ -142,3 +142,31 @@ it('does not cut a snippet at a private-use code point the transcript wrote', as ) expect(snippet).toContain('qqqqq') }) + +it('marks a phrase hit as one run, stop words included', async () => { + harness = await openSessionSearchHarness('ss-snippet-phrase-run') + addSyntheticSession(harness.db, { + id: 1, + text: 'Agent: the code already has several fixes for blank restores, including replaying' + }) + + const result = harness.engine.search({ query: 'the code already has several fixes' }) + expect(result.planner.route).toBe('phrase') + expect(result.hits[0]?.evidence?.snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}the code already has several fixes${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) +}) + +it('marks every typed word of an AND hit, stop words included', async () => { + harness = await openSessionSearchHarness('ss-snippet-and-words') + addSyntheticSession(harness.db, { id: 1, text: 'fixes for the restore path, several of them' }) + + const result = harness.engine.search({ query: 'several fixes for the restore' }) + expect(result.planner.route).toBe('and') + const snippet = result.hits[0]?.evidence?.snippet ?? '' + for (const word of ['several', 'fixes', 'for', 'the', 'restore']) { + expect(snippet).toContain( + `${SESSION_SEARCH_SNIPPET_MARK_OPEN}${word}${SESSION_SEARCH_SNIPPET_MARK_CLOSE}` + ) + } +}) diff --git a/src/main/ai-vault-search/session-search-snippet.ts b/src/main/ai-vault-search/session-search-snippet.ts index 1204cdd10a8..e11dd9dca90 100644 --- a/src/main/ai-vault-search/session-search-snippet.ts +++ b/src/main/ai-vault-search/session-search-snippet.ts @@ -4,11 +4,13 @@ import { SESSION_SEARCH_SNIPPET_MARK_OPEN } from './session-search-engine-types' import { + andExpression, orExpression, + phraseExpression, scopedExpression, type SessionSearchQueryPlan } from './session-search-query-planner' -import type { SessionSearchScope } from './session-search-engine-types' +import type { SessionSearchRoute, SessionSearchScope } from './session-search-engine-types' // What FTS5 wraps a match in before this module rewrites it to the public // marks. Private-use code points, and not `[[`, because two different jobs here @@ -35,15 +37,17 @@ export const EMPTY_SNIPPET: SessionSearchSnippet = { text: '', truncated: false /** * The window of one message that shows why it matched. * - * The expression is the plan's OR form rather than the route's, so a hit found - * through typo repair is marked with the repaired terms it was actually - * retrieved by, and a phrase hit still marks each of its words. + * Marked with the expression the route retrieved by, so a phrase hit is one + * highlight over the words as typed, stop words included, and an OR hit marks + * each term it was found through. The plan is the effective one, so a hit + * found through typo repair is marked with the repaired terms. */ export function sessionSearchSnippet( db: SyncDatabase, scope: SessionSearchScope, rowid: number, - plan: SessionSearchQueryPlan + plan: SessionSearchQueryPlan, + route: SessionSearchRoute ): SessionSearchSnippet { // Why: the identifier shadow column is word soup; a hit that also matches in a // prose column should be shown from there. Column -1 (any column) is the @@ -83,10 +87,8 @@ export function sessionSearchSnippet( JOIN sessions s ON s.id = m.session_row_id WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)` ) - .get(scopedExpression(scope, orExpression(plan.terms)), rowid) as - | Record - | undefined - if (!row) { + .get(scopedExpression(scope, routeExpression(plan, route)), rowid) + if (!isSnippetRow(row)) { return EMPTY_SNIPPET } // A snippet with nothing highlighted tells the user nothing; omit it. @@ -103,6 +105,24 @@ export function sessionSearchSnippet( } } +function isSnippetRow(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + Object.values(value).every((column) => typeof column === 'string') + ) +} + +function routeExpression(plan: SessionSearchQueryPlan, route: SessionSearchRoute): string { + if (route.endsWith('phrase')) { + return phraseExpression(plan.phrase) + } + if (route.endsWith('and')) { + return andExpression(plan.phrase) + } + return orExpression(plan.terms) +} + /** One run of the snippet's own text, or one mark FTS5 put between two runs. */ type SnippetPiece = { kind: 'text'; value: string } | { kind: 'mark'; value: string }