starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-28 16:02:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into brennanb2025/codex-goal-visible

Browse Source
This commit is contained in:
Merge Sim
2026-09-10 22:35:50 -07:00
parent 2d5b0a9436 2ecde717b4
commit 17a86c40ae
195 changed files with 5525 additions and 5610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
-1
View File
@@ -103,7 +103,6 @@ docs/**
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-session-search-query-tuning.md
!docs/reference/agent-status-store.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
config/scripts/session-search-query-benchmark.ts
-192
View File
@@ -1,192 +0,0 @@
import { rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import {
createSessionParseStats,
parseAgentSessionFileCached,
resetSessionParseCacheForTests
} from '../../src/main/ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers'
import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine'
import type {
SessionSearchRequest,
SessionSearchScope
} from '../../src/main/ai-vault-search/session-search-engine-types'
import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import type SyncDatabase from '../../src/main/sqlite/sync-database'
import {
writeSyntheticTranscriptCorpus,
type SyntheticCorpus,
type SyntheticCorpusOptions
} from '../../src/main/ai-vault-search/session-search-synthetic-corpus'
import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
// What a query costs, and what the session candidate limit buys. Everything
// runs through the real store and the real engine over a synthetic corpus;
// never point this at a real transcript tree.
const WARMUP = 5
const SAMPLES = 25
// One query per rung the ladder can take, plus the two shapes that skip it.
const QUERIES: { name: string; request: SessionSearchRequest }[] = [
{ name: 'phrase', request: { query: '"terminal reattach"' } },
{ name: 'identifier', request: { query: 'resolveTerminalPath' } },
{ name: 'path', request: { query: 'src/main/ai-vault/session-transcript-reader.ts' } },
{ name: 'prose', request: { query: 'why is the daemon snapshot stale' } },
{ name: 'typo', request: { query: 'reattahc worktre' } },
{ name: 'common-term', request: { query: 'index' } },
{ name: 'operator-only', request: { query: 'repo:app-3' } },
{ name: 'scoped', request: { query: 'worktree', filters: { scopePaths: ['/repo/app-3'] } } }
]
type Timing = { p50: number; p95: number }
function percentile(sorted: readonly number[], fraction: number): number {
const at = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))
return Math.round((sorted[at] ?? 0) * 100) / 100
}
function timing(samples: number[]): Timing {
const sorted = [...samples].sort((left, right) => left - right)
return { p50: percentile(sorted, 0.5), p95: percentile(sorted, 0.95) }
}
function time(engine: SessionSearchEngine, request: SessionSearchRequest): number {
const started = performance.now()
engine.search(request)
return performance.now() - started
}
async function indexCorpus(
options: SyntheticCorpusOptions
): Promise<{ corpus: SyntheticCorpus; db: SyncDatabase; release: () => void }> {
resetSessionParseCacheForTests()
const corpus = await writeSyntheticTranscriptCorpus(options)
const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => {
throw error
})
const unregister = registerSessionSearchIndexConsumer(store)
const stats = createSessionParseStats()
for (const path of corpus.files) {
await parseAgentSessionFileCached(
await sessionCandidate('claude', path),
process.platform,
stats
)
}
return {
corpus,
// The handle a composed reader gets. Every read here is one synchronous
// statement, which is the contract that comes with it.
db: store.connection,
release: () => {
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
}
}
/** Per-query and overall latency for one scope. */
function scopeReport(db: SyncDatabase, scope: SessionSearchScope): Record<string, unknown> {
const engine = new SessionSearchEngine(db)
const everything: number[] = []
const perQuery: Record<string, Timing & { hits: number; route: string }> = {}
for (const { name, request } of QUERIES) {
const scoped = { ...request, scope }
for (let run = 0; run < WARMUP; run++) {
engine.search(scoped)
}
const samples = Array.from({ length: SAMPLES }, () => time(engine, scoped))
everything.push(...samples)
const result = engine.search(scoped)
perQuery[name] = { ...timing(samples), hits: result.hits.length, route: result.planner.route }
}
return { ...timing(everything), perQuery }
}
/**
* The candidate limit only costs anything once there are more matching sessions
* than the limit, so this runs over many short sessions rather than the wide
* corpus above. Limits are interleaved sample by sample: run back to back, the
* first configuration pays for every page the OS cache had not seen yet and the
* ordering alone moves p95 by more than the limit does.
*/
function candidateSweep(db: SyncDatabase, limits: readonly number[]): Record<string, unknown> {
const request: SessionSearchRequest = { query: 'index', limit: 20 }
const engines = new Map(
limits.map((limit) => [limit, new SessionSearchEngine(db, { sessionCandidateLimit: limit })])
)
const samples = new Map(limits.map((limit) => [limit, [] as number[]]))
for (let run = 0; run < WARMUP; run++) {
for (const engine of engines.values()) {
engine.search(request)
}
}
for (let run = 0; run < SAMPLES; run++) {
for (const limit of limits) {
samples.get(limit)!.push(time(engines.get(limit)!, request))
}
}
const report: Record<string, unknown> = {}
for (const limit of limits) {
const result = engines.get(limit)!.search(request)
report[String(limit)] = {
...timing(samples.get(limit)!),
truncated: result.truncated.candidates,
// Pages a caller could walk before the limit stops handing out sessions.
reachablePages: Math.ceil(limit / (request.limit ?? 20))
}
}
return report
}
const wide = await indexCorpus({ sessions: Number(process.env.SESSIONS ?? 40) })
let report: string
try {
const scope = {
all: scopeReport(wide.db, 'all'),
conversation: scopeReport(wide.db, 'conversation')
}
wide.release()
await rm(wide.corpus.root, { recursive: true, force: true })
// Many short sessions: what makes the candidate limit binding is the session
// count, not the byte count.
const many = await indexCorpus({ sessions: 2500, turnsPerSession: 1, seed: 7 })
try {
report = JSON.stringify(
{
scopeCorpus: {
sessions: wide.corpus.files.length,
transcriptMb: Math.round((wide.corpus.transcriptBytes / 1024 / 1024) * 100) / 100,
messages: wide.corpus.messageCount
},
scope,
candidateCorpus: {
sessions: many.corpus.files.length,
transcriptMb: Math.round((many.corpus.transcriptBytes / 1024 / 1024) * 100) / 100
},
candidateSweep: candidateSweep(many.db, [200, 600, 1200, 2400])
},
null,
2
)
} finally {
many.release()
await rm(many.corpus.root, { recursive: true, force: true })
}
} catch (error) {
await rm(wide.corpus.root, { recursive: true, force: true })
throw error
}
// Why a file as well as stdout: a runner that intercepts console output
// (vitest does) would otherwise swallow the whole report.
const out = process.env.BENCH_OUT
if (out) {
await writeFile(out, `${report}\n`)
}
console.log(report)
config/scripts/session-search-scope-benchmark.ts
-205
View File
@@ -1,205 +0,0 @@
import { rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import {
createSessionParseStats,
parseAgentSessionFileCached,
resetSessionParseCacheForTests
} from '../../src/main/ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers'
import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine'
import type {
SessionSearchRequest,
SessionSearchScope
} from '../../src/main/ai-vault-search/session-search-engine-types'
import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
import type SyncDatabase from '../../src/main/sqlite/sync-database'
import { writeToolHeavyCorpus, type ToolHeavyCorpus } from './session-search-tool-heavy-corpus'
// What each scope costs on an index the size of a real transcript tree.
//
// The 10.5 MB corpus in `session-search-query-benchmark.ts` sizes the route
// ladder; this one sizes the corpus. `conversation` is a column filter over the
// one FTS table rather than a second table of its own, and the whole cost of
// that decision is how much of `messages_fts` a conversation query has to read
// past — which is set by how much of a transcript is tool output.
//
// Synthetic, always: this must never be pointed at a real transcript.
const WARMUP = 5
/** Conversation-shaped queries; every term is one the prose actually uses. */
const QUERIES = [
'terminal reattach',
'stale snapshot',
'daemon cursor',
'worktree index',
'publish transaction',
'relay daemon',
'session cursor',
'because stale',
'terminal worktree',
'index snapshot',
'reattach cursor',
'transaction relay',
'snapshot session',
'daemon publish',
'worktree terminal',
'cursor index',
'stale relay',
'session transaction',
'publish snapshot',
'reattach daemon'
]
async function indexCorpus(
corpus: ToolHeavyCorpus
): Promise<{ db: SyncDatabase; release: () => void }> {
resetSessionParseCacheForTests()
const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => {
throw error
})
const unregister = registerSessionSearchIndexConsumer(store)
const stats = createSessionParseStats()
for (const path of corpus.files) {
await parseAgentSessionFileCached(
await sessionCandidate('claude', path),
process.platform,
stats
)
}
return {
// The store's own handle, which is what a composed reader gets: every
// retrieval is one synchronous statement, so nothing pins a WAL snapshot.
db: store.connection,
release: () => {
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
}
}
type Timing = { p50: number; p95: number }
function timing(samples: readonly number[]): Timing {
const sorted = [...samples].sort((left, right) => left - right)
const at = (fraction: number): number => {
const index = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))
return Math.round((sorted[index] ?? 0) * 100) / 100
}
return { p50: at(0.5), p95: at(0.95) }
}
/**
* The query sets, one per rung of the ladder the engine may take.
*
* Which rung each one reaches is not forced, it is observed: samples are
* bucketed by the route the engine reports, so the table says what was measured
* rather than what was intended, and a query that lands on a different rung
* than expected shows up as a bucket rather than as a wrong number.
*/
function queries(): string[] {
const run = (index: number, length: number): string =>
Array.from({ length }, (_unused, step) => QUERIES[(index + step) % QUERIES.length]).join(' ')
return [
// Two terms, unquoted: not literal, so straight to OR.
...QUERIES,
// Two terms, quoted: literal, and on this corpus any two of fourteen words
// sit next to each other somewhere, so the phrase rung answers.
...QUERIES.map((query) => `"${query}"`),
// Eight terms, quoted: an ordered run that long does not occur in 105 MB of
// draws from fourteen words, so the phrase rung misses and AND answers.
...QUERIES.map((_query, index) => `"${run(index, 4)}"`)
]
}
type Bucket = { samples: number[]; hits: number }
/**
* Both scopes over the same queries, interleaved scope by scope: run back to
* back, the first one pays for every page the OS cache had not seen and the
* ordering moves p95 more than the scope does.
*/
function scopeReport(db: SyncDatabase): Record<string, unknown> {
const engine = new SessionSearchEngine(db)
const scopes: SessionSearchScope[] = ['all', 'conversation']
const requests: SessionSearchRequest[] = queries().map((query) => ({ query }))
const buckets = new Map<string, Bucket>()
for (let run = 0; run < WARMUP; run++) {
for (const scope of scopes) {
for (const request of requests) {
engine.search({ ...request, scope })
}
}
}
for (const request of requests) {
for (const scope of scopes) {
const started = performance.now()
const result = engine.search({ ...request, scope })
const elapsed = performance.now() - started
const key = `${result.planner.route}/${scope}`
const bucket = buckets.get(key) ?? { samples: [], hits: 0 }
bucket.samples.push(elapsed)
bucket.hits += result.hits.length
buckets.set(key, bucket)
}
}
const report: Record<string, unknown> = {}
for (const [key, bucket] of [...buckets].sort(([left], [right]) => left.localeCompare(right))) {
report[key] = { ...timing(bucket.samples), samples: bucket.samples.length, hits: bucket.hits }
}
return report
}
/** Bytes the FTS table occupies, which is the cost the deleted second table saved. */
function indexBytes(db: SyncDatabase): Record<string, number> | { unavailable: string } {
try {
const sum = (where: string, ...values: string[]): number =>
Number(
(
db
.prepare(`SELECT COALESCE(SUM(pgsize),0) AS bytes FROM dbstat ${where}`)
.get(...values) as { bytes: number }
).bytes
)
return { total: sum(''), messagesFts: sum('WHERE name LIKE ?', 'messages_fts%') }
} catch {
// dbstat is a compile-time option; the latency numbers stand without it.
return { unavailable: 'no dbstat' }
}
}
const corpus = await writeToolHeavyCorpus({
targetBytes: Number(process.env.CORPUS_MB ?? 100) * 1024 * 1024,
toolShare: Number(process.env.TOOL_SHARE ?? 0.9)
})
let report: string
const indexed = await indexCorpus(corpus)
try {
report = JSON.stringify(
{
corpus: {
sessions: corpus.files.length,
transcriptMb: Math.round((corpus.transcriptBytes / 1024 / 1024) * 100) / 100,
toolShareOfMessageText:
Math.round((corpus.toolBytes / (corpus.toolBytes + corpus.proseBytes)) * 1000) / 1000
},
indexBytes: indexBytes(indexed.db),
route: scopeReport(indexed.db)
},
null,
2
)
} finally {
indexed.release()
await rm(corpus.root, { recursive: true, force: true })
}
const out = process.env.BENCH_OUT
if (out) {
await writeFile(out, `${report}\n`)
}
console.log(report)
config/scripts/session-search-tool-heavy-corpus.ts
-152
View File
@@ -1,152 +0,0 @@
import { mkdtemp, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
// The corpus the scope benchmark runs over. Written here rather than by
// `session-search-synthetic-corpus.ts` because what it costs to answer a
// conversation query out of the one FTS table turns on the property that
// generator fixes: how much of a transcript is tool output.
//
// Synthetic, always. This must never be pointed at a real transcript.
const PROSE = [
'terminal',
'reattach',
'worktree',
'the',
'index',
'cursor',
'publish',
'transaction',
'relay',
'daemon',
'snapshot',
'because',
'stale',
'session'
]
// Tool output is paths, hashes and log lines — and the same words the
// conversation uses, because a `rg` over this repository prints them. That
// overlap is what the benchmark turns on: it is what makes a conversation
// term's posting list carry rows the column filter then has to discard. A tool
// vocabulary disjoint from the prose would leave nothing to discard and measure
// the wrong thing.
const TOOL_ONLY = [
'src/main/ai-vault/session-transcript-reader.ts',
'node_modules/.pnpm/typescript@5.9.2',
'0x00007ff8',
'ENOENT',
'drwxr-xr-x',
'2026-09-10T00:00:00.000Z',
'sha256:9f2c1a',
'chunk-VHQ4NWQK.js',
'warning:',
'resolveTerminalPath',
'byteOffset',
'MAX_RETRIES'
]
// Half the tool tokens are conversation words. Deliberately pessimistic: the
// more of a query term lives in `tool_text`, the more the column filter costs,
// so a number measured here holds on a real transcript tree.
const TOOL = [...PROSE, ...TOOL_ONLY]
function mulberry32(seed: number): () => number {
let state = seed >>> 0
return () => {
state = (state + 0x6d2b79f5) >>> 0
let t = Math.imul(state ^ (state >>> 15), 1 | state)
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
function words(random: () => number, vocabulary: readonly string[], count: number): string {
const out: string[] = []
for (let index = 0; index < count; index++) {
out.push(vocabulary[Math.floor(random() * vocabulary.length)]!)
}
return out.join(' ')
}
export type ToolHeavyCorpus = {
root: string
files: string[]
transcriptBytes: number
toolBytes: number
proseBytes: number
}
/**
* Claude JSONL transcripts whose tool output is `toolShare` of the message text.
* One turn is a user question, an assistant answer, a tool call and its output;
* only the last one grows with the share.
*/
export async function writeToolHeavyCorpus(args: {
targetBytes: number
toolShare: number
seed?: number
}): Promise<ToolHeavyCorpus> {
const random = mulberry32(args.seed ?? 11)
const root = await mkdtemp(join(tmpdir(), 'orca-search-convfts-'))
const files: string[] = []
const proseWordsPerTurn = 160
// Tool and prose words are not the same length, so the share is over bytes.
const proseBytesPerTurn = proseWordsPerTurn * 6
const toolWordCount = Math.max(
1,
Math.round((proseBytesPerTurn * args.toolShare) / (1 - args.toolShare) / 22)
)
let transcriptBytes = 0
let toolBytes = 0
let proseBytes = 0
for (let session = 0; transcriptBytes < args.targetBytes; session++) {
const sessionId = `00000000-0000-4000-8000-${String(session).padStart(12, '0')}`
const lines: string[] = []
for (let turn = 0; turn < 40; turn++) {
const at = new Date(1740000000000 + turn * 60_000).toISOString()
const question = words(random, PROSE, 40)
const answer = words(random, PROSE, proseWordsPerTurn - 40)
const output = words(random, TOOL, toolWordCount)
proseBytes += Buffer.byteLength(question) + Buffer.byteLength(answer)
toolBytes += Buffer.byteLength(output)
lines.push(
JSON.stringify({
type: 'user',
sessionId,
timestamp: at,
cwd: `/repo/app-${session % 7}`,
gitBranch: 'main',
message: { role: 'user', content: question }
}),
JSON.stringify({
type: 'assistant',
sessionId,
timestamp: at,
message: {
role: 'assistant',
model: 'claude-fable-5',
content: [
{ type: 'text', text: answer },
{ type: 'tool_use', name: 'Bash', input: { command: 'rg needle' } }
]
}
}),
JSON.stringify({
type: 'user',
sessionId,
timestamp: at,
message: {
role: 'user',
content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: output }]
}
})
)
}
const path = join(root, `${sessionId}.jsonl`)
const body = `${lines.join('\n')}\n`
await writeFile(path, body)
transcriptBytes += Buffer.byteLength(body)
files.push(path)
}
return { root, files, transcriptBytes, toolBytes, proseBytes }
}
docs/reference/agent-session-search-query-tuning.md
-218
View File
@@ -1,218 +0,0 @@
# Agent session search: query tuning
What a search costs, and what the knobs in `src/main/ai-vault-search/session-search-engine.ts`
buy. Every number here comes from `config/scripts/session-search-query-benchmark.ts`
over the synthetic corpus in `session-search-synthetic-corpus.ts`, except the
`conversation_fts` shoot-out, which writes its own corpus because the answer
turns on how much of a transcript is tool output. Nothing in this file was
measured against a real transcript, and neither benchmark must ever be pointed
at one.
## Running it
The benchmark is a top-level-await module that imports the main-process tree by
extensionless path, so it needs a bundler-backed runner rather than bare `node`:
```sh
cat > src/main/ai-vault-search/zz-bench.test.ts <<'EOF'
import { it } from 'vitest'
it('runs', { timeout: 1_800_000 }, async () => {
await import('../../../config/scripts/session-search-query-benchmark')
})
EOF
BENCH_OUT=/tmp/ss-query-bench.json pnpm test src/main/ai-vault-search/zz-bench.test.ts
rm src/main/ai-vault-search/zz-bench.test.ts
```
The `conversation_fts` shoot-out below runs the same way, importing
`config/scripts/session-search-conversation-fts-benchmark` instead, with
`CORPUS_MB` and `TOOL_SHARE` to size and shape its corpus. `config/scripts` is
not inside any typecheck project, so while that throwaway test exists `tsc`
reports TS6307 for each script it pulls in; delete it and the run is clean
again.
`BENCH_OUT` exists because vitest intercepts `console.log`; the report is written
to that path as well as printed.
## Scope: what the second FTS table buys a reader
Corpus: 40 synthetic Claude transcripts, 10.5 MB, 9,600 messages, indexed through
the real store. Eight queries, one per rung of the route ladder plus the two
shapes that skip it; 5 warm-up runs and 25 samples each. Apple silicon, warm page
cache, machine otherwise idle. Milliseconds, and p95 over 25 samples moves
several milliseconds run to run if anything else is competing for the disk.
| Scope | p50 | p95 |
| -------------- | ---- | ---- |
| `all` | 7.22 | 8.94 |
| `conversation` | 5.33 | 7.86 |
Per query, `all` then `conversation` (p50 / p95):
| Query | `all` | `conversation` |
| ------------------------------------------------ | ------------ | -------------- |
| `"terminal reattach"` (phrase) | 5.24 / 8.42 | 2.97 / 3.24 |
| `resolveTerminalPath` (identifier) | 7.55 / 8.94 | 6.47 / 6.72 |
| `src/main/…/session-transcript-reader.ts` (path) | 8.69 / 10.12 | 7.78 / 8.04 |
| `why is the daemon snapshot stale` (prose) | 7.84 / 8.57 | 5.90 / 7.01 |
| `reattahc worktre` (typo repair) | 7.30 / 7.39 | 5.53 / 5.89 |
| `index` (common term) | 5.45 / 5.66 | 3.81 / 4.02 |
| `repo:app-3` (operator only) | 0.12 / 0.16 | 0.10 / 0.10 |
| `worktree` scoped to one cwd | 1.47 / 1.63 | 1.25 / 1.49 |
Reading it:
- `conversation` is about 1.4x faster at p50 and 1.1x at p95, and it is a column
filter over the same table rather than a table of its own. Narrowing to the
two prose columns is what buys the gap: fewer postings to score. It is also
the scope where a match is something a person wrote rather than something a
tool printed.
- A `scopePaths` query is the cheapest real search on the page. It is the one
narrowing SQL can express exactly, so it seeks `sessions_cwd_key` and hands
ranking a small candidate set.
- The operator-only figure is a floor, not a typical cost. `repo:` and `path:`
are applied in JS over retrieved rows (see `session-search-row-filter` for why
they cannot be pushed into SQL), so their cost tracks how many sessions the
walk has to read before it fills a candidate set. This corpus has 40 sessions,
which is one page of that walk; an index where few sessions match the operator
will read up to the ceiling in `session-search-retrieval` instead.
## What the conversation scope costs at real corpus size
`conversation` was a second FTS table holding a copy of the two prose columns.
It is a column filter now — `{user_text assistant_text}: (…)` with bm25 weights
that zero the other two — and PR 2 deleted the table on the strength of the
shoot-out this section used to hold: the filter came in at 1.16-1.36x the p95 of
the dedicated table, under the 2x bar, while the table cost a tenth of the index
to maintain. What follows is what the shipped schema actually does, measured
again on the same corpus after the table went and tool rows were capped.
Corpus: Claude transcripts from `config/scripts/session-search-tool-heavy-corpus.ts`,
105 MB, indexed through the real store, at two points in the 80-97% band a real
transcript tree sits in. Half the tokens in tool output are words the
conversation also uses, so a conversation term really does have postings the
filter must discard. Twenty queries per rung, both scopes interleaved query by
query, warm cache; `config/scripts/session-search-scope-benchmark.ts`, run twice.
| Tool share | Rung | `all` p50 / p95 | `conversation` p50 / p95 |
| ---------- | ------ | --------------- | ------------------------ |
| 86% | phrase | 16.69 / 17.48 | 13.08 / 13.52 |
| 86% | or | 31.91 / 35.74 | 22.25 / 23.87 |
| 86% | and | 70.04 / 74.00 | 53.47 / 59.39 |
| 93% | phrase | 9.14 / 13.36 | 7.23 / 8.51 |
| 93% | or | 16.46 / 18.70 | 12.34 / 14.88 |
| 93% | and | 39.65 / 43.44 | 31.05 / 32.92 |
Three things to read out of it.
**The filter is a win, not a cost.** Every rung is faster narrow than wide, by
1.2x to 1.4x at p50. The shoot-out compared the filter against a table built for
exactly this query; against the wide table it replaces, it does what the second
table did, which is read fewer postings.
**The `and` rung is where the corpus size shows.** Those queries are eight terms,
chosen so no ordered run that long occurs and the phrase rung has to miss; a
real two-term AND sits nearer the phrase row. It is also the noisiest: the
second run's p95 reached 140 ms on one bucket, which is what twenty samples of a
70 ms query buys. Read the p50 column.
**The index is far smaller than the shoot-out's was.** 57 MB at 93% tool output
and 103 MB at 86%, against roughly 150 MB for `messages_fts` alone before PR 2
capped an indexed tool row at 3,072 characters. Most of a tool-heavy transcript
is now not in the index at all, which moves every number above and is the larger
effect of the two.
What is **not** measured here is relevance, and the column filter does carry one
ranking difference the deleted table did not. FTS5's bm25 normalises by the
whole row's length and has no per-column length, so two rows with identical
prose score differently when one also holds tool output. The rowid set is
unchanged, which is what the deletion was decided on; the order within it can
move. `session-search-engine.test.ts` pins the direction.
## `sessionCandidateLimit`
The reviewer's F13: this is a tunable default, not a constant. It bounds how many
sessions the SQL hands ranking, so it bounds both retrieval cost and how deep a
caller can page before the answer simply stops.
The limit only costs anything once more sessions match than the limit allows, so
this is measured over a second corpus: 2,500 one-turn transcripts, 10.9 MB, every
one of them matching the query. Limits are interleaved sample by sample, because
run back to back the first configuration pays for every page the OS cache had not
seen and the ordering alone moves p95 further than the limit does.
| Limit | p50 | p95 | Pages of 20 a caller can reach |
| ----- | ----- | ----- | ------------------------------ |
| 200 | 6.85 | 7.21 | 10 |
| 600 | 7.93 | 8.36 | 30 |
| 1200 | 9.55 | 10.53 | 60 |
| 2400 | 12.32 | 13.45 | 120 |
600 is the default: it costs about 16% over 200 at p50 and buys three times the
reachable depth, and the curve only turns steep past 1200. A host with a much
larger index can raise it; the result's `truncated.candidates` says when the limit
was the thing that cut the answer, so a caller never has to guess.
What is **not** measured here is relevance. These numbers say what a limit costs,
not what it retrieves. The MRR figures quoted in the BM25 weights
(`session-search-retrieval.ts`) and in the identifier shadow column
(`session-search-identifier-split.ts`) come from the original retrieval shoot-out
on real transcripts and are not reproducible from this repository. Any change to
the limit justified on relevance grounds needs an eval set, not this benchmark.
## What typo repair costs
The repair is the one rung whose cost tracks the size of the vocabulary rather
than the size of a result. It only runs for a term the scope has no posting for,
so an ordinary query never pays it; a query of nonsense pays it once per term.
Measured over a synthetic vocabulary of 1.6 M distinct terms, every term in two
rows so none is filtered out:
| Query | p50 |
| -------------------------------------- | ------ |
| one known term (no repair) | 11 ms |
| one unknown term | 10 ms |
| 39 unknown 12-character terms (480 ch) | 387 ms |
| 12 unknown 40-character terms | 99 ms |
Two things follow. The cost is linear in unknown terms and in vocabulary size,
and `search` is synchronous, so a 512-character query of nonsense holds the
thread for a third of a second on an index that large. And the scoped-count fix
made this cheaper rather than dearer — it was 737 ms before — because ordering
the vocabulary scan by term drops the sort that ordering by `doc` required, and
the counts it added are at most eight bounded probes per prefix. A cap on
unknown terms per query is recorded as a follow-up in the split plan.
## Page warmup, dropped
PR 2 deferred `warm()` — a sliced read of `messages` that pulls its pages into
the OS cache before the first query — to whoever knew which pages a read
touches. It is not re-added here, for two reasons. The measurement that
justified it (first query 1.3 s to 0.45 s) was on a 4 GB index, and neither
corpus in this file is within an order of magnitude of that, so PR 4 cannot
show a win: removing the call moved the 10.5 MB corpus's p50 by less than the
run-to-run spread. And it is a cancellable background pass, which needs an owner
with a lifecycle; a query library that holds no timers has nothing to hang the
`stopped()` on, and a fire-and-forget async read from a synchronous `search` is
a rejection nothing can supervise. It belongs with the indexer in PR 3b, which
already owns starting and stopping work.
## Not settled here
Which process may open, unlink and rebuild the index is PR 3b's decision. A
second handle that finds an older schema version replaces the file while a live
store keeps answering from the unlinked inode, and this PR is what first makes
that reachable, because it is the first thing that reads. What PR 4 does is
refuse to make it worse. The engine carries its own schema — the vocabulary, the
query log and the generation triggers — and re-creates whatever of it is missing
on every search, so a dropped object heals rather than degrading.
The one it cannot re-create is the vocabulary's source, because `messages_fts`
is the store's. With one FTS table that is also the end of the degrade: there is
no second corpus to answer from, so an engine over an index mid-rebuild names
typo repair as unavailable and then fails on the table it cannot read, which is
the honest outcome — an empty page would read as an answer. `unavailable` can
therefore no longer be reported alongside a successful search, and PR 5 should
decide whether the field survives into the contract; it becomes reachable again
the day something opens the index read-only.
src/main/ai-vault-search/session-search-clock.ts
+24
View File
@@ -0,0 +1,24 @@
// Why injected rather than the globals: every freshness guarantee this indexer
// makes is "within one reconcile interval", and a guarantee stated in wall time
// is only a claim until a test can advance the clock and watch it hold.
/** Opaque to the indexer; a fake clock hands back whatever it likes. */
export type SessionSearchTimerHandle = object | number
export type SessionSearchClock = {
now(): number
setTimeout(callback: () => void, ms: number): SessionSearchTimerHandle
clearTimeout(handle: SessionSearchTimerHandle): void
}
export const systemSessionSearchClock: SessionSearchClock = {
now: () => Date.now(),
setTimeout: (callback, ms) => {
const timer = setTimeout(callback, ms)
// Nothing here should hold the process open: the index is a cache, and a
// pending reconcile is never a reason to keep a CLI or a child alive.
timer.unref?.()
return timer
},
clearTimeout: (handle) => clearTimeout(handle as NodeJS.Timeout)
}
src/main/ai-vault-search/session-search-degraded-roots.ts
+88
View File
@@ -0,0 +1,88 @@
import type { AiVaultScanIssue } from '../../shared/ai-vault-types'
import type { SessionSearchDirectoryReader } from './session-search-directory-listings'
/** A scan root this pass could not read through, and what stopped it. */
export type SessionSearchDegradedRoot = { root: string; reason: string }
/**
* Roots a pass could not read, derived from that pass alone.
*
* There is no root-health state machine any more and nothing is carried between
* passes: "degraded" now means one of two things this pass observed, both of
* which are readdir results.
*
* 1. Discovery recorded a scan issue against the root itself — a stalled WSL
* distro, a gate refusal, an unreadable tree.
* 2. The retirement walk could not prove a file the index holds under that root
* either present or gone, because a directory between the file and the root
* refused to list, or because the root itself is not there.
* 3. A root that yielded no transcripts refuses to list at all. The file walker
* swallows a readdir failure and returns, so without this an EACCES root and
* an agent that was never installed both arrive as "no files" — reporting
* the first as an empty index is the loss-of-contact-as-absence mistake
* docs/reference/ssh-execution-boundary.md forbids.
*
* The second is what reports a detached volume, and it needs no memory of
* previous passes: the evidence is the index's own rows plus this pass's
* readdir errors. A root the index holds nothing under and cannot list is
* reported by the third; a root that is simply missing is not reported at all,
* because that is what an agent nobody installed looks like.
*/
export function scanIssueDegradedRoots(
roots: readonly string[],
issues: readonly AiVaultScanIssue[]
): SessionSearchDegradedRoot[] {
const degraded = new Map<string, string>()
for (const issue of issues) {
// 'notice' rows are scanner commentary; a per-file failure is not a root's.
if (issue.kind !== 'notice' && roots.includes(issue.path)) {
degraded.set(issue.path, issue.message)
}
}
return [...degraded].map(([root, reason]) => ({ root, reason }))
}
/** One entry per root, first reason kept, so a pass reports each root once. */
export function mergeDegradedRoots(
...groups: readonly (readonly SessionSearchDegradedRoot[])[]
): SessionSearchDegradedRoot[] {
const merged = new Map<string, string>()
for (const group of groups) {
for (const degraded of group) {
if (!merged.has(degraded.root)) {
merged.set(degraded.root, degraded.reason)
}
}
}
return [...merged].map(([root, reason]) => ({ root, reason }))
}
// A missing root is not a broken one: an uninstalled agent's root answers
// exactly this, and the index holding rows under it is what the retirement
// walk reports instead.
const MISSING_ROOT = new Set(['ENOENT', 'ENOTDIR'])
/**
* Roots that yielded no transcripts and cannot be listed either.
*
* Only roots a pass found empty are read: one that returned files is readable
* by construction. The read shares the pass's listing cache, so a root the
* retirement walk also has to ask about costs one readdir between them.
*/
export async function unreadableRoots(
roots: readonly string[],
listings: SessionSearchDirectoryReader,
signal?: AbortSignal
): Promise<SessionSearchDegradedRoot[]> {
const degraded: SessionSearchDegradedRoot[] = []
for (const root of roots) {
if (signal?.aborted) {
break
}
const listing = await listings.namesIn(root, signal)
if (!listing.listed && !(listing.code !== null && MISSING_ROOT.has(listing.code))) {
degraded.push({ root, reason: listing.message })
}
}
return degraded
}
src/main/ai-vault-search/session-search-deleted-sources.test.ts
+356
View File
@@ -0,0 +1,356 @@
import { chmod, mkdir, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { parserPublishesMessages } from '../ai-vault/session-scanner-agent-parser'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import { retireDeletedSessionSearchSources } from './session-search-deleted-sources'
import {
SessionSearchDirectoryListings,
type SessionSearchDirectoryListing,
type SessionSearchDirectoryReader
} from './session-search-directory-listings'
import {
openSessionSearchIndexerHarness,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
import { SessionSearchStore } from './session-search-store'
// The invariants this file exists to pin are written at the top of
// session-search-deleted-sources.ts. Each one is named in the tests below.
const CAN_DENY_READ = process.platform !== 'win32' && process.getuid?.() !== 0
let harness: SessionSearchIndexerHarness
let store: SessionSearchStore
let removed: string[]
beforeEach(async () => {
resetTranscriptConsumersForTests()
harness = await openSessionSearchIndexerHarness('ss-deleted-sources')
removed = []
store = new SessionSearchStore(harness.databasePath)
// Only the removal matters here; the store's own removal path has its own tests.
store.removeFile = (path: string) => removed.push(path)
})
afterEach(async () => {
store.close()
await harness.cleanup()
})
/** A reader that answers with whatever a stalled mount would, per directory. */
function readerAnswering(
answers: Record<string, SessionSearchDirectoryListing>
): SessionSearchDirectoryReader {
return {
namesIn: (directory) =>
Promise.resolve(
answers[directory] ?? { listed: false, code: 'ENOENT', message: 'no such directory' }
)
}
}
function retire(
paths: readonly string[],
options: {
roots?: readonly string[]
emptiedRoots?: ReadonlySet<string>
enumeratedContainers?: ReadonlyMap<string, ReadonlySet<string>>
listings?: SessionSearchDirectoryReader
directoryLimit?: number
} = {}
) {
return retireDeletedSessionSearchSources({
store,
paths,
roots: options.roots ?? [harness.roots.claudeProjectsDir ?? ''],
emptiedRoots: options.emptiedRoots,
enumeratedContainers: options.enumeratedContainers,
listings: options.listings ?? new SessionSearchDirectoryListings(),
directoryLimit: options.directoryLimit
})
}
// I4: a file the user deleted retires on the first pass that proves it, with no
// waiting period, because its directory listed and it was not in the listing.
it('retires a deleted file the moment its own directory lists without it', async () => {
const kept = join(harness.claudeProjectDir, 'kept.jsonl')
await mkdir(harness.claudeProjectDir, { recursive: true })
await writeFile(kept, '{}')
const deleted = join(harness.claudeProjectDir, 'deleted.jsonl')
const result = await retire([kept, deleted])
expect(result.retired).toEqual([deleted])
expect(removed).toEqual([deleted])
// A file that is still there is settled, not watched: it is neither retired
// nor carried into the next pass as unfinished business.
expect(result.unverifiable).toEqual([])
expect(result.degradedRoots).toEqual([])
})
// I4, the other shape: the directory itself is gone, so the question moves up
// one level and the root answers it.
it('retires a whole project directory the user deleted', async () => {
const sibling = join(harness.roots.claudeProjectsDir ?? '', 'other', 'kept.jsonl')
await mkdir(join(harness.roots.claudeProjectsDir ?? '', 'other'), { recursive: true })
await writeFile(sibling, '{}')
const gone = join(harness.claudeProjectDir, 'inside-a-deleted-project.jsonl')
const result = await retire([gone])
expect(result.retired).toEqual([gone])
expect(result.unverifiable).toEqual([])
})
// I1 and I2: a root that is not there proves nothing. The walk stops at the
// configured root and never asks what is above it, so a home directory on an
// unmounted volume — the shape a detached drive or a dropped SSH mount takes —
// leaves every row exactly where it was.
it('keeps every row under a root that is not there', async () => {
const root = harness.roots.claudeProjectsDir ?? ''
const held = [join(harness.claudeProjectDir, 'one.jsonl'), join(root, 'flat.jsonl')]
const result = await retire(held)
expect(result.retired).toEqual([])
expect(result.unverifiable).toEqual(held)
// The root is named, once, so a caller can say which tree is unreachable.
expect(result.degradedRoots).toEqual([{ root, reason: `${root} could not be listed.` }])
})
// I3: the same answer with no memory at all. Nothing here is carried from a
// previous pass, which is what makes the first sweep after a restart — when a
// volume is most likely to be missing — behave like every other pass.
it('keeps a missing root on a pass that has seen nothing before it', async () => {
const root = harness.roots.claudeProjectsDir ?? ''
const held = join(harness.claudeProjectDir, 'one.jsonl')
const first = await retire([held], { emptiedRoots: new Set() })
const second = await retire([held], { emptiedRoots: new Set() })
expect([first.retired, second.retired]).toEqual([[], []])
expect(second.degradedRoots.map((one) => one.root)).toEqual([root])
})
// I2: an unreadable directory is not an empty one. EACCES stops the walk where
// it is rather than being walked up like a missing component.
it.skipIf(!CAN_DENY_READ)('keeps rows under a directory that refuses to list', async () => {
const blocked = join(harness.roots.claudeProjectsDir ?? '', 'blocked')
await mkdir(blocked, { recursive: true })
const hidden = join(blocked, 'hidden.jsonl')
await writeFile(hidden, '{}')
await chmod(blocked, 0o000)
try {
const result = await retire([hidden])
expect(result.retired).toEqual([])
expect(result.unverifiable).toEqual([hidden])
expect(result.degradedRoots.map((one) => one.root)).toEqual([harness.roots.claudeProjectsDir])
} finally {
await chmod(blocked, 0o755)
}
})
// I2, without needing a filesystem that can produce it: a stalled network mount
// answers EIO or a WSL gate refusal, and neither is ENOENT. This is the SSH and
// WSL case — loss of contact is never evidence of absence.
it('keeps rows when a directory answers with a transport failure', async () => {
const root = harness.roots.claudeProjectsDir ?? ''
const held = join(harness.claudeProjectDir, 'one.jsonl')
for (const listing of [
{ listed: false as const, code: 'EIO', message: 'input/output error' },
{ listed: false as const, code: 'ETIMEDOUT', message: 'the mount stopped answering' },
{ listed: false as const, code: null, message: 'The distro stopped responding.' }
]) {
const result = await retire([held], {
listings: readerAnswering({ [harness.claudeProjectDir]: listing })
})
expect(result.retired).toEqual([])
expect(result.degradedRoots).toEqual([{ root, reason: listing.message }])
}
})
// The one bit of memory, and the only thing it buys: a root that held
// transcripts on the previous pass and lists empty on this one gets one pass of
// grace, so a directory swapped out for a moment cannot retire a tree.
it('holds a root that went from holding transcripts to empty in one pass', async () => {
const root = harness.roots.claudeProjectsDir ?? ''
await mkdir(root, { recursive: true })
const held = join(harness.claudeProjectDir, 'one.jsonl')
const grace = await retire([held], { emptiedRoots: new Set([root]) })
expect(grace.retired).toEqual([])
expect(grace.unverifiable).toEqual([held])
// The next pass has no transition to point at, so the empty listing is what
// it says it is: the user emptied the root.
const after = await retire([held], { emptiedRoots: new Set() })
expect(after.retired).toEqual([held])
})
// A flat-layout agent, where the mountpoint IS the session directory, is the
// one shape the grace exists for: there is no intermediate directory whose
// absence could stop the walk.
it('holds a flat root that emptied in one pass, and retires it on the next', async () => {
const root = harness.roots.copilotSessionsDir ?? ''
await mkdir(root, { recursive: true })
const held = join(root, 'session.jsonl')
expect((await retire([held], { roots: [root], emptiedRoots: new Set([root]) })).retired).toEqual(
[]
)
expect((await retire([held], { roots: [root] })).retired).toEqual([held])
})
// OpenClaw's discovery merges two directories into one delimiter-joined label.
// Roots reach this function as the real directories behind that label, so one
// of them being unreachable never touches the other's rows.
it('judges each merged-root directory on its own', async () => {
const current = join(harness.roots.openclawStateDir ?? '', 'agents')
const legacy = join(harness.roots.openclawLegacyStateDir ?? '', 'agents')
const onMissing = join(current, 'main', 'sessions', 'mounted.jsonl')
const deleted = join(legacy, 'main', 'sessions', 'deleted.jsonl')
await mkdir(join(legacy, 'main', 'sessions'), { recursive: true })
const result = await retire([onMissing, deleted], { roots: [current, legacy] })
expect(result.retired).toEqual([deleted])
expect(result.unverifiable).toEqual([onMissing])
expect(result.degradedRoots.map((one) => one.root)).toEqual([current])
})
// A row under no configured root is judged by its own directory and nothing
// above it, so a moved profile is never retired on the strength of a root that
// no longer covers it.
it('judges a row under no configured root by its own directory', async () => {
const orphanDir = join(harness.root, 'moved-profile')
await mkdir(orphanDir, { recursive: true })
const gone = join(orphanDir, 'gone.jsonl')
const present = join(orphanDir, 'present.jsonl')
await writeFile(present, '{}')
const result = await retire([gone, present], { roots: [] })
expect(result.retired).toEqual([gone])
// No configured root owns it, so nothing is reported as degraded for it.
expect(result.degradedRoots).toEqual([])
})
// I8. A synthetic row names a container and an entry inside it. Walking the
// row's own path would report every one of them gone, and walking only the
// container proves nothing about the entry: a session deleted inside a database
// that is still there would never be retired at all.
it('proves a synthetic row against its container, not against its own path', async () => {
const db = join(harness.root, 'opencode.db')
await writeFile(db, '')
const kept = `${db}#session-1`
const deleted = `${db}#session-2`
const enumeratedContainers = new Map([[db, new Set(['session-1'])]])
const result = await retire([kept, deleted], { roots: [], enumeratedContainers })
expect(result.retired).toEqual([deleted])
expect(result.unverifiable).toEqual([])
})
it('keeps a synthetic row when this pass did not enumerate its container', async () => {
const db = join(harness.root, 'opencode.db')
await writeFile(db, '')
const row = `${db}#session-1`
// A cycle asks for the newest N per agent, so a row it did not return may be
// the one after them. It enumerates nothing and therefore proves nothing.
await expect(retire([row], { roots: [] })).resolves.toMatchObject({
retired: [],
unverifiable: [row]
})
// An enumeration that returned nothing at all is not evidence either: a
// database whose schema this scanner no longer recognises reads as empty
// with no error, and believing it would retire every session in one pass.
await expect(
retire([row], { roots: [], enumeratedContainers: new Map([[db, new Set<string>()]]) })
).resolves.toMatchObject({ retired: [], unverifiable: [row] })
})
it('retires a synthetic row when the container it came from is gone', async () => {
const db = join(harness.root, 'opencode.db')
await writeFile(db, '')
const row = `${db}#session-1`
const enumeratedContainers = new Map([[db, new Set(['session-1'])]])
await expect(retire([row], { roots: [], enumeratedContainers })).resolves.toMatchObject({
retired: []
})
await rm(db)
await expect(retire([row], { roots: [], enumeratedContainers })).resolves.toMatchObject({
retired: [row]
})
})
// Nothing in this PR can hold a synthetic row: the index pass refuses a source
// whose parser decodes its messages where the message channel cannot reach
// them, and OpenCode's SQLite sessions are read on a worker thread. The rule
// above is the guard for the day that changes -- without it the walk would read
// `<db>#<id>` as a filename and retire every such row the moment it appeared.
it('does not index a source whose messages the channel cannot reach', () => {
const db = join(harness.root, 'opencode.db')
expect(
parserPublishesMessages({
agent: 'opencode',
codexHome: null,
file: { path: `${db}#session-1`, mtimeMs: 1, modifiedAt: '', sizeBytes: 0 }
})
).toBe(false)
})
// Round 12, F1. The cap counts directories because that is what costs: rows
// sharing one are a single read and then map lookups.
it('caps the directories one pass reads, not the rows it answers', async () => {
const roots = [harness.claudeProjectDir]
const inside = (folder: string, name: string): string =>
join(harness.claudeProjectDir, folder, name)
for (const folder of ['one', 'two', 'three']) {
await mkdir(join(harness.claudeProjectDir, folder), { recursive: true })
}
// Four rows in each of three directories: three reads, twelve answers.
const paths = ['one', 'two', 'three'].flatMap((folder) =>
['a', 'b', 'c', 'd'].map((name) => inside(folder, name))
)
const result = await retire(paths, { roots, directoryLimit: 2 })
// Two directories' worth answered, all eight of their rows, and the third
// directory's four left for the pass after this one.
expect(result.retired).toEqual(paths.slice(0, 8))
expect(result.unchecked).toEqual(paths.slice(8))
})
// The starvation this replaced: an unreadable directory answers `unverifiable`
// for every row under it and never becomes readable, so a cap on rows let one
// such directory hold the walk for as long as the permission stayed wrong.
it.skipIf(!CAN_DENY_READ)(
'is not starved by many rows under one unreadable directory',
async () => {
const locked = join(harness.claudeProjectDir, 'locked')
await mkdir(locked, { recursive: true })
const blocked = Array.from({ length: 520 }, (_unused, index) =>
join(locked, `locked-${index}.jsonl`)
)
const deleted = join(harness.claudeProjectDir, 'deleted.jsonl')
await chmod(locked, 0o000)
try {
const result = await retire([...blocked, deleted], { directoryLimit: 512 })
expect(result.retired).toEqual([deleted])
expect(result.unverifiable).toHaveLength(blocked.length)
expect(result.unchecked).toEqual([])
} finally {
await chmod(locked, 0o700)
}
}
)
it('reads each directory once however many files it is asked about', async () => {
await mkdir(harness.claudeProjectDir, { recursive: true })
const listings = new SessionSearchDirectoryListings()
await retire(
Array.from({ length: 50 }, (_unused, index) =>
join(harness.claudeProjectDir, `gone-${index}.jsonl`)
),
{ listings }
)
expect(listings.size).toBe(1)
})
src/main/ai-vault-search/session-search-deleted-sources.ts
+263
View File
@@ -0,0 +1,263 @@
import { basename, dirname } from 'node:path'
import type { SessionSearchDegradedRoot } from './session-search-degraded-roots'
import type { SessionSearchDirectoryReader } from './session-search-directory-listings'
import { isUnderScanRoot } from './session-search-scan-roots'
import { splitSyntheticSessionSource } from './session-search-synthetic-sources'
import type { SessionSearchStore } from './session-search-store'
/*
* Retirement invariants. Every one of these is a test; changing this file means
* changing the list, not working around it.
*
* I1. A row is retired only when its file is PROVEN gone: some directory
* between the file and its configured root lists successfully, and the next
* path component toward the file is absent from that listing.
* I2. If no directory from the file's parent up to the configured root can be
* listed, nothing is proven and no row is dropped. ENOENT/ENOTDIR is walked
* up (the directory itself is a missing component of some ancestor);
* EACCES, EIO, a WSL gate refusal, anything else, is unverifiable at once.
* I3. The rule is the same on the first pass after a process start and on every
* later pass. It needs no memory of what previous passes saw, because the
* walk is bounded at the configured root and never reasons about what is
* above it.
* I4. A file, or a project directory, the user really deleted retires on the
* first pass that proves it. There is no waiting period and no census.
* I8. A row whose path names an entry inside a container rather than a file of
* its own is proven the same way, one level up: the container must be
* present, and the pass must have enumerated it in full and successfully.
* A listing is a listing whether it comes from readdir or from a database.
*
* What I3 costs, stated rather than hidden: a volume mounted at exactly a
* configured root, unmounted so that the mountpoint stays present and lists
* empty, is indistinguishable from a root the user emptied. It retires. The
* realistic unmount shapes do not: a mount above the root leaves the root
* itself missing (the walk stops at the root boundary), and an unreadable root
* is an error, not a listing. One bit per root buys the remaining grace: a root
* that held transcripts on the previous pass and holds none on this one is
* unverifiable for that pass, so a single flap cannot retire a tree.
*/
// Walked up rather than believed: a directory that ENOENTs is itself the
// missing component its parent has to be asked about.
const MISSING_DIRECTORY = new Set(['ENOENT', 'ENOTDIR'])
export type SessionSearchRetirement = {
/** Paths proven gone and dropped from the index. */
retired: string[]
/** Rows kept: this pass could prove the file neither present nor gone. */
unverifiable: string[]
/** Paths the per-pass cap left for next time. */
unchecked: string[]
/** Roots owning at least one unverifiable verdict, with the reason. */
degradedRoots: SessionSearchDegradedRoot[]
}
export type SessionSearchRetirementArgs = {
store: SessionSearchStore
/** Held paths this pass did not discover; everything else is still there. */
paths: readonly string[]
/** The real directories this pass walked; the longest one containing a path bounds its walk. */
roots: readonly string[]
/** Roots that listed transcripts on the previous pass and none on this one. */
emptiedRoots?: ReadonlySet<string>
/**
* Containers this pass enumerated in full, with the ids each holds. Only a
* census builds it; see session-search-synthetic-sources.ts for the bar a
* container has to meet before it appears here.
*/
enumeratedContainers?: ReadonlyMap<string, ReadonlySet<string>>
/** One readdir per directory per pass, shared with the rest of the pass. */
listings: SessionSearchDirectoryReader
/**
* Directories this walk may read before the pass moves on.
*
* Directories, not rows. A row whose walk finds its directory already read is
* answered from the pass's cache and costs nothing, so counting rows made an
* unreadable directory able to starve the whole walk: five hundred rows under
* one EACCES directory are one readdir and five hundred identical
* unverifiable verdicts, and a row for a file the user really deleted, sorted
* behind them, was never reached on any pass.
*/
directoryLimit?: number
signal?: AbortSignal
}
type SessionSearchSourceVerdict =
| { verdict: 'gone' }
| { verdict: 'present' }
| { verdict: 'unverifiable'; reason: string }
/**
* Retires index rows for sources that are provably gone.
*
* One function, called by both the sweep and the cycle, because either one
* alone deleting a user's history the first time a mount is missing is the bug
* this feature kept shipping. There is no separate root fence: the walk cannot
* reach a verdict of `gone` without a successful listing, so an unreadable or
* missing root produces `unverifiable` structurally rather than by a guard
* somebody has to remember to call (docs/reference/ssh-execution-boundary.md:
* loss of contact is never evidence of absence).
*/
export async function retireDeletedSessionSearchSources(
args: SessionSearchRetirementArgs
): Promise<SessionSearchRetirement> {
const { store, paths, signal } = args
const emptiedRoots = args.emptiedRoots ?? new Set<string>()
const directoryLimit = args.directoryLimit ?? Number.POSITIVE_INFINITY
// Every directory this walk asked for, whether the pass had already read it
// or not. What it bounds is real work: a repeat of one already in here is a
// map lookup, and only a name that is new to it can cost a readdir.
const asked = new Set<string>()
const listings: SessionSearchDirectoryReader = {
namesIn: (directory, signal) => {
asked.add(directory)
return args.listings.namesIn(directory, signal)
}
}
const retirement: SessionSearchRetirement = {
retired: [],
unverifiable: [],
unchecked: [],
degradedRoots: []
}
const degraded = new Map<string, string>()
for (const [index, path] of paths.entries()) {
// A synthetic row names a container and an entry inside it, never a file of
// its own; walking the row's own path would report every one of them gone.
const synthetic = splitSyntheticSessionSource(path)
const filePath = synthetic?.container ?? path
// Why capped at all: the sweep hands over every path it holds and did not
// discover, and under an unmount that is the whole index. What is left is
// simply still undiscovered next pass, so the walk finishes over the ones
// that follow rather than holding this one.
//
// Spent past the bound only by a row that starts somewhere new. One this
// walk has already read is answered from the map, so refusing it would buy
// nothing and would leave the budget hostage to whichever directory the
// rows happened to be sorted by.
if (signal?.aborted || (asked.size >= directoryLimit && !asked.has(dirname(filePath)))) {
retirement.unchecked.push(...paths.slice(index))
break
}
const root = configuredRootFor(filePath, args.roots)
const containerProof = await proveSource(filePath, root ?? dirname(filePath), {
listings,
emptiedRoots,
signal
})
const proof = synthetic
? proveSyntheticSource(synthetic, containerProof, args.enumeratedContainers)
: containerProof
if (proof.verdict === 'gone') {
store.removeFile(path)
retirement.retired.push(path)
continue
}
if (proof.verdict === 'present') {
continue
}
retirement.unverifiable.push(path)
// Only a configured root is an alarm worth raising: a row under no root
// this scan walks is already reported on its own, as an orphan.
if (root !== null && !degraded.has(root)) {
degraded.set(root, proof.reason)
}
}
retirement.degradedRoots = [...degraded].map(([root, reason]) => ({ root, reason }))
return retirement
}
/**
* Walks from the file toward its configured root, asking each directory whether
* the next component toward the file is there. The first directory that answers
* decides; a directory that is itself missing moves the question up one level.
*
* The loop cannot pass the configured root, which is what makes the whole thing
* memoryless: everything above the root — a home directory on an unmounted
* volume, a detached drive, an SSH mount that is not there — is out of scope by
* construction rather than by a state machine that has to remember it.
*/
async function proveSource(
path: string,
root: string,
context: {
listings: SessionSearchDirectoryReader
emptiedRoots: ReadonlySet<string>
signal?: AbortSignal
}
): Promise<SessionSearchSourceVerdict> {
let directory = dirname(path)
let child = basename(path)
while (directory === root || isUnderScanRoot(directory, root)) {
const listing = await context.listings.namesIn(directory, context.signal)
if (!listing.listed) {
if (listing.code !== null && MISSING_DIRECTORY.has(listing.code)) {
const parent = dirname(directory)
if (parent === directory) {
break
}
child = basename(directory)
directory = parent
continue
}
return { verdict: 'unverifiable', reason: listing.message }
}
if (listing.names.has(child)) {
return { verdict: 'present' }
}
if (directory === root && context.emptiedRoots.has(root)) {
// One pass of grace, so a root that blinks empty for a moment — a sync
// client mid-swap, a mount that has not settled — cannot retire a tree.
return {
verdict: 'unverifiable',
reason: 'Listed no transcripts where it listed some on the previous pass.'
}
}
return { verdict: 'gone' }
}
return { verdict: 'unverifiable', reason: `${root} could not be listed.` }
}
/**
* A synthetic row is proven by its container's own enumeration, one level above
* where the filesystem walk stops.
*
* The container has to be present first: a database on a volume that is not
* there proves nothing about the sessions inside it, and a database that is
* gone takes its sessions with it. Only then does the enumeration decide, and
* only when this pass made one that was exhaustive and successful -- a cycle
* asks for the newest N per agent, so an id it did not return may just be the
* one after them.
*/
function proveSyntheticSource(
synthetic: { container: string; id: string },
containerProof: SessionSearchSourceVerdict,
enumerated?: ReadonlyMap<string, ReadonlySet<string>>
): SessionSearchSourceVerdict {
if (containerProof.verdict !== 'present') {
return containerProof
}
const ids = enumerated?.get(synthetic.container)
// An enumeration that returned nothing at all is not evidence that the
// container holds nothing: a source whose schema this scanner no longer
// recognises reads as empty with no error to see, and believing it would
// retire every entry in one pass.
if (!ids || ids.size === 0) {
return {
verdict: 'unverifiable',
reason: `${synthetic.container} was not enumerated in full this pass.`
}
}
return ids.has(synthetic.id) ? { verdict: 'present' } : { verdict: 'gone' }
}
/** Longest configured root containing the path, or null for a row under none. */
function configuredRootFor(path: string, roots: readonly string[]): string | null {
let owner: string | null = null
for (const root of roots) {
if (isUnderScanRoot(path, root) && (owner === null || root.length > owner.length)) {
owner = root
}
}
return owner
}
src/main/ai-vault-search/session-search-directory-listings.test.ts
+61
View File
@@ -0,0 +1,61 @@
import { mkdir, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { beforeEach, expect, it, vi } from 'vitest'
import { SessionSearchDirectoryListings } from './session-search-directory-listings'
const { readdir } = vi.hoisted(() => ({ readdir: vi.fn() }))
vi.mock('../native-chat/wsl-transcript-fs-access', () => ({
wslGatedReaddir: readdir
}))
beforeEach(() => {
readdir.mockReset()
})
// A WSL root is a UNC path into the distro, and reading it with raw `fs` is
// what makes a stalled distro look like an empty directory. The gated primitive
// is the same one discovery walks with, so a refusal arrives as an error the
// walk treats as unverifiable rather than as "nothing here".
it('reads through the gated primitive, on the scan lane', async () => {
const unc = '\\\\wsl$\\Ubuntu\\home\\me\\.claude\\projects'
readdir.mockResolvedValueOnce([{ name: 'one.jsonl' }])
const listings = new SessionSearchDirectoryListings()
const listing = await listings.namesIn(unc)
expect(readdir).toHaveBeenCalledWith(unc, 'scan', undefined)
expect(listing).toEqual({ listed: true, names: new Set(['one.jsonl']) })
})
it('reports the code a failed read carried, so ENOENT and EACCES stay apart', async () => {
readdir.mockRejectedValueOnce(Object.assign(new Error('permission denied'), { code: 'EACCES' }))
const listings = new SessionSearchDirectoryListings()
expect(await listings.namesIn('/blocked')).toEqual({
listed: false,
code: 'EACCES',
message: 'permission denied'
})
})
it('reads a directory once per pass, error or not', async () => {
readdir.mockRejectedValue(Object.assign(new Error('gone'), { code: 'ENOENT' }))
const listings = new SessionSearchDirectoryListings()
await listings.namesIn('/gone')
await listings.namesIn('/gone')
expect(readdir).toHaveBeenCalledTimes(1)
expect(listings.size).toBe(1)
})
it('is a real directory read when nothing is mocked out from under it', async () => {
readdir.mockImplementation(async (path: string) => {
const { readdir: real } = await import('node:fs/promises')
return (await real(path, { withFileTypes: true })) as unknown
})
const root = join(tmpdir(), `ss-listings-${process.pid}`)
await mkdir(root, { recursive: true })
await writeFile(join(root, 'present.jsonl'), '{}')
const listing = await new SessionSearchDirectoryListings().namesIn(root)
expect(listing.listed && listing.names.has('present.jsonl')).toBe(true)
})
src/main/ai-vault-search/session-search-directory-listings.ts
+70
View File
@@ -0,0 +1,70 @@
import { wslGatedReaddir } from '../native-chat/wsl-transcript-fs-access'
/** One directory read: the names it holds, or what stopped the read. */
export type SessionSearchDirectoryListing =
| { listed: true; names: ReadonlySet<string> }
| { listed: false; code: string | null; message: string }
/**
* What the retirement walk needs of a directory: its names, or why not.
*
* An interface rather than the class, so a test can hand the walk an EIO or a
* gate refusal — the shapes a stalled network mount answers with, which no
* temporary directory can be made to produce.
*/
export type SessionSearchDirectoryReader = {
namesIn(directory: string, signal?: AbortSignal): Promise<SessionSearchDirectoryListing>
}
/**
* Every directory one pass had to read, read once.
*
* The retirement walk asks the same directories about many files — a project
* directory holds hundreds of transcripts — and under an unmount every path
* under a root walks up through the same ancestors. One readdir per directory
* per pass keeps that bounded, and it also makes the pass self-consistent: two
* files in one directory cannot get contradictory verdicts because the
* directory changed between them.
*
* Reads go through the same gated primitive discovery uses, so a WSL UNC path
* is routed to the distro's helper process rather than read with raw fs, and a
* gate refusal arrives as an error rather than as an empty directory.
*/
export class SessionSearchDirectoryListings implements SessionSearchDirectoryReader {
private readonly listings = new Map<string, SessionSearchDirectoryListing>()
async namesIn(directory: string, signal?: AbortSignal): Promise<SessionSearchDirectoryListing> {
const cached = this.listings.get(directory)
if (cached) {
return cached
}
const listing = await readDirectory(directory, signal)
this.listings.set(directory, listing)
return listing
}
/** Directories read this pass; only tests and cost accounting need it. */
get size(): number {
return this.listings.size
}
}
async function readDirectory(
directory: string,
signal?: AbortSignal
): Promise<SessionSearchDirectoryListing> {
try {
const entries = await wslGatedReaddir(directory, 'scan', signal)
return { listed: true, names: new Set(entries.map((entry) => entry.name)) }
} catch (error) {
const code =
error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
? error.code
: null
return {
listed: false,
code,
message: error instanceof Error ? error.message : String(error)
}
}
}
src/main/ai-vault-search/session-search-engine-test-fixture.ts
-113
View File
@@ -1,113 +0,0 @@
import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchEngine, type SessionSearchEngineOptions } from './session-search-engine'
import { cwdKey } from './session-search-file-records'
import { identifierShadowText } from './session-search-identifier-split'
import { SessionSearchStore } from './session-search-store'
import {
openSessionSearchIndexFile,
type SessionSearchIndexFile
} from './session-search-index-test-fixture'
// Synthetic index rows for the query tests. The write path has its own tests;
// driving it here would make every retrieval assertion depend on the parser.
export type SessionSearchHarness = {
/** The engine's own connection; the store next to it keeps a second, private one. */
db: SyncDatabase
/** A real writer on the same file, so a test can move the index under the engine. */
store: SessionSearchStore
engine: SessionSearchEngine
close: () => Promise<void>
}
export async function openSessionSearchHarness(
name: string,
options: SessionSearchEngineOptions = {}
): Promise<SessionSearchHarness> {
const index: SessionSearchIndexFile = await openSessionSearchIndexFile(name)
const store = new SessionSearchStore(index.path, (error) => {
throw error
})
// Constructed before any row is planted, because constructing it is what
// installs the generation triggers the planted rows have to move.
const engine = new SessionSearchEngine(index.db, options)
return {
db: index.db,
store,
engine,
close: async () => {
store.close()
await index.close()
}
}
}
export type SyntheticSession = {
id: number
cwd?: string | null
text?: string
/** Rows of `text` to write; one session with many rows is one hit. */
rows?: number
role?: TranscriptMessageRole
/**
* Written into `tool_text` alongside `text`, which is the one row shape the
* conversation scope has to exclude while the `all` scope keeps it.
*/
toolText?: string
agent?: string
updatedAt?: string
messageCount?: number
/** Written into `files`, which is what makes the source `present`. */
filePath?: string | null
/** `sessions.file_path`: the transcript `path:` searches alongside cwd. */
sessionFilePath?: string
}
/** One session and its message rows, in both FTS tables the way the writer does. */
export function addSyntheticSession(db: SyncDatabase, session: SyntheticSession): void {
const {
id,
cwd = '/repo/app',
text = 'needle',
rows = 1,
role = 'user',
toolText = '',
agent = 'claude',
updatedAt = `2026-09-${String((id % 28) + 1).padStart(2, '0')}T00:00:00.000Z`,
messageCount = rows,
filePath = `/synthetic/${id}.jsonl`,
sessionFilePath = `/synthetic/${id}.jsonl`
} = session
db.prepare(
`INSERT INTO sessions(id,agent,session_id,file_path,title,cwd,cwd_key,updated_at,message_count,resume_command)
VALUES (?,?,?,?,'fixture',?,?,?,?,'resume')`
).run(id, agent, String(id), sessionFilePath, cwd, cwdKey(cwd), updatedAt, messageCount)
if (filePath !== null) {
db.prepare(
'INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES (?,0,1740000000000,?)'
).run(filePath, id)
}
for (let row = 0; row < rows; row++) {
const messageId = Number(
db
.prepare('INSERT INTO messages(session_row_id,role,ts) VALUES (?,?,?)')
.run(id, role, updatedAt).lastInsertRowid
)
const user = role === 'user' ? text : ''
const assistant = role === 'assistant' ? text : ''
const tool = role === 'tool' ? `${text} ${toolText}`.trim() : toolText
db.prepare(
'INSERT INTO messages_fts(rowid,user_text,assistant_text,tool_text,identifiers) VALUES (?,?,?,?,?)'
).run(messageId, user, assistant, tool, identifierShadowText(`${text} ${toolText}`))
}
}
export function markFork(db: SyncDatabase, ids: readonly number[], hash: string): void {
for (const id of ids) {
db.prepare('UPDATE sessions SET content_hash = ?, content_hash_count = 8 WHERE id = ?').run(
hash,
id
)
}
}
src/main/ai-vault-search/session-search-engine-types.ts
-157
View File
@@ -1,157 +0,0 @@
import type { AiVaultAgent } from '../../shared/ai-vault-types'
import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers'
import type { SessionSearchUnavailableFeature } from './session-search-query-schema'
// ENGINE types, deliberately not in src/shared: nothing here is a wire type.
// PR 5 owns the public contract and lifts what a caller may actually receive;
// until then a field can be added, renamed or dropped without a compat story.
export const SESSION_SEARCH_LIMIT_DEFAULT = 20
export const SESSION_SEARCH_LIMIT_MAX = 100
// Longer than this is not a query, and FTS5 pays for every term it plans.
export const SESSION_SEARCH_QUERY_MAX_LENGTH = 512
// Snippet match markers. Why doubled: single brackets are everywhere in code
// transcripts (`arr[0]`, regex classes, markdown links) and would read as
// matches; doubled ones are rare.
export const SESSION_SEARCH_SNIPPET_MARK_OPEN = '[['
export const SESSION_SEARCH_SNIPPET_MARK_CLOSE = ']]'
/**
* Which corpus answers the query.
*
* - `conversation`: user and assistant turns only, as a column filter over
* `messages_fts` (see `scopedExpression`).
* - `all`: those turns plus tool calls and tool output, and the identifier
* shadow column, from `messages_fts`.
*
* The engine searches exactly the scope it is given. Switching corpus as the
* user types is a UI policy and lives in the panel (PR 7); an engine that
* second-guessed the scope would make a result impossible to reproduce from
* its own request.
*/
export type SessionSearchScope = 'conversation' | 'all'
export type SessionSearchSort = 'relevance' | 'newest'
export type SessionSearchFilters = {
agents?: readonly AiVaultAgent[]
/** Only sessions whose cwd is that path or inside it. */
scopePaths?: readonly string[]
/** ISO timestamp; only sessions updated at or after it. */
since?: string
sort?: SessionSearchSort
}
export type SessionSearchRequest = {
query: string
/** Default `all`. */
scope?: SessionSearchScope
limit?: number
/** From a previous response's `page.cursor`; only valid in its own generation. */
cursor?: string
filters?: SessionSearchFilters
}
export type SessionSearchRoute = 'phrase' | 'and' | 'or' | 'typo+phrase' | 'typo+and' | 'typo+or'
/**
* How the query was executed. Diagnostics, not an answer: PR 5 decides which of
* these a caller ever sees (the reviewer's F5/F7 want them behind `debug`).
*/
export type SessionSearchPlannerReport = {
route: SessionSearchRoute
/**
* The whole body the repaired plan searched, in query order, when any term
* was changed. Not just the corrected terms: a caller rendering "searched
* for" needs the query it actually ran, and a repair never drops a term the
* original kept. A corrected term carries the index's own spelling, which the
* tokenizer has case-folded; untouched terms keep the case they were typed in.
*/
repairedTerms?: string[]
/** The corpus the route ran against; today always the requested scope. */
tier: SessionSearchScope
}
/**
* Where a source stands according to the index's own `files` table. The query
* path never stats a transcript, so it can report that the index has a live
* file record for a session or that it has none, and never that a source is
* gone: only a proven deletion may claim `missing`, and proving one is the
* indexer's job (docs/reference/ssh-execution-boundary.md).
*/
export type SessionSearchSourcePresence = 'present' | 'unverifiable'
export type SessionSearchEvidence = {
role: TranscriptMessageRole
timestamp: string | null
/** FTS5 snippet with the matched terms wrapped in `[[` `]]`. */
snippet: string
/** The snippet hit the engine's per-hit ceiling and was cut. */
snippetTruncated?: boolean
}
export type SessionSearchHit = {
agent: AiVaultAgent
sessionId: string
filePath: string
codexHome: string | null
title: string
cwd: string | null
branch: string | null
updatedAt: string | null
messageCount: number
resumeCommand: string
score: number
/** Sessions folded into this hit (forks sharing an opening prefix); absent when unique. */
duplicateCount?: number
source: SessionSearchSourcePresence
/** Null when the operators alone put this session on the page, with no text match. */
evidence: SessionSearchEvidence | null
}
export type SessionSearchPage = {
/** Null when this page is the last one. */
cursor: string | null
hasMore: boolean
}
export type SessionSearchTruncation = {
/**
* Ranking saw only the first `sessionCandidateLimit` sessions, so a session
* past that cut cannot appear on any page of this query.
*/
candidates: boolean
/** Hits on this page whose snippet was cut. */
snippets: number
/**
* The query itself was cut before it was searched: past the length ceiling,
* or past the number of terms the planner will plan. The terms that survived
* were searched in full, so a hit is still a hit; a miss is not proof of
* absence.
*/
query: boolean
}
export type SessionSearchResponse = {
hits: SessionSearchHit[]
/**
* Engine features the index on disk cannot serve, empty on a current index.
* A route ladder missing its repair rung still answers; saying so is what
* keeps the answer honest.
*/
unavailable: readonly SessionSearchUnavailableFeature[]
planner: SessionSearchPlannerReport
page: SessionSearchPage
truncated: SessionSearchTruncation
/** The index snapshot these hits came from; a cursor is only valid within it. */
generation: number
durationMs: number
}
export function resolveSessionSearchLimit(limit: number | undefined): number {
// Why clamped here and not at the caller: a non-positive limit becomes
// `slice(0, -1)`, which silently drops the last hit of every page.
const requested = Number.isInteger(limit) ? (limit as number) : SESSION_SEARCH_LIMIT_DEFAULT
return Math.min(Math.max(1, requested), SESSION_SEARCH_LIMIT_MAX)
}
src/main/ai-vault-search/session-search-engine.test.ts
-471
View File
@@ -1,471 +0,0 @@
import { afterEach, describe, expect, it } from 'vitest'
import { SESSION_SEARCH_QUERY_MAX_LENGTH } from './session-search-engine-types'
import type { SessionSearchRequest, SessionSearchResponse } from './session-search-engine-types'
import { planSessionSearchQuery } from './session-search-query-planner'
import { ensureSessionSearchQuerySchema } from './session-search-query-schema'
import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet'
import {
addSyntheticSession,
markFork,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
async function open(name: string, options = {}): Promise<SessionSearchHarness> {
harness = await openSessionSearchHarness(name, options)
return harness
}
function ids(result: SessionSearchResponse): string[] {
return result.hits.map((hit) => hit.sessionId)
}
describe('the route ladder tries phrase, then AND, then repair, then OR', () => {
async function routeFor(
text: string,
request: SessionSearchRequest
): Promise<SessionSearchResponse> {
const { db, engine } = await open('ss-engine-route')
addSyntheticSession(db, { id: 1, text })
return engine.search(request)
}
it('takes the phrase route when the tokens are adjacent and in order', async () => {
const result = await routeFor('the alpha beta gamma line', { query: '"alpha beta"' })
expect(result.planner.route).toBe('phrase')
expect(ids(result)).toEqual(['1'])
})
it('falls to AND when the tokens are present but not adjacent', async () => {
const result = await routeFor('beta separated alpha', { query: '"alpha beta"' })
expect(result.planner.route).toBe('and')
expect(ids(result)).toEqual(['1'])
})
it('falls to OR for prose, where no phrase was ever claimed', async () => {
const result = await routeFor('the relay dropped a frame', { query: 'relay frames dropped' })
expect(result.planner.route).toBe('or')
expect(ids(result)).toEqual(['1'])
})
it('repairs a typo before the OR fallback, and says which terms it changed', async () => {
const { db, engine } = await open('ss-engine-typo')
// Two copies: the repair only suggests a term the index really holds.
addSyntheticSession(db, { id: 1, text: 'the coalesces path is slow' })
addSyntheticSession(db, { id: 2, text: 'coalesces again here' })
const result = engine.search({ query: 'coalescs' })
expect(result.planner.route).toBe('typo+or')
expect(result.planner.repairedTerms).toEqual(['coalesces'])
expect(ids(result).sort()).toEqual(['1', '2'])
})
it('keeps every term a repaired literal was typed with', async () => {
const { db, engine } = await open('ss-engine-typo-literal')
addSyntheticSession(db, { id: 1, text: 'parseJson the data' })
addSyntheticSession(db, { id: 2, text: 'parseJson the data again' })
// `parseJsonn(the, data)` is literal because of its punctuation; the
// corrected spelling read on its own is prose. Re-planning without carrying
// the original decision across would drop `the` and report a body that was
// never typed.
// A corrected term comes back in the index's own spelling, which unicode61
// has folded; the terms the repair left alone keep the case they were typed.
const result = engine.search({ query: 'parseJsonn(the, data)' })
expect(result.planner.repairedTerms).toEqual(['parsejson', 'the', 'data'])
})
it('does not repair a term the index already holds', async () => {
const { db, engine } = await open('ss-engine-no-typo')
addSyntheticSession(db, { id: 1, text: 'coalesces' })
const result = engine.search({ query: 'coalesces' })
expect(result.planner.repairedTerms).toBeUndefined()
expect(result.planner.route).toBe('or')
})
it('reports the scope it searched as the planner tier', async () => {
const { db, engine } = await open('ss-engine-tier')
addSyntheticSession(db, { id: 1, text: 'needle' })
expect(engine.search({ query: 'needle' }).planner.tier).toBe('all')
expect(engine.search({ query: 'needle', scope: 'conversation' }).planner.tier).toBe(
'conversation'
)
})
})
describe('scope picks the corpus and never switches it', () => {
async function corpus(): Promise<SessionSearchHarness> {
const opened = await open('ss-engine-scope')
addSyntheticSession(opened.db, { id: 1, text: 'harbor pilot manifest', role: 'user' })
addSyntheticSession(opened.db, { id: 2, text: 'harbor tool output line', role: 'tool' })
return opened
}
it('searches conversation turns only under `conversation`', async () => {
const { engine } = await corpus()
expect(ids(engine.search({ query: 'harbor', scope: 'conversation' }))).toEqual(['1'])
})
it('includes tool output under `all`, which is the default', async () => {
const { engine } = await corpus()
expect(ids(engine.search({ query: 'harbor', scope: 'all' })).sort()).toEqual(['1', '2'])
expect(ids(engine.search({ query: 'harbor' })).sort()).toEqual(['1', '2'])
})
it('returns nothing rather than widening when the narrow scope misses', async () => {
// The panel's two-tier typing is a UI policy (PR 7). An engine that widened
// here would make a result impossible to reproduce from its own request.
const { engine } = await corpus()
const result = engine.search({ query: 'output', scope: 'conversation' })
expect(result.hits).toEqual([])
expect(result.planner.tier).toBe('conversation')
})
it('matches an identifier through its pieces only in the full corpus', async () => {
const { db, engine } = await open('ss-engine-identifiers')
addSyntheticSession(db, { id: 1, text: 'resolveTerminalPath' })
// The identifier shadow column lives in messages_fts alone.
expect(ids(engine.search({ query: 'terminal path' }))).toEqual(['1'])
expect(engine.search({ query: 'terminal path', scope: 'conversation' }).hits).toEqual([])
})
})
describe('the conversation scope is a column filter, and it binds the whole query', () => {
it('refuses an AND whose second term lives only in tool output', async () => {
// The filter binds to the expression it prefixes. `{cols}: (a AND b)`
// filters both terms; `{cols}: a AND b` filters only `a` and searches tool
// output for the rest, which is a conversation search answering from a
// column it promised not to read.
const { db, engine } = await open('ss-engine-scope-binding')
addSyntheticSession(db, { id: 1, text: 'alpha gamma beta' })
addSyntheticSession(db, { id: 2, text: 'alpha gamma', toolText: 'beta' })
// Quoted, so the query is literal; not adjacent, so the phrase rung misses
// and the AND rung is the one that answers.
const query = '"alpha" beta'
const wide = engine.search({ query, scope: 'all' })
expect(wide.planner.route).toBe('and')
expect(ids(wide).sort()).toEqual(['1', '2'])
const narrowed = engine.search({ query, scope: 'conversation' })
expect(narrowed.planner.route).toBe('and')
expect(ids(narrowed)).toEqual(['1'])
})
it('ranks a conversation hit down for tool output it will not show', async () => {
// The one behavioural difference the column filter carries, pinned rather
// than wished away. FTS5's bm25 normalises by the whole row's length and
// has no per-column length, so two rows with identical prose do not score
// identically when one of them also holds tool output. A dedicated
// two-column table scored them the same. The rowid set is unchanged, which
// is what the decision was measured on; the order within it can move.
const { db, engine } = await open('ss-engine-scope-weights')
addSyntheticSession(db, { id: 1, text: 'harbor pilot' })
addSyntheticSession(db, { id: 2, text: 'harbor pilot', toolText: 'unrelated '.repeat(40) })
const narrowed = engine.search({ query: 'harbor', scope: 'conversation' })
expect(ids(narrowed)).toEqual(['1', '2'])
expect(narrowed.hits[0]!.score).toBeGreaterThan(narrowed.hits[1]!.score)
})
it('never snippets a conversation hit out of tool output', async () => {
const { db, engine } = await open('ss-engine-scope-snippet')
addSyntheticSession(db, { id: 1, text: 'harbor pilot', toolText: 'harbor tool output line' })
const [hit] = engine.search({ query: 'harbor', scope: 'conversation' }).hits
expect(hit?.evidence?.snippet).toContain('pilot')
expect(hit?.evidence?.snippet).not.toContain('output')
// And asked for a tool-only row directly, it has nothing to show.
addSyntheticSession(db, { id: 2, text: 'harbor tool output line', role: 'tool' })
const rowid = Number(
(db.prepare('SELECT max(id) AS id FROM messages').get() as { id: number }).id
)
const plan = planSessionSearchQuery('harbor')
expect(sessionSearchSnippet(db, 'conversation', rowid, plan)).toEqual(EMPTY_SNIPPET)
expect(sessionSearchSnippet(db, 'all', rowid, plan).text).toContain('output')
})
})
describe('a session is one hit, however many of its rows matched', () => {
it.each(['relevance', 'newest'] as const)(
'keeps a short session on the %s page beside a 650-row session',
async (sort) => {
const { db, engine } = await open('ss-engine-aggregate', { sessionCandidateLimit: 600 })
addSyntheticSession(db, { id: 1, rows: 650, updatedAt: '2026-09-06T00:00:00.000Z' })
addSyntheticSession(db, {
id: 2,
text: 'needle padding',
updatedAt: '2026-09-05T00:00:00.000Z'
})
// Collapsing to one row per session happens before the candidate limit,
// so the 650-row session cannot crowd the one-row session off the page on
// either order; which of them ranks first is the sort's business.
expect(ids(engine.search({ query: 'needle', filters: { sort } })).sort()).toEqual(['1', '2'])
}
)
it('folds forks the same way for an operator-only page as for a text page', async () => {
const { db, engine } = await open('ss-engine-forks')
for (const id of [1, 2, 3, 4]) {
addSyntheticSession(db, { id, updatedAt: `2026-09-0${id}T00:00:00.000Z` })
}
markFork(db, [1, 2, 3, 4], 'shared-fork-prefix')
const operatorOnly = engine.search({ query: 'repo:app' })
const withText = engine.search({ query: 'needle repo:app' })
expect(ids(operatorOnly)).toEqual(['4'])
expect(operatorOnly.hits[0]?.duplicateCount).toBe(4)
expect(ids(withText)).toEqual(ids(operatorOnly))
expect(withText.hits[0]?.duplicateCount).toBe(4)
})
it('answers an operator-only query with the newest sessions and no evidence', async () => {
const { db, engine } = await open('ss-engine-operator-only')
addSyntheticSession(db, { id: 1, updatedAt: '2026-09-01T00:00:00.000Z' })
addSyntheticSession(db, { id: 2, updatedAt: '2026-09-09T00:00:00.000Z' })
const result = engine.search({ query: 'repo:app' })
expect(ids(result)).toEqual(['2', '1'])
expect(result.hits[0]?.evidence).toBeNull()
})
it('has no hits for a query with neither text nor operators', async () => {
const { db, engine } = await open('ss-engine-empty')
addSyntheticSession(db, { id: 1 })
expect(engine.search({ query: ' ' }).hits).toEqual([])
})
})
describe('filters narrow retrieval, not just the page', () => {
it('finds a scoped match behind 600 out-of-scope rows', async () => {
const { db, engine } = await open('ss-engine-scoped')
addSyntheticSession(db, { id: 1, cwd: '/unrelated', rows: 600 })
addSyntheticSession(db, { id: 2, cwd: '/target', text: 'needle padding' })
expect(ids(engine.search({ query: 'needle', filters: { scopePaths: ['/target'] } }))).toEqual([
'2'
])
})
it('falls back to a later rung when the exact hit is out of scope', async () => {
const { db, engine } = await open('ss-engine-scoped-route')
addSyntheticSession(db, { id: 1, cwd: '/unrelated', text: 'resolveTerminalPath' })
addSyntheticSession(db, { id: 2, cwd: '/target', text: 'resolve terminal path' })
expect(
ids(engine.search({ query: 'resolveTerminalPath', filters: { scopePaths: ['/target'] } }))
).toEqual(['2'])
})
})
describe('evidence', () => {
it('takes each snippet from that hit’s own best message', async () => {
const { db, engine } = await open('ss-engine-snippet')
// Written first, so its row owns the lowest rowid: the row a dropped rowid
// constraint would hand back for every hit.
addSyntheticSession(db, {
id: 1,
text: 'hydration marmoset appears once in a long paragraph about routing and caching',
updatedAt: '2026-09-01T00:00:00.000Z'
})
addSyntheticSession(db, {
id: 2,
text: 'hydration capybara',
updatedAt: '2026-09-09T00:00:00.000Z'
})
const hits = engine.search({ query: 'hydration' }).hits
expect(hits[0]?.evidence?.snippet).toContain('capybara')
expect(hits[0]?.evidence?.snippet).not.toContain('marmoset')
expect(hits.find((hit) => hit.sessionId === '1')?.evidence?.snippet).toContain('marmoset')
})
it('shows the prose column rather than the identifier shadow when both match', async () => {
const { db, engine } = await open('ss-engine-snippet-shadow')
addSyntheticSession(db, {
id: 1,
text: 'resolveTerminalPath is broken and the terminal never comes up for a pane, which is odd because every other pane on this host resolves its path'
})
const snippet = engine.search({ query: 'terminal path' }).hits[0]?.evidence?.snippet ?? ''
expect(snippet).toContain('[[')
expect(snippet).not.toContain('resolve [[terminal]] [[path]]')
})
it('flags a snippet it had to cut, and counts it on the result', async () => {
const { db, engine } = await open('ss-engine-snippet-truncated')
// The window is twelve tokens wide, and one of them is 4000 characters, so
// the token count is no bound at all on what a hit carries.
addSyntheticSession(db, { id: 1, text: `needle ${'x'.repeat(4000)}` })
const result = engine.search({ query: 'needle' })
expect(result.hits[0]?.evidence?.snippetTruncated).toBe(true)
expect(result.hits[0]?.evidence?.snippet.length).toBeLessThan(600)
expect(result.truncated.snippets).toBe(1)
})
it('leaves an ordinary snippet unflagged', async () => {
const { db, engine } = await open('ss-engine-snippet-whole')
addSyntheticSession(db, { id: 1, text: 'needle in a short line' })
const result = engine.search({ query: 'needle' })
expect(result.hits[0]?.evidence?.snippetTruncated).toBeUndefined()
expect(result.truncated.snippets).toBe(0)
})
})
describe('source presence comes from the files table, never a stat', () => {
it('calls a session with a live file record present', async () => {
const { db, engine } = await open('ss-engine-presence')
addSyntheticSession(db, { id: 1 })
expect(engine.search({ query: 'needle' }).hits[0]?.source).toBe('present')
})
it('calls a session with no file record unverifiable, and still returns it', async () => {
// Loss of contact is never evidence of absence: the hit stays on the page.
const { db, engine } = await open('ss-engine-presence-unknown')
addSyntheticSession(db, { id: 1, filePath: null })
const hits = engine.search({ query: 'needle' }).hits
expect(hits).toHaveLength(1)
expect(hits[0]?.source).toBe('unverifiable')
})
})
describe('the engine carries its own schema and puts it back', () => {
it('installs the vocabulary and the log over an index a writer built alone', async () => {
// The store creates none of these: PR 3's indexer can fill a whole index
// before anything opens an engine over it.
const { db, engine } = await open('ss-engine-installs')
addSyntheticSession(db, { id: 1, text: 'the coalesces path is slow' })
addSyntheticSession(db, { id: 2, text: 'coalesces again here' })
const result = engine.search({ query: 'coalescs' })
expect(result.unavailable).toEqual([])
expect(result.planner.route).toBe('typo+or')
expect(ids(result).sort()).toEqual(['1', '2'])
})
it('re-creates a vocabulary that vanished under a live engine', async () => {
const { db, engine } = await open('ss-engine-vocab-vanishes')
addSyntheticSession(db, { id: 1, text: 'coalesces here now' })
addSyntheticSession(db, { id: 2, text: 'coalesces again here' })
expect(engine.search({ query: 'coalescs' }).planner.route).toBe('typo+or')
db.exec('DROP TABLE messages_vocab')
const after = engine.search({ query: 'coalescs' })
expect(after.unavailable).toEqual([])
expect(after.planner.route).toBe('typo+or')
})
it('names the feature it cannot serve when the vocabulary has no source left', async () => {
// What an index being rebuilt by another handle looks like from here. The
// vocabulary can be created over a missing `messages_fts` and every query
// against it then fails, so the probe reads the source, not the view.
//
// With one FTS table there is no scope left to answer from, so this is now
// the boundary of the degrade: the engine names the feature and the search
// fails loudly on the table it cannot read, rather than returning an empty
// page that looks like an answer.
const { db, engine } = await open('ss-engine-vocab-source-gone')
addSyntheticSession(db, { id: 1, text: 'coalesces here now', role: 'user' })
db.exec('DROP TABLE messages_vocab; DROP TABLE messages_fts')
expect(ensureSessionSearchQuerySchema(db)).toEqual(['typo-repair'])
for (const scope of ['all', 'conversation'] as const) {
expect(() => engine.search({ query: 'coalesces', scope })).toThrow(/no such (fts5 )?table/i)
}
})
it('picks the feature back up when the source comes back', async () => {
const { db, engine } = await open('ss-engine-vocab-returns')
addSyntheticSession(db, { id: 1, text: 'coalesces here now' })
addSyntheticSession(db, { id: 2, text: 'coalesces again here' })
const fts = (
db.prepare("SELECT sql FROM sqlite_master WHERE name = 'messages_fts'").get() as {
sql: string
}
).sql
db.exec('DROP TABLE messages_vocab; DROP TABLE messages_fts')
expect(ensureSessionSearchQuerySchema(db)).toEqual(['typo-repair'])
db.exec(fts)
// Two, because the vocabulary only offers a term at least two rows carry.
addSyntheticSession(db, { id: 3, text: 'coalesces one more time' })
addSyntheticSession(db, { id: 4, text: 'coalesces once again' })
// Nothing throws on the way back up, so the recovery cannot come from the
// error path; it comes from the probe running per search.
const restored = engine.search({ query: 'coalescs' })
expect(restored.unavailable).toEqual([])
expect(restored.planner.route).toBe('typo+or')
})
})
describe('a query the engine had to cut says so', () => {
it('answers a query whose cap falls inside an astral character', async () => {
// The cut is on a whole code point rather than a code unit, so nothing
// downstream is handed half a surrogate pair. That is hygiene rather than a
// behaviour: the planner's tokenizer does not treat a lone surrogate as a
// token character, so it drops out of the terms either way. What this pins
// is that the boundary is answerable at all.
const { db, engine } = await open('ss-engine-surrogate-cap')
const kept = 'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH - 2)
addSyntheticSession(db, { id: 1, text: kept })
const result = engine.search({ query: `${kept} 😀 tail` })
expect(result.truncated.query).toBe(true)
expect(result.hits.map((hit) => hit.sessionId)).toEqual(['1'])
})
it('loads a candidate set larger than one batch of bound ids', async () => {
// The id list is as long as the candidate limit and every id is a bound
// parameter. No SQLite this stack can run refuses 1,100 of them, so this
// pins that batching returns the same answer, not that it rescues one.
const { db, engine } = await open('ss-engine-id-batching', {
sessionCandidateLimit: 1200
})
for (let id = 1; id <= 1100; id++) {
addSyntheticSession(db, { id, text: 'needle' })
}
const result = engine.search({ query: 'needle', limit: 5 })
expect(result.hits).toHaveLength(5)
expect(result.truncated.candidates).toBe(false)
})
it('reports truncation when the planner drops terms past its cap', async () => {
// The 56th term is the only one that matches. Without the flag this is a
// confident empty answer to a query the engine never finished reading.
const { db, engine } = await open('ss-engine-term-cap')
addSyntheticSession(db, { id: 1, text: 'onlyattheend' })
const query = `${Array.from({ length: 55 }, (_unused, n) => `term${n}`).join(' ')} onlyattheend`
const result = engine.search({ query })
expect(result.hits).toEqual([])
expect(result.truncated.query).toBe(true)
})
it('reports truncation when the query is longer than the engine will plan', async () => {
const { db, engine } = await open('ss-engine-length-cap')
addSyntheticSession(db, { id: 1, text: 'needle' })
const result = engine.search({ query: `needle ${'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH)}` })
expect(result.truncated.query).toBe(true)
})
it('claims no truncation for a query that fit', async () => {
const { db, engine } = await open('ss-engine-no-cap')
addSyntheticSession(db, { id: 1, text: 'needle' })
expect(engine.search({ query: 'needle' }).truncated.query).toBe(false)
})
})
describe('a query longer than the engine will plan is cut, not refused', () => {
it('cuts one enormous token down to the cap before FTS5 ever sees it', async () => {
const { db, engine } = await open('ss-engine-long-query')
// The planner already caps how many terms it will plan, so a long query of
// ordinary words is bounded without this. What is not bounded is a single
// token: one 100 kB word is one term, and FTS5 would carry the whole thing
// into the MATCH expression. The cut is observable because the indexed
// token is exactly the capped length.
addSyntheticSession(db, { id: 1, text: 'x'.repeat(SESSION_SEARCH_QUERY_MAX_LENGTH) })
expect(ids(engine.search({ query: 'x'.repeat(4000) }))).toEqual(['1'])
})
})
describe('unicode terms survive the round trip', () => {
it.each(['café', 'C', 'R', 'x', '修復', '안녕하세요'])('searches %s', async (text) => {
const { db, engine } = await open('ss-engine-unicode')
addSyntheticSession(db, { id: 1, text })
expect(engine.search({ query: text }).hits).toHaveLength(1)
})
})
src/main/ai-vault-search/session-search-engine.ts
-349
View File
@@ -1,349 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import type { TranscriptMessageRole } from '../ai-vault/session-transcript-consumers'
import { sliceAtCodeUnitLimit } from '../ai-vault/session-scanner-text-normalization'
import {
hasAiVaultSearchQueryOperators,
splitAiVaultSearchQuery,
type AiVaultSearchQuerySplit
} from '../../shared/ai-vault-search-query-operators'
import { matchesAiVaultQueryOperators } from '../../shared/ai-vault-session-filters'
import {
resolveSessionSearchLimit,
SESSION_SEARCH_QUERY_MAX_LENGTH,
type SessionSearchHit,
type SessionSearchRequest,
type SessionSearchResponse,
type SessionSearchScope,
type SessionSearchSourcePresence
} from './session-search-engine-types'
import { readIndexGeneration } from './session-search-index-generation'
import {
rankSessionHits,
type MessageRow,
type RankedSession,
type SessionRow
} from './session-search-hit-ranking'
import {
decodeSessionSearchCursor,
encodeSessionSearchCursor,
sessionSearchPageKey
} from './session-search-page-cursor'
import { planSessionSearchQuery } from './session-search-query-planner'
import { logSessionSearchQuery } from './session-search-query-log'
import {
SessionSearchRetrieval,
type RetrievalScope,
type Retrieved
} from './session-search-retrieval'
import { sessionRowFilter } from './session-search-row-filter'
import {
ensureSessionSearchQuerySchema,
type SessionSearchUnavailableFeature
} from './session-search-query-schema'
import { EMPTY_SNIPPET, sessionSearchSnippet } from './session-search-snippet'
import { sessionSourcePresence } from './session-search-source-presence'
/**
* Sessions retrieved before ranking cuts the page.
*
* Not a fixed constant (the reviewer's F13): it is the knob that trades page
* completeness for retrieval cost, and the right value depends on index size.
* Measurements behind this default, and what changing it costs, are in
* docs/reference/agent-session-search-query-tuning.md.
*/
export const SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT = 600
/** One ranked list plus what produced it; a page is a slice of `ranked`. */
type RankedPage = {
ranked: RankedSession[]
/** Null when no text was searched, so there is nothing to snippet from. */
retrieved: Retrieved | null
/**
* Retrieval may have missed a session: a cap ended it, not the data. True
* whether the candidate limit filled or the operator walk gave up scanning.
*/
incomplete: boolean
}
export type SessionSearchEngineOptions = {
sessionCandidateLimit?: number
/** Oldest transcript mtime a hit may come from; PR 3 derives it from retention. */
retentionCutoffMs?: number | null
/** Write each query to `search_log`. Off unless a caller asks (see query-log). */
logQueries?: boolean
}
/**
* Ranked session search over the PR 2 index.
*
* A library: it holds no timers, reads no settings, and knows nothing about
* Electron, IPC or a panel. It is handed a connection rather than opening one,
* because which process may open, rebuild or unlink the index file is PR 3b's
* decision and not a query engine's.
*
* **Every read here is a single statement, and no read transaction is ever
* open across an `await`.** There is no `BEGIN` on this path, no `.iterate()`
* outliving its statement, and `search` is synchronous end to end. That is a
* constraint PR 2 measured rather than a style: a reader that pins a WAL
* snapshot holds off every checkpoint behind it, and the same 47 MB of writes
* that leave a 9.9 MB WAL grew to 266 MB with one `BEGIN` + `SELECT` held open.
*
* One search is one synchronous pass, and every page of it is a slice of the
* same ranked list. That list is rebuilt per page rather than streamed, which
* is what makes a page repeatable: within one index generation the same request
* ranks the same way, and a cursor from any other generation is refused.
*
* That fence is strict on purpose, and the cost is worth stating plainly: any
* committed read moves the generation, so while a backfill is running an
* outstanding cursor will be refused, often within a second. Pagination is
* usable against a settled index and unreliable against one still filling. The
* rejection carries both generations, so a caller that sees `stale-generation`
* knows the index moved rather than that it holds a bad cursor, and can quietly
* re-issue page one instead of showing anyone an error.
*/
export class SessionSearchEngine {
private retrieval: SessionSearchRetrieval
private readonly candidateLimit: number
/** Re-probed whenever a query proves it stale; see `withCapabilityRetry`. */
private unavailable: readonly SessionSearchUnavailableFeature[]
constructor(
private readonly db: SyncDatabase,
private readonly options: SessionSearchEngineOptions = {}
) {
this.candidateLimit = options.sessionCandidateLimit ?? SESSION_SEARCH_CANDIDATE_LIMIT_DEFAULT
// Installed here and not on the first search, so the generation triggers are
// watching before anything this engine will be asked to page over is
// written, and so retrieval below prepares against tables that exist.
this.unavailable = ensureSessionSearchQuerySchema(this.db)
this.retrieval = new SessionSearchRetrieval(this.db, !this.unavailable.includes('typo-repair'))
}
search(request: SessionSearchRequest): SessionSearchResponse {
const startedAt = performance.now()
this.probeCapabilities()
const generation = readIndexGeneration(this.db)
const scope = request.scope ?? 'all'
const sort = request.filters?.sort ?? 'relevance'
// Not a bare `slice`: cutting between a surrogate pair leaves a lone half
// that no tokenizer can match and that a caller cannot echo back.
const capped = sliceAtCodeUnitLimit(request.query, SESSION_SEARCH_QUERY_MAX_LENGTH)
const split = splitAiVaultSearchQuery(capped)
const retrievalScope: RetrievalScope = {
scope,
sort,
filter: sessionRowFilter(request.filters ?? {}, this.options.retentionCutoffMs ?? null),
matchesOperators: operatorPredicate(split),
candidateLimit: this.candidateLimit
}
// Decoded before any retrieval: a cursor the engine will refuse must not
// cost a query, and the caller has to hear about it either way.
const pageKey = sessionSearchPageKey(request)
const offset = request.cursor
? decodeSessionSearchCursor(request.cursor, generation, pageKey)
: 0
const plan = planSessionSearchQuery(split.text)
const { ranked, retrieved, incomplete } = this.withCapabilityRetry(() =>
plan.terms.length === 0
? this.operatorOnly(split, retrievalScope)
: this.text(plan, retrievalScope, sort)
)
const limit = resolveSessionSearchLimit(request.limit)
const page = ranked.slice(offset, offset + limit)
const hits = this.hits(page, scope, retrieved)
const hasMore = ranked.length > offset + limit
const response: SessionSearchResponse = {
hits,
unavailable: this.unavailable,
planner: {
route: retrieved?.route ?? 'or',
tier: scope,
...(retrieved?.repairedTerms ? { repairedTerms: retrieved.repairedTerms } : {})
},
page: {
hasMore,
cursor: hasMore ? encodeSessionSearchCursor(generation, offset + limit, pageKey) : null
},
truncated: {
// Decided by retrieval, which is the only layer that knows whether a cap
// ended it. Deriving it from the hits cannot work: an operator walk that
// gave up at its scan ceiling returns no hits, and so does a search that
// genuinely matched nothing.
candidates: incomplete,
snippets: hits.filter((hit) => hit.evidence?.snippetTruncated).length,
query: capped.length < request.query.length || plan.truncated
},
generation,
durationMs: performance.now() - startedAt
}
if (this.options.logQueries) {
logSessionSearchQuery(this.db, {
query: request.query,
route: response.planner.route,
hits: hits.length,
durationMs: response.durationMs
})
}
return response
}
/**
* Where the engine's own schema is created and checked, once per search.
*
* A capability is a fact about the file, not about this object: another handle
* can rebuild the index under a live connection, so a verdict cached in the
* constructor is wrong for the rest of the engine's life in both directions —
* it would keep reaching for a table that went away, and never pick one back
* up when it returned. Retrieval is only rebuilt when the answer changes, so
* the steady-state cost is one indexed lookup and nothing else.
*/
private probeCapabilities(): void {
const unavailable = ensureSessionSearchQuerySchema(this.db)
if (unavailable.join() === this.unavailable.join()) {
return
}
this.unavailable = unavailable
this.retrieval = new SessionSearchRetrieval(this.db, !unavailable.includes('typo-repair'))
}
/**
* Runs a retrieval, and re-probes once if it turns out the index no longer
* has what an earlier probe found.
*
* `probeCapabilities` already runs per search, so this only covers the window
* between that probe and the statement that reaches for the table. Losing a
* table there is a thrown error rather than a wrong verdict, so it re-probes
* and runs the search again.
*/
private withCapabilityRetry(run: () => RankedPage): RankedPage {
try {
return run()
} catch (error) {
if (!isMissingTableError(error)) {
throw error
}
this.probeCapabilities()
return run()
}
}
/**
* Operators with no free text still name a scope, so the answer is the newest
* sessions inside it. Ranked through the same path as a text query, because
* forks must fold here exactly as they do there or the same sessions answer
* `repo:x` and `word repo:x` differently. There is no relevance signal
* without text, so the order is always newest.
*/
private operatorOnly(split: AiVaultSearchQuerySplit, scope: RetrievalScope): RankedPage {
if (!hasAiVaultSearchQueryOperators(split)) {
return { ranked: [], retrieved: null, incomplete: false }
}
const { sessions, incomplete } = this.retrieval.recent(scope)
return { ranked: rankSessionHits(sessions, new Map(), 'newest'), retrieved: null, incomplete }
}
private text(
plan: ReturnType<typeof planSessionSearchQuery>,
scope: RetrievalScope,
sort: 'relevance' | 'newest'
): RankedPage {
const retrieved = this.retrieval.run(plan, scope)
// `match` already grouped to one best row per session.
const best = new Map<number, MessageRow>(retrieved.rows.map((row) => [row.session_row_id, row]))
// Operators cut here, after retrieval, so the candidate count still reports
// what the SQL limit saw: that is what tells a caller the limit was binding.
const sessions = this.retrieval.loadSessions([...best.keys()], scope)
// Counted before the operator predicate and before fork folding: the SQL
// LIMIT is what could have hidden a session, and it saw the unfiltered set.
return {
ranked: rankSessionHits(sessions, best, sort),
retrieved,
incomplete: best.size >= this.candidateLimit
}
}
/** Snippets and source presence are paid for by the page, never by the list. */
private hits(
page: readonly RankedSession[],
scope: SessionSearchScope,
retrieved: Retrieved | null
): SessionSearchHit[] {
const presence = sessionSourcePresence(
this.db,
page.map((entry) => entry.session.id)
)
return page.map((entry) => this.hit(entry, scope, retrieved, presence))
}
private hit(
entry: RankedSession,
scope: SessionSearchScope,
retrieved: Retrieved | null,
presence: ReadonlyMap<number, SessionSearchSourcePresence>
): SessionSearchHit {
const { session, message } = entry
const snippet =
message && retrieved
? sessionSearchSnippet(this.db, scope, message.rowid, retrieved.plan)
: EMPTY_SNIPPET
return {
...sessionFields(session),
score: entry.score,
...(entry.duplicateCount > 1 ? { duplicateCount: entry.duplicateCount } : {}),
source: presence.get(session.id) ?? 'unverifiable',
evidence: message
? {
role: message.role as TranscriptMessageRole,
timestamp: message.ts,
snippet: snippet.text,
...(snippet.truncated ? { snippetTruncated: true } : {})
}
: null
}
}
}
// SQLite reports a table that went away at the statement that reaches for it.
// `fts5` is in the message when the table is the vocabulary's target, which is
// the one an index rebuilt under a live connection loses first.
const MISSING_TABLE = /no such (fts5 )?table/i
function isMissingTableError(error: unknown): boolean {
return error instanceof Error && MISSING_TABLE.test(error.message)
}
/**
* The one reading of `repo:` / `path:`: the sessions panel's own predicate, over
* the columns the index stores. The engine has no project map, so a session's
* repo label falls back to its folder label, which is what the panel does for
* every session it cannot resolve a project for.
*/
function operatorPredicate(split: AiVaultSearchQuerySplit): (session: SessionRow) => boolean {
if (!hasAiVaultSearchQueryOperators(split)) {
return () => true
}
return (session) =>
matchesAiVaultQueryOperators(
{ cwd: session.cwd, filePath: session.file_path },
{ repoTerms: split.repoTerms, pathTerms: split.pathTerms }
)
}
function sessionFields(
session: SessionRow
): Omit<SessionSearchHit, 'score' | 'evidence' | 'source' | 'duplicateCount'> {
return {
agent: session.agent,
sessionId: session.session_id,
filePath: session.file_path,
codexHome: session.codex_home,
title: session.title,
cwd: session.cwd,
branch: session.branch,
updatedAt: session.updated_at,
messageCount: session.message_count,
resumeCommand: session.resume_command
}
}
src/main/ai-vault-search/session-search-file-write.test.ts
+13 -4
View File
@@ -137,8 +137,11 @@ it('rolls a whole file back when a write throws part way through its transaction
expect(matches(index.db, 'messages_fts', 'firstgeneration')).toBe(3)
expect(store.indexedFile(SYNTHETIC_TRANSCRIPT, null)?.byteOffset).toBe(40)
expect(errors).toHaveLength(1)
// The file is owed a re-read, which is the only reason anything was lost.
expect(store.takeStale().map((candidate) => candidate.file.path)).toEqual([SYNTHETIC_TRANSCRIPT])
// The row itself says the read failed, which is the only reason anything was
// lost and the only record that outlives this read.
expect(
index.db.prepare('SELECT state, fail_count FROM files WHERE path = ?').get(SYNTHETIC_TRANSCRIPT)
).toMatchObject({ state: 'failed', fail_count: 1 })
// And the connection is usable again: a transaction left open by the failure
// would take down every write after it, not just the one that threw.
@@ -590,10 +593,16 @@ it('writes nothing for an incomplete read and owes the file a whole re-read', ()
expect(counts(index.db)).toMatchObject({
sessions: 0,
messages: 0,
files: 0,
full: 0
})
expect(store.pendingFileCount).toBe(1)
// One row, holding nothing but the failure: an incomplete read indexes no
// content, and the count of how often it has happened at this stat is the
// only thing that stops the file being read again on every pass.
expect(index.db.prepare('SELECT byte_offset, state, fail_count FROM files').get()).toMatchObject({
byte_offset: 0,
state: 'failed',
fail_count: 1
})
expect(errors).toEqual([])
})
src/main/ai-vault-search/session-search-fts5-contract.test.ts
-172
View File
@@ -1,172 +0,0 @@
import { mkdtemp } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { removeTree } from '../../shared/windows-transient-lock-removal'
import type SyncDatabase from '../sqlite/sync-database'
import { indexTokens } from './session-search-query-planner'
import { ensureSessionSearchQuerySchema } from './session-search-query-schema'
import { openSessionSearchDatabase } from './session-search-schema'
// SQLite/FTS5 behaviours the query layer depends on. Each one cost a live
// debugging session; a refactor that reintroduces the trap fails here.
const FIRST_ROWID = 101
const SECOND_ROWID = 202
let tempRoots: string[] = []
afterEach(async () => {
await Promise.all(tempRoots.map((root) => removeTree(root)))
tempRoots = []
})
async function openDatabase(): Promise<SyncDatabase> {
const root = await mkdtemp(join(tmpdir(), 'orca-fts5-contract-'))
tempRoots.push(root)
return openSessionSearchDatabase(join(root, 'index.sqlite'))
}
function insertMessageRow(db: SyncDatabase, rowid: number, text: string): void {
db.prepare(
`INSERT INTO messages_fts(rowid, user_text, assistant_text, tool_text, identifiers)
VALUES (?, ?, '', '', '')`
).run(rowid, text)
}
describe('FTS5 aux functions take the table name, never an alias', () => {
it('rejects bm25 over an aliased table and accepts the table-name form', async () => {
const db = await openDatabase()
insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one')
expect(() =>
db.prepare('SELECT bm25(f) AS score FROM messages_fts f WHERE f MATCH ?').all('alpha')
).toThrow(/no such column: f/)
const scored = db
.prepare('SELECT bm25(messages_fts) AS score FROM messages_fts WHERE messages_fts MATCH ?')
.all('alpha') as { score: number }[]
expect(scored).toHaveLength(1)
expect(Number.isFinite(scored[0]?.score)).toBe(true)
db.close()
})
it('rejects snippet over an aliased table too', async () => {
const db = await openDatabase()
insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one')
expect(() =>
db
.prepare(
"SELECT snippet(f, -1, '[', ']', '…', 12) AS s FROM messages_fts f WHERE f MATCH ?"
)
.all('alpha')
).toThrow(/no such column: f/)
db.close()
})
})
describe('a rowid constraint beside MATCH is honoured only as a subselect', () => {
it('ignores `rowid = ?` and returns every match, first row first', async () => {
const db = await openDatabase()
insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one')
insertMessageRow(db, SECOND_ROWID, 'alpha capybara two')
const rows = db
.prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?')
.all('alpha', SECOND_ROWID) as { rowid: number }[]
// The planner drops the constraint entirely: both rows come back.
expect(rows.map((row) => row.rowid)).toEqual([FIRST_ROWID, SECOND_ROWID])
// A caller reading one row therefore gets the first match, not the one asked for.
const single = db
.prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?')
.get('alpha', SECOND_ROWID) as { rowid: number } | undefined
expect(single?.rowid).toBe(FIRST_ROWID)
db.close()
})
it('ignores `rowid IN (?)` the same way', async () => {
const db = await openDatabase()
insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one')
insertMessageRow(db, SECOND_ROWID, 'alpha capybara two')
const rows = db
.prepare('SELECT rowid FROM messages_fts WHERE messages_fts MATCH ? AND rowid IN (?)')
.all('alpha', SECOND_ROWID) as { rowid: number }[]
expect(rows.map((row) => row.rowid)).toEqual([FIRST_ROWID, SECOND_ROWID])
db.close()
})
it('honours `rowid IN (SELECT ?)` even with the session join on', async () => {
const db = await openDatabase()
db.prepare(
`INSERT INTO sessions(id,agent,session_id,file_path,title,resume_command)
VALUES (1,'claude','1','/synthetic/1','fixture','')`
).run()
for (const rowid of [FIRST_ROWID, SECOND_ROWID]) {
db.prepare("INSERT INTO messages(id,session_row_id,role) VALUES (?,1,'user')").run(rowid)
}
insertMessageRow(db, FIRST_ROWID, 'alpha marmoset one')
insertMessageRow(db, SECOND_ROWID, 'alpha capybara two')
// The shape the snippet read uses: the joins are what subtract a row whose
// session a purge cut loose, and they must not cost the rowid constraint
// its effect.
const snippet = db
.prepare(
`SELECT snippet(messages_fts, -1, '[', ']', '…', 12) AS s
FROM messages_fts
JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id
WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)`
)
.get('alpha', SECOND_ROWID) as { s: string } | undefined
expect(snippet?.s).toContain('capybara')
expect(snippet?.s).not.toContain('marmoset')
db.close()
})
})
describe('sessions.file_path is deliberately not unique', () => {
it('accepts two sessions sharing one store path', async () => {
const db = await openDatabase()
const insert = db.prepare(
`INSERT INTO sessions(agent, session_id, file_path, title, resume_command)
VALUES (?, ?, ?, ?, ?)`
)
// OpenCode and Cursor keep every session in one SQLite store; files.path is the key.
const storePath = '/home/user/.local/share/opencode/storage.db'
insert.run('opencode', 'ses_one', storePath, 'first', 'opencode --session ses_one')
expect(() =>
insert.run('opencode', 'ses_two', storePath, 'second', 'opencode --session ses_two')
).not.toThrow()
const rows = db
.prepare('SELECT session_id FROM sessions WHERE file_path = ? ORDER BY session_id')
.all(storePath) as { session_id: string }[]
expect(rows.map((row) => row.session_id)).toEqual(['ses_one', 'ses_two'])
db.close()
})
})
describe('the planner tokenizer draws the same boundaries as unicode61', () => {
// unicode61 folds case and strips Latin diacritics on both index and query side.
function asIndexed(token: string): string {
return token.toLowerCase().normalize('NFD').replaceAll(/\p{M}/gu, '')
}
it('produces exactly the terms fts5vocab reports for the same text', async () => {
const db = await openDatabase()
// The vocabulary is the engine's own object, not the store's.
ensureSessionSearchQuerySchema(db)
const corpus =
'resolveTerminalPath src/main/foo-bar.ts a.b C++ #123 修复 café naïve MAX_TOKEN x'
insertMessageRow(db, FIRST_ROWID, corpus)
const indexed = (
db.prepare('SELECT term FROM messages_vocab ORDER BY term').all() as { term: string }[]
).map((row) => row.term)
expect([...new Set(indexTokens(corpus).map(asIndexed))].sort()).toEqual(indexed)
db.close()
})
})
src/main/ai-vault-search/session-search-hit-ranking.test.ts
-102
View File
@@ -1,102 +0,0 @@
import { describe, expect, it } from 'vitest'
import { rankSessionHits, type MessageRow, type SessionRow } from './session-search-hit-ranking'
function session(id: number, overrides: Partial<SessionRow> = {}): SessionRow {
return {
id,
agent: 'claude',
session_id: String(id),
file_path: `/synthetic/${id}.jsonl`,
codex_home: null,
title: 'fixture',
cwd: '/repo/app',
branch: null,
updated_at: '2026-09-01T00:00:00.000Z',
message_count: 1,
resume_command: 'resume',
content_hash: null,
content_hash_count: 0,
...overrides
}
}
function match(id: number, score: number): MessageRow {
return { rowid: id, score, session_row_id: id, role: 'user', ts: null }
}
function matches(...rows: MessageRow[]): Map<number, MessageRow> {
return new Map(rows.map((row) => [row.session_row_id, row]))
}
describe('order', () => {
it('ranks by score under relevance and by recency under newest', () => {
const sessions = [
session(1, { updated_at: '2026-09-01T00:00:00.000Z' }),
session(2, { updated_at: '2026-09-09T00:00:00.000Z' })
]
const scores = matches(match(1, 10), match(2, 1))
expect(rankSessionHits(sessions, scores, 'relevance').map((e) => e.session.id)).toEqual([1, 2])
expect(rankSessionHits(sessions, scores, 'newest').map((e) => e.session.id)).toEqual([2, 1])
})
it.each(['relevance', 'newest'] as const)(
'breaks a %s tie by session, whatever order retrieval handed them over in',
(sort) => {
// A cursor is an offset into this list, so two entries that tie must not
// be free to swap between pages. Retrieval hands sessions over in
// whatever order the `IN (...)` lookup produced, which SQL does not
// promise, so the order below is deliberately reversed.
const sessions = [6, 5, 4, 3, 2, 1].map((id) => session(id))
const scores = matches(...sessions.map((entry) => match(entry.id, 5)))
expect(rankSessionHits(sessions, scores, sort).map((entry) => entry.session.id)).toEqual([
1, 2, 3, 4, 5, 6
])
}
)
it('prefers the shorter session when two match equally well', () => {
// The length prior: `0.02 · ln(1 + messages)`, subtracted per session.
const sessions = [session(1, { message_count: 5000 }), session(2, { message_count: 2 })]
const ranked = rankSessionHits(sessions, matches(match(1, 5), match(2, 5)), 'relevance')
expect(ranked.map((entry) => entry.session.id)).toEqual([2, 1])
expect(ranked[0]!.score).toBeGreaterThan(ranked[1]!.score)
})
})
describe('forks fold into one answer', () => {
const fork = (id: number, updatedAt: string): SessionRow =>
session(id, {
updated_at: updatedAt,
content_hash: 'shared-opening-prefix',
content_hash_count: 8
})
it('keeps the newest copy and counts the rest', () => {
const sessions = [
fork(1, '2026-09-01T00:00:00.000Z'),
fork(2, '2026-09-09T00:00:00.000Z'),
fork(3, '2026-09-05T00:00:00.000Z')
]
const ranked = rankSessionHits(
sessions,
matches(match(1, 9), match(2, 1), match(3, 5)),
'relevance'
)
expect(ranked).toHaveLength(1)
expect(ranked[0]!.session.id).toBe(2)
expect(ranked[0]!.duplicateCount).toBe(3)
})
it('leaves sessions with no shared prefix alone', () => {
const sessions = [session(1), session(2)]
const ranked = rankSessionHits(sessions, matches(match(1, 9), match(2, 5)), 'relevance')
expect(ranked.map((entry) => entry.duplicateCount)).toEqual([1, 1])
})
})
it('scores a session that matched no text at zero, less its length prior', () => {
// The operator-only page: there is no relevance signal, only an order.
const ranked = rankSessionHits([session(1, { message_count: 9 })], new Map(), 'newest')
expect(ranked[0]!.message).toBeNull()
expect(ranked[0]!.score).toBeLessThan(0)
})
src/main/ai-vault-search/session-search-hit-ranking.ts
-109
View File
@@ -1,109 +0,0 @@
import type { AiVaultAgent } from '../../shared/ai-vault-types'
import { isCollapsibleContentHash } from './session-search-content-hash'
import type { SessionSearchSort } from './session-search-engine-types'
// Subtracted per session: `0.02 · ln(1 + messages)`; slightly positive on both eval sets.
const LENGTH_PRIOR = 0.02
export type SessionRow = {
id: number
agent: AiVaultAgent
session_id: string
file_path: string
codex_home: string | null
title: string
cwd: string | null
branch: string | null
updated_at: string | null
message_count: number
resume_command: string
content_hash: string | null
content_hash_count: number
}
/** The one message that stands for a session: its best-scoring match. */
export type MessageRow = {
rowid: number
score: number
session_row_id: number
role: string
ts: string | null
}
export type RankedSession = {
session: SessionRow
/** Null on an operator-only page: the session matched no text at all. */
message: MessageRow | null
score: number
duplicateCount: number
}
/**
* Everything between "these sessions matched" and "this is the ranked list":
* the length prior, fork folding and the caller's order. Retrieval stays in SQL
* and nothing here touches the database.
*
* The whole list is returned, not a page: a cursor indexes into it, and slicing
* here would make page two a different ranking from page one. The engine cuts
* the page and only then pays for a snippet.
*/
export function rankSessionHits(
sessions: readonly SessionRow[],
matches: ReadonlyMap<number, MessageRow>,
sort: SessionSearchSort
): RankedSession[] {
const scored = collapseForks(
sessions.map((session) => {
const message = matches.get(session.id) ?? null
return {
session,
message,
score: (message?.score ?? 0) - LENGTH_PRIOR * Math.log(1 + session.message_count),
duplicateCount: 1
}
})
)
// Why a total order and not just the key: a cursor is an offset into this
// list, so two entries that tie must not be free to swap between pages.
scored.sort(
(left, right) =>
(sort === 'newest'
? (right.session.updated_at ?? '').localeCompare(left.session.updated_at ?? '')
: right.score - left.score) || left.session.id - right.session.id
)
return scored
}
/**
* Folds forked copies of one conversation into a single entry: same opening
* prefix, newest `updated_at` wins, the rest become `duplicateCount`. Done here
* and not at write time so index rows stay per file (cursors and deletes).
*/
function collapseForks(scored: RankedSession[]): RankedSession[] {
const groups = new Map<string, RankedSession[]>()
for (const entry of scored) {
const { content_hash: hash, content_hash_count: count, id } = entry.session
const key = isCollapsibleContentHash(hash, count) ? `hash:${hash}` : `session:${id}`
const group = groups.get(key)
if (group) {
group.push(entry)
} else {
groups.set(key, [entry])
}
}
const collapsed: RankedSession[] = []
for (const group of groups.values()) {
if (group.length === 1) {
collapsed.push(group[0]!)
continue
}
const winner = group.reduce((best, entry) => (isNewer(entry, best) ? entry : best))
collapsed.push({ ...winner, duplicateCount: group.length })
}
return collapsed
}
function isNewer(entry: RankedSession, best: RankedSession): boolean {
const order = (entry.session.updated_at ?? '').localeCompare(best.session.updated_at ?? '')
return order === 0 ? entry.score > best.score : order > 0
}
src/main/ai-vault-search/session-search-index-consumer.test.ts
+27 -58
View File
@@ -10,7 +10,7 @@ import {
userMessages,
type SessionSearchIndexFile
} from './session-search-index-test-fixture'
import { SessionSearchStore, STALE_PATH_LIMIT } from './session-search-store'
import { SessionSearchStore } from './session-search-store'
let index: SessionSearchIndexFile
let store: SessionSearchStore
@@ -41,6 +41,13 @@ function cursor(): number | null | undefined {
return store.indexedFile(SYNTHETIC_TRANSCRIPT, null)?.byteOffset
}
/** What the row itself says it still owes, which is the only record there is. */
function owed(): { state: string; fail_count: number } | undefined {
return index.db
.prepare('SELECT state, fail_count FROM files WHERE path = ?')
.get(SYNTHETIC_TRANSCRIPT) as { state: string; fail_count: number } | undefined
}
it('appends onto its own cursor and carries the content hash forward', async () => {
replayTranscriptRead({
messages: userMessages('first half', 3),
@@ -64,7 +71,7 @@ it('appends onto its own cursor and carries the content hash forward', async ()
.get() as { hash: string; count: number }
expect(second.count).toBe(first.count + 2)
expect(second.hash).not.toBe(first.hash)
expect(store.takeStale()).toEqual([])
expect(owed()).toMatchObject({ state: 'current', fail_count: 0 })
})
it('appends onto a file it read through and decoded no session from', async () => {
@@ -76,7 +83,7 @@ it('appends onto a file it read through and decoded no session from', async () =
outcome: { session: null, byteOffset: 100 }
})
expect(cursor()).toBe(100)
expect(store.takeStale()).toEqual([])
expect(owed()).toMatchObject({ state: 'current', fail_count: 0 })
replayTranscriptRead({
mode: 'append',
@@ -87,7 +94,7 @@ it('appends onto a file it read through and decoded no session from', async () =
expect(indexedMessages()).toBe(2)
expect(cursor()).toBe(220)
expect(store.takeStale()).toEqual([])
expect(owed()).toMatchObject({ state: 'current', fail_count: 0 })
})
it('declines an append that starts past its own cursor and records the file', async () => {
@@ -107,7 +114,7 @@ it('declines an append that starts past its own cursor and records the file', as
expect(indexedMessages()).toBe(3)
expect(cursor()).toBe(100)
expect(store.takeStale().map((candidate) => candidate.file.path)).toEqual([SYNTHETIC_TRANSCRIPT])
expect(owed()).toMatchObject({ state: 'due' })
})
it('declines a file whose identity changed under the same path', async () => {
@@ -127,7 +134,7 @@ it('declines a file whose identity changed under the same path', async () => {
})
expect(indexedMessages()).toBe(2)
expect(store.takeStale()).toHaveLength(1)
expect(owed()?.state).not.toBe('current')
})
it('never advances the cursor for an incomplete read', async () => {
@@ -152,7 +159,7 @@ it('never advances the cursor for an incomplete read', async () => {
}
).n
).toBe(3)
expect(store.takeStale()).toHaveLength(1)
expect(owed()?.state).not.toBe('current')
})
it('indexes nothing at all from a read that was incomplete from the start', async () => {
@@ -167,7 +174,11 @@ it('indexes nothing at all from a read that was incomplete from the start', asyn
expect(index.db.prepare('SELECT count(*) AS n FROM messages').get()).toEqual({
n: 0
})
expect(cursor()).toBeUndefined()
// No cursor, because nothing was read through. The row exists all the same:
// it is where the failure is counted, and a file that fails on its first read
// is exactly the one that has no row of its own to count on.
expect(cursor()).toBe(0)
expect(owed()).toMatchObject({ state: 'failed', fail_count: 1 })
})
it('drops a file whose parser returned no session', async () => {
@@ -207,7 +218,9 @@ it('writes nothing for a source whose parser cannot reach the channel', async ()
expect(index.db.prepare('SELECT count(*) AS n FROM sessions').get()).toEqual({
n: 0
})
expect(store.takeStale()).toEqual([])
// No row at all, which is the record: the next pass reads a path the
// file table does not name.
expect(owed()).toBeUndefined()
})
it('ignores a candidate older than the retention cutoff', async () => {
@@ -217,53 +230,9 @@ it('ignores a candidate older than the retention cutoff', async () => {
expect(index.db.prepare('SELECT count(*) AS n FROM sessions').get()).toEqual({
n: 0
})
expect(store.takeStale()).toEqual([])
})
it('stops writing while the store refuses writes, but remembers what it skipped', async () => {
store.setAcceptingWrites(false)
replayTranscriptRead({ messages: userMessages('paused', 3) })
expect(index.db.prepare('SELECT count(*) AS n FROM sessions').get()).toEqual({
n: 0
})
expect(errors).toEqual([])
// A pause is exactly the window in which every read is declined. Forgetting
// them would leave the whole paused span unindexed with nothing to replay it.
expect(store.takeStale().map((candidate) => candidate.file.path)).toEqual([SYNTHETIC_TRANSCRIPT])
})
it('keeps the paused re-read set when the retention window is reconfigured', async () => {
store.setAcceptingWrites(false)
replayTranscriptRead({ messages: userMessages('paused', 2) })
expect(store.pendingFileCount).toBe(1)
// The set records what still has to be read, not what is worth keeping. A
// window that now excludes this file is enforced where the re-read is
// dispatched, so nothing is written and the file leaves the set there.
store.setRetentionCutoffMs(Date.now())
expect(store.pendingFileCount).toBe(1)
store.setAcceptingWrites(true)
expect(store.takeStale()).toHaveLength(1)
replayTranscriptRead({ messages: userMessages('outside the window now', 2) })
expect(index.db.prepare('SELECT count(*) AS n FROM sessions').get()).toEqual({
n: 0
})
expect(store.pendingFileCount).toBe(0)
})
it('drops the oldest record rather than growing without a bound, and says so', () => {
store.setAcceptingWrites(false)
for (let index = 0; index < STALE_PATH_LIMIT + 5; index++) {
store.markStale(syntheticCandidate({ path: `/transcript-${index}.jsonl` }))
}
expect(store.pendingFileCount).toBe(STALE_PATH_LIMIT)
expect(store.droppedPendingFileCount).toBe(5)
const kept = store.takeStale().map((candidate) => candidate.file.path)
expect(kept).not.toContain('/transcript-0.jsonl')
expect(kept).toContain(`/transcript-${STALE_PATH_LIMIT + 4}.jsonl`)
// No row at all, which is the record: the next pass reads a path the
// file table does not name.
expect(owed()).toBeUndefined()
})
it('keeps the session list running when the index write fails', async () => {
@@ -282,7 +251,7 @@ it('keeps the session list running when the index write fails', async () => {
})
).not.toThrow()
expect(errors.length).toBeGreaterThan(0)
expect(store.takeStale()).toHaveLength(1)
expect(owed()?.state).not.toBe('current')
})
it('unregisters cleanly, leaving later reads unindexed', async () => {
@@ -369,5 +338,5 @@ it('keeps a proven file identity when a later read cannot stat it', async () =>
expect(indexedMessages()).toBe(4)
expect(cursor()).toBe(200)
expect(store.takeStale()).toHaveLength(1)
expect(owed()?.state).not.toBe('current')
})
src/main/ai-vault-search/session-search-index-consumer.ts
+47 -19
View File
@@ -7,6 +7,7 @@ import {
type TranscriptReadOutcome,
type TranscriptReadStart
} from '../ai-vault/session-transcript-consumers'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import { fileIdentity } from './session-search-file-cursor'
import type { SessionSearchFileWrite } from './session-search-index-writer'
import type { SessionSearchStore } from './session-search-store'
@@ -16,30 +17,22 @@ import type { SessionSearchStore } from './session-search-store'
*
* It keeps its own cursor in the `files` table and never consults the parse
* cache: the two answer different questions and diverge the moment either
* declines a read. Three refusals, each of which leaves the cursor where it
* was and records the file for a later whole re-read:
* declines a read.
*
* - `beginRead` returns null when this index's cursor is behind the offset an
* `append` continues from, or when the file's identity changed.
* - a buffering failure stops the read's rows without failing the session list.
* - an `incomplete` outcome never commits; those rows are not the whole span.
* Every refusal leaves the cursor where it was and writes what the next pass
* needs on the row itself, because the row is the only thing that outlives this
* read. A declined append is `due`: the index is behind on a span no append
* reaches, so the file has to be read whole. A read that started and did not
* commit is `failed`, counted, and stamped with the stat it failed at, which is
* what stops an unreadable transcript being retried on every pass for ever.
*/
export class SessionSearchIndexConsumer implements TranscriptConsumer {
constructor(private readonly store: SessionSearchStore) {}
beginRead(start: TranscriptReadStart): TranscriptReadConsumer | null {
const { candidate } = start
if (!this.store.acceptsCandidate(candidate)) {
// A pause is a reason not to write now, not a reason to forget the read.
// `markStale` applies the retention rule itself, so a candidate that is
// out of scope rather than merely paused is still dropped here.
this.store.markStale(candidate)
return null
}
// A parser that decodes where the channel cannot reach it reports every read
// as incomplete. Declining here is not the same as being behind: no re-read
// would help, so the file is not recorded either.
if (!parserPublishesMessages(candidate)) {
this.noteUnreachableParser(candidate)
return null
}
if (start.mode === 'append') {
@@ -48,7 +41,8 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
// This index never saw the span before `previousByteOffset`; appending
// here would leave a hole no later read can fill. A null cursor is the
// file a chunked read left half written, which no offset continues.
this.store.markStale(candidate)
// Either way the next pass has to read this file from the start.
this.store.setFileState(candidate.file.path, 'due')
return null
}
}
@@ -59,11 +53,42 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
start.identity
)
if (!write) {
this.store.markStale(candidate)
// A closed store, a candidate outside the retention window, or a row that
// moved under this read. Only a row that exists has anything to record.
this.store.setFileState(candidate.file.path, 'due')
return null
}
return new SessionSearchReadConsumer(this.store, start, write)
}
/**
* A source no read can ever index, recorded as one this index has seen.
*
* A parser that decodes where the message channel cannot reach it -- OpenCode's
* SQLite sessions today -- publishes nothing, so no read of it will ever
* commit a row. Leaving the file table silent about it is not free: the next
* pass sees a path the index holds nothing for, asks for a read, and asking
* over a warm cache drops the session list's own resume point. The sidebar's
* fold is thrown away and the whole database is decoded again, on every pass,
* for ever.
*
* The row written is the shape the store already has for a read that went
* through and decoded no session: cursor at the file's size, no session row.
* The decide step then skips it until its stat moves, and the retirement walk
* retires it like any other row when it goes.
*/
private noteUnreachableParser(candidate: SessionFileCandidate): void {
const write = this.store.beginWrite(candidate, 'replace', 0)
const committed =
write?.commit({
session: null,
byteOffset: candidate.file.sizeBytes ?? 0,
incomplete: false
}) === true
if (committed) {
this.store.writeCommitted(candidate)
}
}
}
class SessionSearchReadConsumer implements TranscriptReadConsumer {
@@ -105,7 +130,10 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
this.store.writeCommitted(candidate)
return
}
this.store.markStale(candidate)
// Counted against the stat it failed at, not merely recorded: a transcript
// the reader cannot open fails identically on every pass, and only a change
// to this stat can mean the file itself changed.
this.store.setFileState(candidate.file.path, 'failed', candidate.file.mtimeMs)
}
}
src/main/ai-vault-search/session-search-index-generation.test.ts
-320
View File
@@ -1,320 +0,0 @@
import { appendFile, mkdtemp, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, expect, it } from 'vitest'
import { removeTree } from '../../shared/windows-transient-lock-removal'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchEngine } from './session-search-engine'
import { readIndexGeneration } from './session-search-index-generation'
import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import type { SessionSearchCursorError } from './session-search-page-cursor'
import { openSessionSearchDatabase } from './session-search-schema'
import { SessionSearchStore } from './session-search-store'
import { parseTranscript, userRecord } from './session-search-transcript-fixtures'
let roots: string[] = []
let handles: SyncDatabase[] = []
afterEach(async () => {
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
for (const handle of handles) {
handle.close()
}
handles = []
await Promise.all(roots.map((root) => removeTree(root)))
roots = []
})
async function tempRoot(): Promise<string> {
const root = await mkdtemp(join(tmpdir(), 'orca-search-generation-'))
roots.push(root)
return root
}
/**
* A reader's own handle on the index, with the engine's schema installed.
*
* PR 2's store keeps its connection private, so a reader opens its own — which
* is what the fence has to survive: nothing this handle does moves the
* generation, and it must still see every writer's move.
*/
function reader(path: string): SyncDatabase {
const db = openSessionSearchDatabase(path)
handles.push(db)
// Constructing an engine is what installs the triggers.
new SessionSearchEngine(db)
return db
}
/** Indexes one transcript through the real consumer and returns its path. */
async function indexOneTranscript(root: string, store: SessionSearchStore): Promise<string> {
resetSessionParseCacheForTests()
const sessionId = `aaaaaaaa-0000-4000-8000-${String(roots.length).padStart(12, '0')}`
const path = join(root, `${Math.random().toString(36).slice(2)}.jsonl`)
await writeFile(path, `${userRecord(0, 'generation fixture needle', sessionId)}\n`)
const unregister = registerSessionSearchIndexConsumer(store)
try {
await parseTranscript(path)
} finally {
unregister()
}
return path
}
it('moves the generation forward when a committed read changes what a read returns', async () => {
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
const before = readIndexGeneration(db)
await indexOneTranscript(root, store)
expect(readIndexGeneration(db)).toBeGreaterThan(before)
} finally {
store.close()
}
})
it('moves the generation forward when an append adds rows to a live session', async () => {
// The first read of a file inserts its `files` row; every read after that
// updates it. An append changes a session's rank and its message count, so a
// cursor minted before it indexes into a list that no longer exists.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
const transcript = await indexOneTranscript(root, store)
const indexed = readIndexGeneration(db)
const unregister = registerSessionSearchIndexConsumer(store)
try {
resetSessionParseCacheForTests()
await appendFile(transcript, `${userRecord(1, 'a second needle turn')}\n`)
await parseTranscript(transcript)
} finally {
unregister()
}
expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).toEqual({ c: 2 })
expect(readIndexGeneration(db)).toBeGreaterThan(indexed)
} finally {
store.close()
}
})
it('moves the generation forward when a proven deletion hides a session', async () => {
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
const transcript = await indexOneTranscript(root, store)
const indexed = readIndexGeneration(db)
store.removeFile(transcript)
expect(readIndexGeneration(db)).toBeGreaterThan(indexed)
} finally {
store.close()
}
})
it('moves the generation forward when retention cuts a session loose', async () => {
// Retention deletes the session row and the file row in one transaction, then
// reclaims the messages over many. It is the first half that changes what a
// search returns, and the first half that has to move the generation.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
await indexOneTranscript(root, store)
const indexed = readIndexGeneration(db)
await store.purgeOlderThan(Date.now() + 60_000)
expect(db.prepare('SELECT COUNT(*) AS c FROM sessions').get()).toEqual({ c: 0 })
expect(readIndexGeneration(db)).toBeGreaterThan(indexed)
} finally {
store.close()
}
})
it('moves the generation when a purge reclaims rows nothing can reach', async () => {
// The drain writes only `messages`, and for a while that was argued to change
// no answer. Retrieval never saw those rows; the typo repair's dictionary
// did, because `messages_vocab` is a view over the FTS b-tree and lists a
// term whether or not a reader can reach it. See
// `session-search-orphan-rows.test.ts` for the answer that moved. The price
// of fencing it is a cursor refused once per batch while a purge runs.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
await indexOneTranscript(root, store)
// The shape an interrupted purge leaves: rows with no session row.
db.prepare('DELETE FROM sessions').run()
const orphaned = readIndexGeneration(db)
expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).not.toEqual({ c: 0 })
await store.purgeOlderThan(null)
expect(db.prepare('SELECT COUNT(*) AS c FROM messages').get()).toEqual({ c: 0 })
expect(readIndexGeneration(db)).toBeGreaterThan(orphaned)
} finally {
store.close()
}
})
it("leaves the generation alone when a replace swaps a session's own rows", async () => {
// The same trigger must not fire here, or every re-read of a large transcript
// would move the generation once per deleted row on top of the one bump its
// file record already makes. A replace deletes rows whose session row still
// stands, which is what the trigger's `WHEN` clause tests.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
await indexOneTranscript(root, store)
const rows = db.prepare('SELECT COUNT(*) AS c FROM messages').get() as { c: number }
const indexed = readIndexGeneration(db)
db.prepare('DELETE FROM messages WHERE session_row_id IN (SELECT id FROM sessions)').run()
expect(rows.c).toBeGreaterThan(0)
expect(readIndexGeneration(db)).toBe(indexed)
} finally {
store.close()
}
})
it('leaves the generation alone when a removal hides nothing', async () => {
// A backfill retires paths it never held; if that moved the generation, every
// cursor would be refused for as long as indexing ran.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
await indexOneTranscript(root, store)
const before = readIndexGeneration(db)
store.removeFile('/synthetic/never-indexed.jsonl')
expect(readIndexGeneration(db)).toBe(before)
} finally {
store.close()
}
})
it('keeps the generation across a reopen, because the bump rides its own commit', async () => {
// The bump is inside the transaction that changes visibility, so nothing can
// be lost to a crash and reopening need not invalidate anyone's cursor.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
reader(path)
const first = new SessionSearchStore(path, (error) => {
throw error
})
await indexOneTranscript(root, first)
const indexed = readIndexGeneration(reader(path))
first.close()
const second = new SessionSearchStore(path)
try {
expect(readIndexGeneration(reader(path))).toBe(indexed)
} finally {
second.close()
}
})
it('fences a reader against a writer it does not share a process with', async () => {
// The shape PR 3 creates: the indexer writes from the scanner child while an
// engine reads elsewhere. A generation cached in the reader's memory tracks
// only that reader's own writes, so it would stand still through the
// writer's deletion, honour the stale cursor, and skip a session.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const writer = new SessionSearchStore(path, (error) => {
throw error
})
try {
const transcripts: string[] = []
for (let n = 0; n < 3; n++) {
transcripts.push(await indexOneTranscript(root, writer))
}
const engine = new SessionSearchEngine(db)
const page = engine.search({ query: 'needle', limit: 1 })
expect(page.page.cursor).not.toBeNull()
writer.removeFile(transcripts[0]!)
// The reader never wrote anything, and must still refuse.
try {
engine.search({ query: 'needle', limit: 1, cursor: page.page.cursor! })
expect.unreachable('a page cursor must not survive another writer moving the index')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('stale-generation')
}
} finally {
writer.close()
}
})
it('re-creates a fence something dropped, on the next search', async () => {
// An index whose triggers are gone cannot move its generation, so every stale
// cursor would compare equal and be honoured against a list the caller never
// saw. The engine owns those triggers, so it puts them back.
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const store = new SessionSearchStore(path, (error) => {
throw error
})
try {
await indexOneTranscript(root, store)
const engine = new SessionSearchEngine(db)
db.exec('DROP TRIGGER search_generation_file_update')
engine.search({ query: 'needle' })
const restored = readIndexGeneration(db)
await indexOneTranscript(root, store)
expect(readIndexGeneration(db)).toBeGreaterThan(restored)
} finally {
store.close()
}
})
it('mints a distinct generation per change even when two handles write', async () => {
const root = await tempRoot()
const path = join(root, 'index.sqlite')
const db = reader(path)
const first = new SessionSearchStore(path, (error) => {
throw error
})
const second = new SessionSearchStore(path, (error) => {
throw error
})
try {
const seen: number[] = [readIndexGeneration(db)]
for (const store of [first, second, first, second]) {
await indexOneTranscript(root, store)
seen.push(readIndexGeneration(db))
}
// Read-then-write from two connections would hand out one value twice.
expect(new Set(seen).size).toBe(seen.length)
expect([...seen].sort((left, right) => left - right)).toEqual(seen)
} finally {
second.close()
first.close()
}
})
src/main/ai-vault-search/session-search-index-generation.ts
-97
View File
@@ -1,97 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
const GENERATION_KEY = 'index_generation'
/**
* Names of the triggers that move the generation. Exported so the query schema
* can check they are all still there before an engine trusts a cursor.
*/
export const SESSION_SEARCH_GENERATION_TRIGGERS = [
'search_generation_file_insert',
'search_generation_file_update',
'search_generation_file_delete',
'search_generation_orphan_reclaim'
] as const
const BUMP = `INSERT INTO meta(key, value) VALUES ('${GENERATION_KEY}', '1')
ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1;`
/**
* The fence, as three triggers on `files`.
*
* Why `files`. Every transaction the store opens that can change what a search
* returns writes this table: a committed read upserts the file's cursor beside
* its rows, a chunk of a long read upserts the partial sentinel beside its
* prefix, `removeFile` deletes the row with the session, and retention deletes
* the file row in the same transaction as the session row.
*
* And why `messages` as well, for orphans only. Retention's second half
* reclaims rows whose session row is already gone, and touches neither table
* above. It was left unfenced on the argument that those rows answer nothing,
* which is true of retrieval and was not true of the whole engine: the typo
* repair's dictionary is `messages_vocab`, a view over the FTS b-tree that
* lists a term whether or not a reader can reach the rows carrying it, and
* reclaiming them moved which word a query was repaired to. The repair now
* counts live rows instead, so the common case is fixed at its source; this
* trigger is what makes the fence true rather than nearly true, because the
* vocabulary still decides which candidates survive its scan limit.
*
* The `WHEN` clause is what keeps it free. `removeFile` deletes a session's
* rows while its `sessions` row still stands, so it does not fire here; a
* replace cuts the old `sessions` row loose and leaves its messages to the
* drain (PR 2 round 10). Both already bump through `files`; the drain is the
* only path that deletes a row whose session is gone, and it fires here. The cost of the fence is real and worth naming: a
* cursor outstanding while a purge runs is refused once per batch, which
* `SessionSearchCursorError` reports as `stale-generation` so a caller
* re-issues page one rather than showing anyone an error.
*
* A trigger rather than a call the writer makes, for two reasons. PR 4 does not
* own the writer, and more importantly the fence has to hold for writers this
* process cannot see: the triggers live in the file, so PR 3's indexer in the
* scanner child moves the generation without knowing a reader exists.
*
* Correctness comes from where the increment runs, not from what it counts. It
* is one statement inside the writer's own `BEGIN IMMEDIATE`, so it commits
* with the change it describes and two connections cannot mint one value twice.
* It over-counts in one harmless direction: a read that decoded no session from
* a file the index also held no session for advances a cursor and bumps
* anyway. That refuses a cursor early; it never honours one late.
*/
export const SESSION_SEARCH_GENERATION_SQL = `
CREATE TRIGGER IF NOT EXISTS search_generation_file_insert AFTER INSERT ON files BEGIN
${BUMP}
END;
CREATE TRIGGER IF NOT EXISTS search_generation_file_update AFTER UPDATE ON files BEGIN
${BUMP}
END;
CREATE TRIGGER IF NOT EXISTS search_generation_file_delete AFTER DELETE ON files BEGIN
${BUMP}
END;
CREATE TRIGGER IF NOT EXISTS search_generation_orphan_reclaim AFTER DELETE ON messages
WHEN NOT EXISTS (SELECT 1 FROM sessions WHERE id = OLD.session_row_id) BEGIN
${BUMP}
END;
`
/**
* A monotone id for what the index currently publishes.
*
* A search page is a slice of one ranked list, so a cursor only means anything
* against the snapshot that produced it. Every change to what a read can return
* moves this on, and a cursor minted under an older value is refused rather
* than silently re-run against a list it no longer indexes into.
*
* Read from the database on every call, never cached in a process. The writer
* and the reader need not be the same one: PR 3's indexer runs in the scanner
* child while an engine reads elsewhere, and any number of handles may be open
* on one file. A generation cached in memory only ever tracks that process's
* own writes, so a reader would see another writer's deletions while its
* generation stood still, honour a stale cursor, and skip a session.
*/
export function readIndexGeneration(db: SyncDatabase): number {
const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(GENERATION_KEY) as
| { value: string }
| undefined
const parsed = row ? Number(row.value) : Number.NaN
return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0
}
src/main/ai-vault-search/session-search-index-pass.test.ts
+194
View File
@@ -0,0 +1,194 @@
import { appendFile, rm, stat, utimes } from 'node:fs/promises'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
import { runSessionSearchIndexPass } from './session-search-index-pass'
import { parseTranscript } from './session-search-transcript-fixtures'
import {
claudeLines,
openSessionSearchIndexerHarness,
writeClaudeTranscript,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
import { discoverSessionSearchCandidates } from './session-search-scan-roots'
import { SessionSearchStore } from './session-search-store'
const FIRST = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
const SECOND = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff'
let harness: SessionSearchIndexerHarness
let store: SessionSearchStore
let errors: unknown[]
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
errors = []
harness = await openSessionSearchIndexerHarness('ss-index-pass')
await writeClaudeTranscript(transcript(FIRST), ['the first transcript'], FIRST)
await writeClaudeTranscript(transcript(SECOND), ['the second transcript'], SECOND)
store = openStore()
})
afterEach(async () => {
resetTranscriptConsumersForTests()
store.close()
await harness.cleanup()
})
function transcript(sessionId: string): string {
return join(harness.claudeProjectDir, `${sessionId}.jsonl`)
}
function openStore(): SessionSearchStore {
const opened = new SessionSearchStore(harness.databasePath, (error) => errors.push(error))
registerSessionSearchIndexConsumer(opened)
return opened
}
async function candidates() {
return (
await discoverSessionSearchCandidates(harness.roots, {
limitPerAgent: Number.POSITIVE_INFINITY
})
).candidates
}
/** What a pass hands the read loop: the store's rows, read once. */
function rows() {
return new Map(store.files().map((row) => [row.path, row]))
}
function pass(options: { overdue?: () => boolean } = {}) {
return runSessionSearchIndexPass(store, [], { rows: rows(), ...options })
}
async function passOverAll(options: { overdue?: () => boolean } = {}) {
return runSessionSearchIndexPass(store, await candidates(), { rows: rows(), ...options })
}
function states(): Record<string, string> {
return Object.fromEntries(store.files().map((row) => [row.path, row.state]))
}
it('re-reads nothing it already holds, even with a cold session-list cache', async () => {
const first = await passOverAll()
expect(first.stats.fullParses).toBe(2)
// A restart: the parse cache is gone, the index's `files` table is not.
store.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store = openStore()
const second = await passOverAll()
expect(second.stats).toMatchObject({ fullParses: 0, incremental: 0, reused: 0, bytesRead: 0 })
expect(errors).toEqual([])
})
it('resumes into a grown transcript instead of re-reading it whole', async () => {
await passOverAll()
await appendFile(transcript(FIRST), `${claudeLines(['a later turn'], FIRST, 10).join('\n')}\n`)
const second = await passOverAll()
expect(second.stats).toMatchObject({ incremental: 1, fullParses: 0 })
})
// Nothing is recorded about what a deadline cut off, because being owed is a
// fact about the row: the file is read on the next pass for the same reason it
// was owed on this one.
it('leaves what it ran out of time for owed, with nothing written down', async () => {
const all = await candidates()
const cut = await runSessionSearchIndexPass(store, all, { rows: rows(), overdue: () => true })
expect(cut.outOfTime).toBe(true)
expect(store.files()).toHaveLength(1)
const second = await passOverAll()
expect(second.stats.fullParses).toBe(1)
expect(store.files()).toHaveLength(2)
})
// The deadline is never applied before the pass has read anything, so a single
// transcript larger than one deadline is read alone rather than starved.
it('reads one file even when the deadline has already expired', async () => {
const only = (await candidates()).slice(0, 1)
const alone = await runSessionSearchIndexPass(store, only, { rows: rows(), overdue: () => true })
expect(alone.outOfTime).toBe(false)
expect(store.files()).toHaveLength(1)
})
it('skips a source the reader cannot even open without failing the pass', async () => {
const all = await candidates()
await rm(transcript(FIRST))
await runSessionSearchIndexPass(store, all, { rows: rows() })
// One session indexed, and the missing one recorded as a failed read rather
// than as content the index holds.
expect(harness.read((db) => db.prepare('SELECT count(*) AS n FROM sessions').get())).toEqual({
n: 1
})
expect(states()[transcript(FIRST)]).toBe('failed')
})
// Finding 6: mtime alone is not the freshness key. A transcript that grows
// while keeping its mtime (a same-second append, a restored timestamp) is a
// different file to the index, and reading only mtime would skip it forever.
it('re-reads a file that grew without its mtime moving', async () => {
const path = transcript(FIRST)
// A whole-millisecond stamp, so restoring it later reproduces it exactly.
const frozen = new Date(1_740_000_000_000)
await utimes(path, frozen, frozen)
await passOverAll()
await appendFile(path, `${claudeLines(['a same-mtime append'], FIRST, 20).join('\n')}\n`)
await utimes(path, frozen, frozen)
expect((await stat(path)).mtimeMs).toBe(frozen.getTime())
const second = await passOverAll()
expect(second.stats.fullParses + second.stats.incremental).toBe(1)
})
// Finding 5: the decision reads the session list's cache and then changes it,
// so outside the per-path lane an overlapping list parse stores its entry in
// between and the forced read degrades into a reuse.
it('is not overtaken by a list parse racing the same path', async () => {
const path = transcript(FIRST)
const all = await candidates()
const only = all.filter((candidate) => candidate.file.path === path)
// The list parses this path first, so its cursor covers the file, and again
// concurrently with the index's pass so the two interleave.
await parseTranscript(path)
await Promise.all([
parseTranscript(path),
runSessionSearchIndexPass(store, only, { rows: rows() })
])
expect(harness.read((db) => db.prepare('SELECT count(*) AS n FROM sessions').get())).toEqual({
n: 1
})
})
// Finding 4d: a declined read is a parse that returns normally and indexes
// nothing. It has to leave the row owing a read, not looking covered.
it('leaves a declined read owed rather than recorded as held', async () => {
const only = (await candidates()).slice(0, 1)
// What a store that refuses a write looks like from the consumer's side: the
// read runs, and nothing is written.
store.beginWrite = () => null
const stats = await runSessionSearchIndexPass(store, only, { rows: rows() })
expect(stats.stats.fullParses).toBe(1)
expect(harness.read((db) => db.prepare('SELECT count(*) AS n FROM sessions').get())).toEqual({
n: 0
})
expect(store.files()).toEqual([])
})
it('reads nothing when there is nothing to read', async () => {
expect((await pass()).stats).toMatchObject({ fullParses: 0 })
})
src/main/ai-vault-search/session-search-index-pass.ts
+84
View File
@@ -0,0 +1,84 @@
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
import {
createSessionParseStats,
parseAgentSessionFileCached,
type SessionParseStats
} from '../ai-vault/session-scanner-parse-cache'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import { fileIdentity } from './session-search-file-cursor'
import { sessionSearchReadDecision } from './session-search-read-decision'
import type { SessionSearchFileRow, SessionSearchStore } from './session-search-store'
export type SessionSearchIndexPassOptions = {
signal?: AbortSignal
/** The store's rows for this pass, read once. Absent means the index holds nothing. */
rows: ReadonlyMap<string, SessionSearchFileRow>
/**
* True once the pass has spent its wall-clock deadline. The one bound on how
* long a pass reads for: files and bytes are proxies for time, and the thing
* worth capping is the share of the wall clock an unasked background index
* takes. Never applied before the pass has read anything, so an oversized
* transcript is read alone rather than deferred for ever.
*/
overdue?: () => boolean
}
/**
* Reads whatever the decide step says is owed, until the deadline.
*
* Nothing is recorded about what it did not reach. A candidate the deadline cut
* off is still owed on the next pass for the same reason it was owed on this
* one — its row says so — so there is no queue to keep, nothing to bound, and
* nothing to drop. What the reads themselves leave behind is written by the
* index consumer onto the rows.
*/
export async function runSessionSearchIndexPass(
store: SessionSearchStore,
candidates: readonly SessionFileCandidate[],
options: SessionSearchIndexPassOptions
): Promise<{ stats: SessionParseStats; outOfTime: boolean }> {
const stats = createSessionParseStats()
const cutoffMs = store.retentionCutoff
let read = 0
let outOfTime = false
for (const candidate of candidates) {
throwIfAiVaultScanCancelled(options.signal)
const path = candidate.file.path
const row = options.rows.get(path)
const decision = sessionSearchReadDecision({
candidate,
row,
// Only asked for a path the index holds something for; for the rest the
// decision is already made and this would be a query per new file.
cursor: row ? store.indexedFile(path, fileIdentity(candidate.file)) : null,
cutoffMs
})
if (decision === 'skip') {
continue
}
// The decide step is one cursor lookup, so it runs for the whole list even
// once the deadline has gone: knowing what is owed costs nothing, and the
// count of what a pass left is worth more than the microseconds.
outOfTime ||= read > 0 && options.overdue?.() === true
if (outOfTime) {
continue
}
// The clock the deadline reads is one the owner may close behind: the read
// below writes to the store, so stop here rather than on a shut handle.
throwIfAiVaultScanCancelled(options.signal)
read += 1
try {
await parseAgentSessionFileCached(candidate, process.platform, stats, decision)
} catch (error) {
throwIfAiVaultScanCancelled(options.signal)
// The reader reports a read it could not finish to the consumer, which is
// what records the failure on the row; nothing is counted here.
console.warn(
'[ai-vault-search] indexing skipped',
candidate.agent,
error instanceof Error ? error.name : 'ParseError'
)
}
}
return { stats, outOfTime }
}
src/main/ai-vault-search/session-search-index-writer.test.ts
+2 -4
View File
@@ -112,13 +112,12 @@ it('refuses to commit a write whose file was removed mid-read', () => {
it('declines a behind cursor in beginRead before it ever reaches the store', () => {
const attempted: number[] = []
const stub = {
acceptsCandidate: () => true,
indexedFile: () => ({ byteOffset: 100, mtimeMs: 1, sizeBytes: 1 }),
beginWrite: (_candidate: unknown, _mode: unknown, previousByteOffset: number) => {
attempted.push(previousByteOffset)
return { add: () => undefined, commit: () => true }
},
markStale: () => undefined
setFileState: () => undefined
} as unknown as SessionSearchStore
const consumer = new SessionSearchIndexConsumer(stub)
@@ -144,7 +143,6 @@ it('declines a behind cursor in beginRead before it ever reaches the store', ()
it("hands the read's identity accessor to the store", () => {
const captured: unknown[] = []
const stub = {
acceptsCandidate: () => true,
indexedFile: () => null,
beginWrite: (
_candidate: unknown,
@@ -155,7 +153,7 @@ it("hands the read's identity accessor to the store", () => {
captured.push(identity)
return { add: () => undefined, commit: () => true }
},
markStale: () => undefined
setFileState: () => undefined
} as unknown as SessionSearchStore
const identity = (): null => null
src/main/ai-vault-search/session-search-indexer-options.ts
+49
View File
@@ -0,0 +1,49 @@
import type { SessionSearchClock } from './session-search-clock'
import type { SessionSearchScanRoots } from './session-search-scan-roots'
/** Default cycle. Long enough that a machine with thousands of transcripts is
* not re-statting continuously, short enough that a live conversation shows up
* while the user is still in it. */
export const DEFAULT_SESSION_SEARCH_RECONCILE_INTERVAL_MS = 20_000
/** Newest-N per agent root: the same recency rule the session sidebar applies. */
export const DEFAULT_SESSION_SEARCH_RECENT_PER_AGENT = 12
/**
* A quarter of the interval: the only bound on how long one pass reads for.
*
* The timer re-arms after a pass settles, so a pass that spends its whole
* deadline is followed by a full interval of quiet — five seconds of reading in
* every twenty-five, a fifth of the wall clock, and the stated ceiling is a
* quarter. Files the deadline cut off go back on the queue at full speed rather
* than being read slowly, which is what a load-average back-off did instead.
*/
export const DEFAULT_SESSION_SEARCH_PASS_DEADLINE_FRACTION = 4
/**
* Cycles between whole-machine sweeps: five minutes at the default interval.
*
* A sweep is the only pass that sees a file nothing has told the indexer about
* — an old transcript deleted, a root that came back, a tree restored from a
* backup — so the cadence is what replaces every re-arm-on-recovery rule. A
* warm sweep is stats and readdirs, not reads, because the pass skips anything
* the index already covers at its current stat.
*/
export const DEFAULT_SESSION_SEARCH_FULL_SWEEP_EVERY_CYCLES = 15
/**
* Everything an indexer is. Immutable after construction: a settings change is
* `close()` and a new instance, which is also how the index is thrown away
* (`close()`, `removeSessionSearchDatabase(databasePath)`, construct again).
*/
export type SessionSearchIndexerOptions = {
databasePath: string
roots: SessionSearchScanRoots
/** null = all history; otherwise only transcripts modified within this many days. */
historyDays: number | null
clock?: SessionSearchClock
reconcileIntervalMs?: number
recentPerAgent?: number
/** Wall time one pass may read for; the rest goes back on the queue. */
passDeadlineMs?: number
/** Cycles between whole-machine sweeps. */
fullSweepEveryCycles?: number
onError?: (error: unknown) => void
}
src/main/ai-vault-search/session-search-indexer-test-fixture.ts
+168
View File
@@ -0,0 +1,168 @@
import { mkdir, mkdtemp, rename, rm, stat, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import SyncDatabase from '../sqlite/sync-database'
import { isolatedScanRoots } from '../ai-vault/session-scanner-test-fixtures'
import type { SessionSearchClock, SessionSearchTimerHandle } from './session-search-clock'
import type { SessionSearchScanRoots } from './session-search-scan-roots'
import { assistantRecord, userRecord } from './session-search-transcript-fixtures'
const CLOCK_EPOCH_MS = 1_740_000_000_000
/** Wall time the indexer's guarantee is stated in, under the test's control. */
export class FakeSessionSearchClock implements SessionSearchClock {
private time = CLOCK_EPOCH_MS
private nextId = 1
private nowCalls = 0
private readonly timers = new Map<number, { at: number; callback: () => void }>()
/**
* What each `now()` reading costs. A pass reads the clock once per file it is
* about to read, so this is how a test spends a pass's deadline without
* waiting: it is the wall time the reads themselves take.
*/
costPerNowMs = 0
/**
* Runs on every `now()`, with the call number. The only synchronous seam into
* a running pass: the deadline check is what a pass consults between files.
*/
onNow: ((call: number) => void) | null = null
now(): number {
const at = this.time
this.time += this.costPerNowMs
this.onNow?.(++this.nowCalls)
return at
}
setTimeout(callback: () => void, ms: number): SessionSearchTimerHandle {
const id = this.nextId++
this.timers.set(id, { at: this.time + ms, callback })
return id
}
clearTimeout(handle: SessionSearchTimerHandle): void {
this.timers.delete(handle as number)
}
/** Moves time forward and fires every timer that came due, in order. */
advance(ms: number): void {
this.time += ms
for (const [id, timer] of [...this.timers].sort((left, right) => left[1].at - right[1].at)) {
if (timer.at <= this.time) {
this.timers.delete(id)
timer.callback()
}
}
}
get pendingTimers(): number {
return this.timers.size
}
}
export type SessionSearchIndexerHarness = {
root: string
databasePath: string
roots: SessionSearchScanRoots
claudeProjectDir: string
/** A second connection: the store keeps its own private. */
read: <T>(query: (db: SyncDatabase) => T) => T
/** Plants what a killed writer would have left; nothing in the app writes here. */
write: <T>(query: (db: SyncDatabase) => T) => T
cleanup: () => Promise<void>
}
export async function openSessionSearchIndexerHarness(
name: string
): Promise<SessionSearchIndexerHarness> {
const root = await mkdtemp(join(tmpdir(), `${name}-`))
const roots = isolatedScanRoots(root)
const databasePath = join(root, 'index', 'index.sqlite')
return {
root,
databasePath,
roots,
claudeProjectDir: join(roots.claudeProjectsDir, 'project'),
read: (query) => withConnection(databasePath, true, query),
write: (query) => withConnection(databasePath, false, query),
cleanup: () => rm(root, { recursive: true, force: true })
}
}
function withConnection<T>(
path: string,
readonlyConnection: boolean,
query: (db: SyncDatabase) => T
): T {
const db = new SyncDatabase(path, { readonly: readonlyConnection })
try {
return query(db)
} finally {
db.close()
}
}
/** A native-chat-shaped Claude transcript: the same records the app itself writes. */
export async function writeClaudeTranscript(
path: string,
turns: readonly string[],
sessionId: string
): Promise<void> {
await mkdir(dirname(path), { recursive: true })
await writeFile(path, `${claudeLines(turns, sessionId, 0).join('\n')}\n`)
}
export function claudeLines(
turns: readonly string[],
sessionId: string,
startIndex: number
): string[] {
return turns.flatMap((turn, offset) => [
userRecord(startIndex + offset * 2, turn, sessionId),
assistantRecord(startIndex + offset * 2 + 1, `noted: ${turn}`, sessionId)
])
}
/**
* Replaces a transcript the way an editor or a sync client does: a new inode
* renamed over the old name. Same byte length on purpose, so the only thing
* that can tell the two files apart is their filesystem identity.
*/
export async function renameReplaceTranscript(
path: string,
turns: readonly string[],
sessionId: string
): Promise<void> {
const before = await stat(path)
const replacement = `${path}.replacement`
await writeClaudeTranscript(replacement, turns, sessionId)
await rename(replacement, path)
const later = new Date(before.mtimeMs + 5_000)
await utimes(path, later, later)
}
/**
* A message-graph transcript, the shape OpenClaw, Pi, OMP and Prime Agent
* write. The session id comes from the file name, so callers name the file.
*/
export async function writeMessageGraphTranscript(
path: string,
turns: readonly string[]
): Promise<void> {
await mkdir(dirname(path), { recursive: true })
const lines = turns.flatMap((turn, index) => [
JSON.stringify({
type: 'message',
timestamp: new Date(CLOCK_EPOCH_MS + index * 120_000).toISOString(),
message: { role: 'user', content: turn }
}),
JSON.stringify({
type: 'message',
timestamp: new Date(CLOCK_EPOCH_MS + index * 120_000 + 60_000).toISOString(),
message: { role: 'assistant', content: `noted: ${turn}` }
})
])
await writeFile(path, `${lines.join('\n')}\n`)
}
src/main/ai-vault-search/session-search-indexer.test.ts
+1090
View File
File diff suppressed because it is too large Load Diff
src/main/ai-vault-search/session-search-indexer.ts
+325
View File
@@ -0,0 +1,325 @@
import { systemSessionSearchClock, type SessionSearchClock } from './session-search-clock'
import {
DEFAULT_SESSION_SEARCH_FULL_SWEEP_EVERY_CYCLES,
DEFAULT_SESSION_SEARCH_PASS_DEADLINE_FRACTION,
DEFAULT_SESSION_SEARCH_RECENT_PER_AGENT,
DEFAULT_SESSION_SEARCH_RECONCILE_INTERVAL_MS,
type SessionSearchIndexerOptions
} from './session-search-indexer-options'
import { SessionSearchDirectoryListings } from './session-search-directory-listings'
import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
import { runSessionSearchPass } from './session-search-pass'
import { sessionSearchHistoryCutoffMs } from './session-search-retention-policy'
import { SessionSearchStore, type SessionSearchStateCounts } from './session-search-store'
import type { SessionSearchDegradedRoot } from './session-search-degraded-roots'
import { SessionSearchWorkLoop } from './session-search-work-loop'
/**
* Database paths a live indexer already owns.
*
* One process, one writer, one consumer registration per index. Two indexers on
* one path both register with the reader, so every transcript is read and
* written twice and the second write is fenced by the first at random. The
* recipe for every configuration change is close-then-construct, so the
* ordering that causes this is the one the recipe already rules out; this is
* what says so rather than letting it corrupt quietly.
*/
const liveIndexerPaths = new Set<string>()
export type SessionSearchIndexPhase = 'idle' | 'indexing' | 'current' | 'degraded' | 'closed'
export type SessionSearchIndexStatus = {
phase: SessionSearchIndexPhase
/** Rows whose content matches the file at the stat the row records. */
filesIndexed: number
/** Rows owed a whole read: a declined append, or a window that widened. */
filesDue: number
/** Rows whose last read did not commit. */
filesFailed: number
degradedRoots: SessionSearchDegradedRoot[]
lastReconcileAt: number | null
/** When a whole-machine sweep last finished; null until one has. */
lastSweepCompletedAt: number | null
}
/**
* Owns freshness for the index store: a whole-machine sweep, then a timer that
* keeps the newest N transcripts per agent reconciled and sweeps again every
* `fullSweepEveryCycles`.
*
* A library, not a service. It knows nothing about Electron, the app lifecycle,
* settings storage, IPC or the panel, and nothing here reads a setting or
* registers itself anywhere. Whoever constructs it decides all of that.
*
* **The store is the only memory.** Every question a pass asks between passes —
* what is owed a read, what has failed and how often, what the index holds and
* therefore what may have been deleted, what to report — is answered by a row
* in the `files` table. There is no queue, no watch set, no hold-out map and no
* counter with a reset rule.
*
* What is left here, and why none of it can be a row:
* - `previousRootsWithFiles`, the one bit per root the retirement walk's grace
* needs. Deliberately not durable: see the mountpoint trade in
* `session-search-deleted-sources.ts`.
* - `cyclesSinceSweep` and `sweepNext`, which are about the timer rather than
* about any file, and mean nothing to a second process.
* - `degradedRoots`, `lastReconcileAt` and `lastSweepCompletedAt`: what the last
* pass observed, held so `status()` can answer between passes.
* - `lastCounts`, the one cached query result, read only after `close()` so that
* describing what happened does not reopen a handle the owner has finished
* with. While the indexer is open every call re-queries.
*
* **Immutable after construction.** There is no `pause`, `resume`, `clear` or
* `setHistoryDays`. A configuration change is `close()` and a new instance;
* throwing the index away is
* `close(); removeSessionSearchDatabase(databasePath);` and a new instance.
* Widening retention is a new instance whose opening sweep admits the older
* files; narrowing is the purge that opens every full sweep.
*
* The guarantee it makes: while started, a transcript among the newest N per
* agent that grows, is replaced or is deleted is reflected in the index within
* one reconcile interval. Everything else is reached by the periodic sweep.
*/
export class SessionSearchIndexer {
private readonly ownershipPath: string
private readonly clock: SessionSearchClock
private readonly intervalMs: number
private readonly passDeadlineMs: number
private readonly recentPerAgent: number
private readonly fullSweepEveryCycles: number
private readonly onError: (error: unknown) => void
private readonly loop: SessionSearchWorkLoop
private readonly store: SessionSearchStore
private readonly unregister: () => void
/** Null until a pass has recorded one; an empty set is a real observation. */
private previousRootsWithFiles: ReadonlySet<string> | null = null
private degradedRoots: SessionSearchDegradedRoot[] = []
private lastReconcileAt: number | null = null
private lastSweepCompletedAt: number | null = null
private lastCounts: SessionSearchStateCounts | null = null
private cyclesSinceSweep = 0
private sweepNext = false
private started = false
private closed = false
constructor(private readonly options: SessionSearchIndexerOptions) {
this.ownershipPath = resolve(options.databasePath)
this.clock = options.clock ?? systemSessionSearchClock
this.intervalMs = options.reconcileIntervalMs ?? DEFAULT_SESSION_SEARCH_RECONCILE_INTERVAL_MS
this.passDeadlineMs =
options.passDeadlineMs ??
Math.max(1, Math.floor(this.intervalMs / DEFAULT_SESSION_SEARCH_PASS_DEADLINE_FRACTION))
this.recentPerAgent = options.recentPerAgent ?? DEFAULT_SESSION_SEARCH_RECENT_PER_AGENT
this.fullSweepEveryCycles = Math.max(
1,
options.fullSweepEveryCycles ?? DEFAULT_SESSION_SEARCH_FULL_SWEEP_EVERY_CYCLES
)
const onError = options.onError ?? ((error) => console.warn('[ai-vault-search]', error))
this.onError = onError
this.loop = new SessionSearchWorkLoop({
clock: this.clock,
intervalMs: this.intervalMs,
onFailure: onError
})
if (liveIndexerPaths.has(this.ownershipPath)) {
throw new Error(
`SessionSearchIndexer: ${options.databasePath} already has a live indexer; close it first`
)
}
// Store, registration and indexer share one lifetime, which is what makes
// the object immutable: there is no second open to get out of step with.
// Claimed only once the store is open, because a construction that throws
// has no `close()` to release the claim: registering first would leave the
// path owned by an object that does not exist, and every later attempt at
// it -- including the one that fixes whatever broke the open -- would be
// refused for the life of the process.
this.store = new SessionSearchStore(options.databasePath, onError)
liveIndexerPaths.add(this.ownershipPath)
this.store.setRetentionCutoffMs(this.cutoffMs())
this.unregister = registerSessionSearchIndexConsumer(this.store)
}
/** Runs a full sweep, then reconciles on the interval until closed. */
start(): Promise<void> {
if (this.closed || this.started) {
return this.loop.settled
}
this.started = true
this.sweepNext = true
return this.tick()
}
/**
* Runs one pass now, off the timer. A full pass sweeps every root.
*
* Refused before `start()` and after `close()`: a pass against an indexer
* nobody started writes the index once and leaves it to go stale with no
* timer armed to notice the next change, and a pass against a closed one has
* no store to write to. Both are caller bugs, so both throw rather than
* resolving as though a pass had run.
*/
reconcile(options: { full?: boolean } = {}): Promise<void> {
if (this.closed) {
throw new Error('SessionSearchIndexer.reconcile: the indexer is closed')
}
if (!this.started) {
throw new Error('SessionSearchIndexer.reconcile: start() first')
}
this.sweepNext ||= options.full === true
return this.tick()
}
/**
* What the index holds, read from the rows rather than tallied.
*
* A second connection can compute every number here with one `GROUP BY`,
* which is the point: nothing is counted as it happens, so nothing can drift
* from what the database actually holds or need a rule about when to reset.
*/
status(): SessionSearchIndexStatus {
// A closed indexer reports what it last knew: opening a shut handle to
// answer a call whose whole job is to describe what happened is how a close
// came to report a database error to the owner who asked for it.
const settled = (this.closed ? this.lastCounts : this.readCounts()) ?? {
current: 0,
due: 0,
failed: 0
}
return {
phase: this.phase(settled),
filesIndexed: settled.current,
filesDue: settled.due,
filesFailed: settled.failed,
degradedRoots: this.degradedRoots.map((root) => ({ ...root })),
lastReconcileAt: this.lastReconcileAt,
lastSweepCompletedAt: this.lastSweepCompletedAt
}
}
/** Stops everything. Nothing queued before this call may run afterwards. */
close(): void {
if (this.closed) {
return
}
// Read before the handle goes, so a status call afterwards reports what the
// index last held rather than opening a database its owner has finished with.
this.lastCounts = this.readCounts() ?? this.lastCounts
this.closed = true
// The loop, not just its timer: a task queued before this call would
// otherwise still run against a store this line is about to close.
this.loop.close()
this.unregister()
this.store.close()
liveIndexerPaths.delete(this.ownershipPath)
}
/** Tests only: everything else drives this through the timer. */
settled(): Promise<void> {
return this.loop.settled
}
private readCounts(): SessionSearchStateCounts | null {
try {
const counts = this.store.stateCounts()
this.lastCounts = counts
return counts
} catch (error) {
this.onError(error)
return this.lastCounts
}
}
/**
* `current` is a claim, so it takes all three: no row owed a read, no row
* whose last read failed, and a whole sweep that finished. `idle` is the
* other end of it — an indexer nobody started has not promised to index
* anything, and calling that `current` would claim an index nobody built is
* up to date.
*/
private phase(counts: SessionSearchStateCounts): SessionSearchIndexPhase {
if (this.closed) {
return 'closed'
}
if (!this.started) {
return 'idle'
}
// A root the pass could not read, or a file it could not read: both are gaps
// the index knows about and cannot close on its own.
if (this.degradedRoots.length > 0 || counts.failed > 0) {
return 'degraded'
}
return counts.due === 0 && this.lastSweepCompletedAt !== null ? 'current' : 'indexing'
}
private tick(): Promise<void> {
return this.loop.queue(
(signal) => this.pass(signal),
() => void this.tick()
)
}
private async pass(signal: AbortSignal): Promise<void> {
// The window moves with the clock, and the decide step reads it from the
// store. Setting it once at construction leaves a sweep purging rows that
// the very next candidate check happily re-indexes.
this.store.setRetentionCutoffMs(this.cutoffMs())
// The one bound on a pass: wall time. What it does not reach is still owed,
// because a row says so and nothing had to be written down.
const startedAt = this.clock.now()
const full = this.sweepNext
// Taken on entry, not cleared on the way out: a `reconcile({ full: true })`
// raised while this pass is running sets it again, and clearing it at the
// end would erase that request along with this pass's own.
this.sweepNext = false
try {
const result = await runSessionSearchPass({
store: this.store,
roots: this.options.roots,
full,
recentPerAgent: this.recentPerAgent,
previousRootsWithFiles: this.previousRootsWithFiles ?? undefined,
overdue: () => this.clock.now() - startedAt >= this.passDeadlineMs,
// One readdir per directory for the whole pass, shared by every step.
listings: new SessionSearchDirectoryListings(),
signal
})
if (!result.completed) {
// A pass cut short learned nothing about root health, and publishing its
// empty findings would clear a live alarm. A sweep stays owed.
this.sweepNext ||= full
return
}
this.degradedRoots = result.degradedRoots
this.previousRootsWithFiles = result.rootsWithFiles
this.lastReconcileAt = this.clock.now()
// A backlog outside the recency window is only visible to a sweep, so a
// pass that ran out of time asks for one. It is self-limiting: the first
// pass that finishes its reads hands the interval back to cycles.
this.sweepNext ||= result.outOfTime
if (full) {
this.lastSweepCompletedAt = this.lastReconcileAt
this.cyclesSinceSweep = 0
return
}
// A root that came back, a tree restored from a backup, an old transcript
// deleted: only a sweep sees any of it, and the count of cycles is the
// whole rule for when one is owed.
this.cyclesSinceSweep += 1
if (this.cyclesSinceSweep >= this.fullSweepEveryCycles) {
this.sweepNext = true
}
} catch (error) {
// The flag is this method's to hold, so it is this method's to give back:
// a pass that threw part way learned nothing, and losing it here would
// leave nothing armed to try again.
this.sweepNext ||= full
throw error
}
}
private cutoffMs(): number | null {
return sessionSearchHistoryCutoffMs(this.options.historyDays, this.clock.now())
}
}
import { resolve } from 'node:path'
src/main/ai-vault-search/session-search-lifecycle-matrix.test.ts
+378
View File
@@ -0,0 +1,378 @@
import { chmod, mkdir, rename, rm } from 'node:fs/promises'
import { delimiter, dirname, join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchIndexer } from './session-search-indexer'
import type { SessionSearchIndexerOptions } from './session-search-indexer-options'
import { removeSessionSearchDatabase } from './session-search-schema'
import {
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
writeClaudeTranscript,
writeMessageGraphTranscript,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
/*
* The lifecycle matrix: every operation a caller can perform, against every
* shape an unreachable root takes, against both ways discovery reports a root.
*
* The indexer is immutable, so "every operation" is a shorter list than it was:
* `pause`, `resume`, `clear`, `setHistoryDays` and `invalidate` are gone, and
* the two of them a caller still needs — a settings change and throwing the
* index away — are here as what replaced them, a new instance over the same
* path. In their place are the two passes the immutable design added: the
* periodic sweep, and a pass whose wall-clock deadline expires on its first file.
*
* What each cell asserts:
* A. No row is retired for a file that still exists. Throwing the index away
* is the one exception, and it is stated per operation rather than excused.
* B. The unreachable root is named in `degradedRoots`, by a real directory
* path — never the delimiter-joined label a merged discovery reports.
* C. The phase is never `current` while a root is degraded.
* D. Once the root is reachable again, a sweep indexes everything under it.
*
* Round 6 ran this as a throwaway harness on the previous design; it lives in
* the repository now. Two of its shapes changed with the stateless walk. The
* "present but empty mountpoint" shape is gone, because a readable root that
* lists nothing is no longer treated as unreachable — that is a root the user
* emptied, and `session-search-deleted-sources.ts` states the trade. In its
* place is a root whose transcripts sit behind an unreadable subdirectory,
* which is the partial-tree case the old shape never covered.
*/
const CAN_DENY_READ = process.platform !== 'win32' && process.getuid?.() !== 0
const INTERVAL_MS = 20_000
const SESSIONS = ['aaaaaaaa', 'bbbbbbbb', 'cccccccc']
type RootShape = {
name: string
/** Where the unreachable root's transcripts live, and where its files go. */
detachedRoot: (harness: SessionSearchIndexerHarness) => string
detachedFile: (harness: SessionSearchIndexerHarness, session: string) => string
writeDetached: (path: string, session: string) => Promise<void>
healthyFile: (harness: SessionSearchIndexerHarness, session: string) => string
writeHealthy: (path: string, session: string) => Promise<void>
}
const OPENCLAW_SESSION_DIR = join('agents', 'main', 'sessions')
const ROOT_SHAPES: RootShape[] = [
{
name: 'roots discovery reports one per directory',
detachedRoot: (harness) => harness.roots.claudeProjectsDir ?? '',
detachedFile: (harness, session) => join(harness.claudeProjectDir, `${session}.jsonl`),
writeDetached: (path, session) =>
writeClaudeTranscript(path, [`detached ${session}`], fullSessionId(session)),
healthyFile: (harness, session) => join(harness.roots.piSessionsDir ?? '', `${session}.jsonl`),
writeHealthy: (path, session) => writeMessageGraphTranscript(path, [`healthy ${session}`])
},
{
name: 'roots a merged discovery joins into one label',
detachedRoot: (harness) => join(harness.roots.openclawStateDir ?? '', 'agents'),
detachedFile: (harness, session) =>
join(harness.roots.openclawStateDir ?? '', OPENCLAW_SESSION_DIR, `${session}.jsonl`),
writeDetached: (path, session) => writeMessageGraphTranscript(path, [`detached ${session}`]),
healthyFile: (harness, session) =>
join(harness.roots.openclawLegacyStateDir ?? '', OPENCLAW_SESSION_DIR, `${session}.jsonl`),
writeHealthy: (path, session) => writeMessageGraphTranscript(path, [`healthy ${session}`])
}
]
type UnreachableShape = {
name: string
needsDeniedRead: boolean
/**
* Whether an empty index can see this at all. Reading the root itself is the
* one probe a pass makes with no rows to go on: a root that answers ENOENT is
* what an uninstalled agent answers too, and a readable root with an
* unreadable subdirectory is swallowed by the file walker, which returns
* rather than reporting. Both are invisible until the index holds a row under
* the root, which is the evidence the retirement walk runs on.
*/
visibleWithNoRows: boolean
detach: (root: string, transcriptDir: string, parked: string) => Promise<void>
attach: (root: string, transcriptDir: string, parked: string) => Promise<void>
}
const UNREACHABLE_SHAPES: UnreachableShape[] = [
{
name: 'the root itself is not there',
needsDeniedRead: false,
visibleWithNoRows: false,
detach: (root, _transcriptDir, parked) => rename(root, parked),
attach: (root, _transcriptDir, parked) => rename(parked, root)
},
{
name: 'the root refuses to list',
needsDeniedRead: true,
visibleWithNoRows: true,
detach: (root) => chmod(root, 0o000),
attach: (root) => chmod(root, 0o755)
},
{
name: 'the transcripts sit behind a directory that refuses to list',
needsDeniedRead: true,
visibleWithNoRows: false,
detach: (_root, transcriptDir) => chmod(transcriptDir, 0o000),
attach: (_root, transcriptDir) => chmod(transcriptDir, 0o755)
}
]
type Operation = {
name: string
/** True when the operation throws the index away, so no row survives it. */
clearsIndex?: boolean
/** Healthy-root sessions the operation deletes from disk. */
deletes?: readonly string[]
/** Construction options for every indexer this cell opens. */
options?: Partial<SessionSearchIndexerOptions>
run: (context: MatrixContext) => Promise<void>
}
const OPERATIONS: Operation[] = [
{ name: 'one cycle', run: (context) => context.cycle() },
{
name: 'two cycles',
run: async (context) => {
await context.cycle()
await context.cycle()
}
},
{
name: 'close and restart',
run: (context) => context.reopen()
},
{
name: 'two full reconciles',
run: async (context) => {
await context.indexer().reconcile({ full: true })
await context.indexer().reconcile({ full: true })
}
},
{
name: 'one healthy transcript deleted',
deletes: SESSIONS.slice(0, 1),
run: (context) => context.cycle()
},
{
name: 'every healthy transcript deleted',
deletes: SESSIONS,
run: async (context) => {
// Twice: a root that goes from holding transcripts to holding none in one
// pass is unverifiable for that pass, so the second is the proving one.
await context.indexer().reconcile({ full: true })
await context.indexer().reconcile({ full: true })
}
},
{
// The cadence that replaced every re-arm-on-recovery rule: no caller asks
// for this sweep, so the cell drives it off the timer alone.
name: 'the periodic sweep comes round',
options: { fullSweepEveryCycles: 2 },
run: async (context) => {
await context.cycle()
await context.cycle()
await context.cycle()
}
},
{
// Every pass is out of wall time from its first file, so each one hands
// almost all of its work back. A pass that read almost nothing must still
// not conclude anything about what it did not reach.
name: 'every pass out of time at its first file',
options: { passDeadlineMs: 0 },
run: async (context) => {
await context.cycle()
await context.cycle()
}
},
{
// What replaced `setHistoryDays`: a new instance over the same database.
// Every transcript here was written just now, so a 30-day window holds all
// of them and no row may be purged.
name: 'reconstructed for a narrower history window',
run: (context) => context.reopen({ historyDays: 30 })
},
{
// What replaced `clear()`, exactly as the PR body documents it.
name: 'the index thrown away and rebuilt',
clearsIndex: true,
run: (context) => context.reopen({ removeDatabase: true })
}
]
type MatrixContext = {
indexer: () => SessionSearchIndexer
/** Closes and constructs again over the same path: the immutable design's one edit. */
reopen: (args?: { historyDays?: number | null; removeDatabase?: boolean }) => Promise<void>
cycle: () => Promise<void>
detachedRoot: string
detachedPaths: string[]
}
function fullSessionId(prefix: string): string {
return `${prefix}-bbbb-4ccc-8ddd-eeeeeeeeeeee`
}
let harness: SessionSearchIndexerHarness
let clock: FakeSessionSearchClock
let indexer: SessionSearchIndexer | null
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
clock = new FakeSessionSearchClock()
harness = await openSessionSearchIndexerHarness('ss-lifecycle')
indexer = null
})
afterEach(async () => {
indexer?.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await harness.cleanup()
})
function open(overrides: Partial<SessionSearchIndexerOptions> = {}): SessionSearchIndexer {
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
reconcileIntervalMs: INTERVAL_MS,
...overrides
})
return indexer
}
/**
* Runs cycles until the index stops growing. Every operation but the
* out-of-time one settles on the first call; that one reads a transcript a pass.
*/
async function driveUntilIndexed(maxCycles: number): Promise<void> {
let held = indexedSessions().length
for (let cycle = 0; cycle < maxCycles; cycle++) {
clock.advance(INTERVAL_MS)
await indexer?.settled()
const now = indexedSessions().length
if (now === held) {
return
}
held = now
}
}
/** Session ids the index answers for, whichever agent wrote them. */
function indexedSessions(): string[] {
return harness
.read(
(db: SyncDatabase) =>
db.prepare('SELECT session_id AS id FROM sessions').all() as { id: string }[]
)
.map((row) => row.id)
.sort()
}
for (const roots of ROOT_SHAPES) {
for (const unreachable of UNREACHABLE_SHAPES) {
describe.skipIf(unreachable.needsDeniedRead && !CAN_DENY_READ)(
`${roots.name}, ${unreachable.name}`,
() => {
for (const operation of OPERATIONS) {
it(operation.name, async () => {
const detachedRoot = roots.detachedRoot(harness)
const detachedPaths = SESSIONS.map((session) => roots.detachedFile(harness, session))
const healthyPaths = SESSIONS.map((session) => roots.healthyFile(harness, session))
for (const [index, session] of SESSIONS.entries()) {
await roots.writeDetached(detachedPaths[index] ?? '', session)
await roots.writeHealthy(healthyPaths[index] ?? '', session)
}
const transcriptDir = dirname(detachedPaths[0] ?? '')
const parked = join(harness.root, 'parked-root')
await open(operation.options).start()
// A deadline that expires on the first file reads one transcript a
// pass, so the setup drives passes until the index has caught up.
await driveUntilIndexed(SESSIONS.length * 2)
const detachedIds = detachedPaths.map((_path, index) =>
roots === ROOT_SHAPES[0]
? fullSessionId(SESSIONS[index] ?? '')
: (SESSIONS[index] ?? '')
)
const healthyIds = SESSIONS.map((session) => session)
expect(indexedSessions()).toEqual([...detachedIds, ...healthyIds].sort())
// One cycle so the watch set holds the recency window, which is the
// state a running indexer is in when a volume goes away.
clock.advance(INTERVAL_MS)
await indexer?.settled()
await unreachable.detach(detachedRoot, transcriptDir, parked)
try {
const kept = SESSIONS.filter((session) => !operation.deletes?.includes(session))
for (const session of operation.deletes ?? []) {
await rm(healthyPaths[SESSIONS.indexOf(session)] ?? '')
}
await operation.run({
indexer: () => indexer as SessionSearchIndexer,
reopen: async (args = {}) => {
indexer?.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
if (args.removeDatabase === true) {
removeSessionSearchDatabase(harness.databasePath)
}
const overrides = { ...operation.options }
if ('historyDays' in args) {
overrides.historyDays = args.historyDays
}
await open(overrides).start()
await driveUntilIndexed(SESSIONS.length * 2)
},
cycle: async () => {
clock.advance(INTERVAL_MS)
await indexer?.settled()
},
detachedRoot,
detachedPaths
})
// A: nothing that still exists lost its rows.
const survivingDetached = operation.clearsIndex ? [] : detachedIds
expect(indexedSessions()).toEqual([...survivingDetached, ...kept].sort())
const status = indexer?.status()
const degraded = status?.degradedRoots.map((root) => root.root) ?? []
// With no rows under it, the only thing a pass can go on is
// whether the root itself refuses to list.
if (operation.clearsIndex && !unreachable.visibleWithNoRows) {
expect(degraded).not.toContain(detachedRoot)
} else {
// B: named, by a real directory rather than a joined label.
expect(degraded).toContain(detachedRoot)
expect(degraded.every((root) => !root.includes(delimiter))).toBe(true)
// C: not current while a root is degraded.
expect(status?.phase).not.toBe('current')
}
} finally {
await unreachable.attach(detachedRoot, transcriptDir, parked)
}
// D: reachable again, a sweep reads the whole tree back.
await mkdir(dirname(healthyPaths[0] ?? ''), { recursive: true })
await indexer?.reconcile({ full: true })
await driveUntilIndexed(SESSIONS.length * 2)
expect(indexedSessions()).toEqual(
[
...detachedIds,
...SESSIONS.filter((session) => !operation.deletes?.includes(session))
].sort()
)
})
}
}
)
}
}
src/main/ai-vault-search/session-search-live-transcript.test.ts
+6 -6
View File
@@ -193,16 +193,16 @@ it('indexes a file the session list already read past, once a whole read is aske
// The append continued from a byte offset the index never saw, so it declined.
expect(sessionsMatching('zygomorphic')).toEqual([])
const behind = store.takeStale()
expect(behind.map((candidate) => candidate.file.path)).toEqual([path])
for (const candidate of behind) {
requestWholeTranscriptRead(candidate.file.path)
}
// The index holds no row for this file at all, and that is the record: a
// path the file table does not name is read from the start by the next pass,
// which is what asks the reader to drop the session list's resume point.
expect(store.files()).toEqual([])
requestWholeTranscriptRead(path)
const reread = await parseTranscript(path)
expect(reread.stats).toMatchObject({ incremental: 0, fullParses: 1 })
expect(errors).toEqual([])
expect(sessionsMatching('zygomorphic')).toEqual([SESSION_ID])
expect(sessionsMatching('opening')).toEqual([SESSION_ID])
expect(store.takeStale()).toEqual([])
expect(store.files().map((row) => row.state)).toEqual(['current'])
})
src/main/ai-vault-search/session-search-merged-roots.test.ts
+135
View File
@@ -0,0 +1,135 @@
import { chmod, rm } from 'node:fs/promises'
import { delimiter, join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchIndexer } from './session-search-indexer'
import {
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
writeMessageGraphTranscript,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
// OpenClaw is the one agent whose roots are alternates for a single install, so
// discovery reports them as ONE discovery whose rootDir is every path joined by
// the platform's path delimiter. That string is not a directory: readdir on it
// answers ENOENT, containment never matches a real file, and a scan issue
// recorded against a real root never compares equal to it. Everything that
// judges a root works on the constituent directories, taken from the same
// source table discovery reads, never by splitting the label -- a directory may
// legally contain the delimiter.
const CAN_DENY_READ = process.platform !== 'win32' && process.getuid?.() !== 0
const INTERVAL_MS = 20_000
let harness: SessionSearchIndexerHarness
let clock: FakeSessionSearchClock
let indexer: SessionSearchIndexer
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
clock = new FakeSessionSearchClock()
harness = await openSessionSearchIndexerHarness('ss-merged-roots')
})
afterEach(async () => {
indexer.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await harness.cleanup()
})
/** OpenClaw reads `<stateDir>/agents/**` and keeps only paths through `sessions`. */
function openclawTranscript(stateDir: string, name: string): string {
return join(stateDir, 'agents', 'main', 'sessions', `${name}.jsonl`)
}
function sessionsMatching(term: string): string[] {
return harness.read((db: SyncDatabase) =>
(
db
.prepare(
`SELECT DISTINCT s.session_id AS id FROM messages_fts
JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id
WHERE messages_fts MATCH ? ORDER BY s.session_id`
)
.all(term) as { id: string }[]
).map((row) => row.id)
)
}
it.skipIf(!CAN_DENY_READ)('fences one merged root without taking its partner down', async () => {
const current = harness.roots.openclawStateDir ?? ''
const legacy = harness.roots.openclawLegacyStateDir ?? ''
const mounted = openclawTranscript(current, 'mounted-session')
const local = openclawTranscript(legacy, 'local-session')
await writeMessageGraphTranscript(mounted, ['a conversation on the mounted volume'])
await writeMessageGraphTranscript(local, ['a conversation on local disk'])
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
reconcileIntervalMs: INTERVAL_MS
})
await indexer.start()
expect(sessionsMatching('conversation').sort()).toEqual(['local-session', 'mounted-session'])
// One of the two roots goes away; the other is untouched.
await chmod(join(current, 'agents'), 0o000)
try {
await indexer.reconcile({ full: true })
const status = indexer.status()
const degraded = status.degradedRoots.map((root) => root.root)
// A real directory, not the joined string discovery reports.
expect(degraded).toContain(join(current, 'agents'))
expect(degraded.every((root) => !root.includes(delimiter))).toBe(true)
// Unprovable, so the unreadable root keeps its rows.
expect(sessionsMatching('mounted')).toEqual(['mounted-session'])
} finally {
await chmod(join(current, 'agents'), 0o755)
}
})
it('retires from one merged root while its partner is healthy', async () => {
const current = harness.roots.openclawStateDir ?? ''
const legacy = harness.roots.openclawLegacyStateDir ?? ''
const going = openclawTranscript(current, 'going-session')
await writeMessageGraphTranscript(going, ['a conversation about to be deleted'])
// A sibling in the same root, so deleting one leaves the root listing files
// and therefore healthy: this is a deletion, not an unmount.
await writeMessageGraphTranscript(openclawTranscript(current, 'sibling-session'), [
'a conversation beside it'
])
await writeMessageGraphTranscript(openclawTranscript(legacy, 'staying-session'), [
'a conversation that stays'
])
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
reconcileIntervalMs: INTERVAL_MS
})
await indexer.start()
expect(sessionsMatching('conversation').sort()).toEqual([
'going-session',
'sibling-session',
'staying-session'
])
// A genuine deletion inside a healthy root still retires normally.
await rm(going)
await indexer.reconcile({ full: true })
expect(sessionsMatching('deleted')).toEqual([])
expect(indexer.status().degradedRoots).toEqual([])
expect(sessionsMatching('conversation').sort()).toEqual(['sibling-session', 'staying-session'])
})
src/main/ai-vault-search/session-search-native-chat-indexing.test.ts
+113
View File
@@ -0,0 +1,113 @@
import { appendFile, mkdir, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchIndexer } from './session-search-indexer'
import {
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
// Reviewer F4, and the plan's fourth open decision: a conversation held in
// Orca's own chat is the same file in the same place as one held in the
// terminal, so it must be searchable through the same path with no panel
// mounted, no scanner service running, and nobody calling refresh. Everything
// below is the library and the filesystem.
const INTERVAL_MS = 20_000
const SESSION_ID = 'cccccccc-dddd-4eee-8fff-000000000000'
const CWD = '/repo/orca'
let harness: SessionSearchIndexerHarness
let clock: FakeSessionSearchClock
let indexer: SessionSearchIndexer
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
clock = new FakeSessionSearchClock()
harness = await openSessionSearchIndexerHarness('ss-native-chat')
})
afterEach(async () => {
indexer.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await harness.cleanup()
})
/** The rows Orca's native chat writes: uuid, block content, cwd on the first turn. */
function nativeChatTurn(uuid: string, role: 'user' | 'assistant', text: string): string {
const timestamp = new Date(1_740_000_000_000 + Number(uuid.slice(-2)) * 60_000).toISOString()
return JSON.stringify({
type: role,
uuid,
sessionId: SESSION_ID,
timestamp,
cwd: CWD,
gitBranch: 'main',
message: {
role,
...(role === 'assistant' ? { model: 'claude-fable-5' } : {}),
content: [{ type: 'text', text }]
}
})
}
function messageTexts(term: string): { role: string; session: string }[] {
return harness.read(
(db: SyncDatabase) =>
db
.prepare(
`SELECT m.role AS role, s.session_id AS session FROM messages_fts
JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id
WHERE messages_fts MATCH ? ORDER BY m.id`
)
.all(term) as { role: string; session: string }[]
)
}
it('indexes a native-chat conversation and its later turns with no panel and no service', async () => {
const path = join(harness.claudeProjectDir, `${SESSION_ID}.jsonl`)
await mkdir(harness.claudeProjectDir, { recursive: true })
await writeFile(
path,
`${[
nativeChatTurn('turn-01', 'user', 'why does the relay drop the lease at 105 seconds'),
nativeChatTurn('turn-02', 'assistant', 'that is the client silence watchdog, not a cliff')
].join('\n')}\n`
)
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
reconcileIntervalMs: INTERVAL_MS
})
await indexer.start()
expect(messageTexts('watchdog')).toEqual([{ role: 'assistant', session: SESSION_ID }])
expect(harness.read((db: SyncDatabase) => db.prepare('SELECT cwd FROM sessions').get())).toEqual({
cwd: CWD
})
// The conversation continues in the panel; nothing tells the index about it.
await appendFile(
path,
`${[
nativeChatTurn('turn-03', 'user', 'and the fleetwide 4408 bursts'),
nativeChatTurn('turn-04', 'assistant', 'those are desktop lease rotations, cohort waves')
].join('\n')}\n`
)
clock.advance(INTERVAL_MS)
await indexer.settled()
expect(messageTexts('cohort')).toEqual([{ role: 'assistant', session: SESSION_ID }])
expect(messageTexts('4408')).toEqual([{ role: 'user', session: SESSION_ID }])
expect(indexer.status().phase).toBe('current')
})
src/main/ai-vault-search/session-search-opencode-decline.test.ts
+177
View File
@@ -0,0 +1,177 @@
import { mkdirSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
// Only the thread hop is replaced: both implementations below are the repo's
// own in-process readers, which the worker entry calls on the other side.
export const openCodeParseCalls: string[] = []
vi.mock('../ai-vault/session-scanner-opencode-sqlite-worker-spawn', async () => {
const list = await import('../ai-vault/session-scanner-opencode-sqlite-list')
const parse = await import('../ai-vault/session-scanner-opencode-sqlite')
const own = await import('./session-search-opencode-decline.test')
return {
resolveOpenCodeSqliteWorkerEntryPath: () => null,
listOpenCodeSqliteSessionsViaWorker: (
args: Parameters<typeof list.listOpenCodeSqliteSessions>[0]
) => list.listOpenCodeSqliteSessions(args),
parseOpenCodeSqliteSessionViaWorker: (
args: Parameters<typeof parse.parseOpenCodeSqliteSession>[0]
) => {
own.openCodeParseCalls.push(args.sessionId)
return parse.parseOpenCodeSqliteSession(args)
}
}
})
import Database from '../sqlite/sync-database'
import { getSessionParseCacheEntry } from '../ai-vault/session-parse-cache-store'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import { buildOpenCodeSqliteCandidatePath } from '../ai-vault/session-scanner-opencode-sqlite-paths'
import { SessionSearchIndexer } from './session-search-indexer'
import {
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
writeClaudeTranscript,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
/*
* Round 12, F3. An OpenCode SQLite session decodes where the message channel
* cannot reach it, so no read of one will ever commit a row. The consumer
* declined it and wrote nothing, which left the file table silent about a
* source discovery returns on every pass: the decide step saw a path the index
* held nothing for, asked for a read, and asking for one over a warm cache
* drops the session list's own resume point. Every OpenCode session was fully
* decoded on every pass and the sidebar's fold was thrown away with it, which
* is the cache STA-1278 and STA-1417 added.
*/
const SESSION = 'ses_r12'
const CLAUDE_SESSION = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
let harness: SessionSearchIndexerHarness
let clock: FakeSessionSearchClock
let indexer: SessionSearchIndexer | null = null
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
clock = new FakeSessionSearchClock()
harness = await openSessionSearchIndexerHarness('ss-opencode-decline')
indexer = null
openCodeParseCalls.length = 0
})
afterEach(async () => {
indexer?.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await harness.cleanup()
})
function writeOpenCodeDb(path: string, sessionId: string): void {
const db = new Database(path)
db.exec(`
CREATE TABLE session (
id TEXT PRIMARY KEY, project_id TEXT NOT NULL, parent_id TEXT, slug TEXT NOT NULL,
directory TEXT NOT NULL, title TEXT NOT NULL, version TEXT NOT NULL, share_url TEXT,
summary_additions INTEGER, summary_deletions INTEGER, summary_files INTEGER,
summary_diffs TEXT, revert TEXT, permission TEXT,
time_created INTEGER NOT NULL, time_updated INTEGER NOT NULL, time_compacting INTEGER,
time_archived INTEGER, workspace_id TEXT, path TEXT, agent TEXT, model TEXT,
cost REAL DEFAULT 0 NOT NULL, tokens_input INTEGER DEFAULT 0 NOT NULL,
tokens_output INTEGER DEFAULT 0 NOT NULL, tokens_reasoning INTEGER DEFAULT 0 NOT NULL,
tokens_cache_read INTEGER DEFAULT 0 NOT NULL, tokens_cache_write INTEGER DEFAULT 0 NOT NULL,
metadata TEXT
);
CREATE TABLE message (
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, time_created INTEGER NOT NULL,
time_updated INTEGER NOT NULL, data TEXT NOT NULL
);
CREATE TABLE project (
id TEXT PRIMARY KEY, worktree TEXT NOT NULL, vcs TEXT, name TEXT, icon_url TEXT,
icon_color TEXT, time_created INTEGER NOT NULL, time_updated INTEGER NOT NULL,
time_initialized INTEGER, sandboxes TEXT NOT NULL, commands TEXT, icon_url_override TEXT
);
CREATE TABLE part (
id TEXT PRIMARY KEY, message_id TEXT NOT NULL, session_id TEXT NOT NULL,
time_created INTEGER NOT NULL, time_updated INTEGER NOT NULL, data TEXT NOT NULL
);
`)
db.prepare(
`INSERT INTO session (id, project_id, parent_id, slug, directory, title, version,
time_created, time_updated, agent, model, cost, tokens_input, tokens_output,
tokens_reasoning, tokens_cache_read, tokens_cache_write)
VALUES (?, 'proj-1', NULL, 'slug-1', '/tmp/opencode', 'OpenCode title', '1.0.0',
?, ?, 'build', '{"id":"glm"}', 0, 1, 1, 0, 0, 0)`
).run(sessionId, 1_740_000_000_000, 1_740_000_100_000)
db.prepare(
`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)`
).run(
'msg-1',
sessionId,
1_740_000_000_000,
1_740_000_000_000,
JSON.stringify({ role: 'user', time: { created: 1_740_000_000_000 } })
)
db.prepare(
`INSERT INTO part (id, message_id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?, ?)`
).run(
'part-1',
'msg-1',
sessionId,
1_740_000_000_000,
1_740_000_000_000,
JSON.stringify({ type: 'text', text: 'hello opencode' })
)
db.prepare(
`INSERT INTO project (id, worktree, name, time_created, time_updated, sandboxes)
VALUES ('proj-1', '/tmp/opencode', 'proj', ?, ?, '[]')`
).run(1_740_000_000_000, 1_740_000_000_000)
db.close()
}
it('reads an OpenCode session once, not on every pass', async () => {
const dbPath = join(harness.root, 'opencode-db', 'opencode.db')
mkdirSync(join(harness.root, 'opencode-db'), { recursive: true })
writeOpenCodeDb(dbPath, SESSION)
const claudePath = join(harness.claudeProjectDir, 'control.jsonl')
await writeClaudeTranscript(claudePath, ['control turn'], CLAUDE_SESSION)
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: { ...harness.roots, opencodeDbPaths: [dbPath] },
historyDays: null,
clock,
reconcileIntervalMs: 20_000,
onError: () => undefined
})
await indexer.start()
const syntheticPath = buildOpenCodeSqliteCandidatePath(dbPath, SESSION)
const openCodeAfterFirst = getSessionParseCacheEntry(syntheticPath)
const claudeAfterFirst = getSessionParseCacheEntry(claudePath)
await indexer.reconcile()
await indexer.reconcile()
// One decode across three passes, and the session list's cached fold for it
// is the same object it was after the first: nothing invalidated it.
expect(openCodeParseCalls).toHaveLength(1)
expect(getSessionParseCacheEntry(syntheticPath)).toBe(openCodeAfterFirst)
// The control, which the index really does hold, is untouched either way.
expect(getSessionParseCacheEntry(claudePath)).toBe(claudeAfterFirst)
// What makes it skippable: a row saying the index has seen this source and
// holds no session for it, which is the shape a read-through-with-no-session
// already leaves.
const rows = harness.read((db) =>
db.prepare('SELECT path, state, session_row_id FROM files ORDER BY path').all()
) as { path: string; state: string; session_row_id: number | null }[]
expect(rows).toHaveLength(2)
expect(rows.find((row) => row.path === syntheticPath)).toMatchObject({
state: 'current',
session_row_id: null
})
expect(indexer.status()).toMatchObject({ filesDue: 0, filesFailed: 0, phase: 'current' })
})
src/main/ai-vault-search/session-search-orphan-rows.test.ts
-186
View File
@@ -1,186 +0,0 @@
import { afterEach, describe, expect, it } from 'vitest'
import type SyncDatabase from '../sqlite/sync-database'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
import { identifierShadowText } from './session-search-identifier-split'
import { readIndexGeneration } from './session-search-index-generation'
import { planSessionSearchQuery } from './session-search-query-planner'
import { sessionSearchSnippet } from './session-search-snippet'
import type { SessionSearchCursorError } from './session-search-page-cursor'
import { SessionSearchTypoRepair } from './session-search-typo-repair'
// Retention deletes a session row in one small transaction and reclaims its
// message rows in batches afterwards, so a `messages` row with no `sessions` row
// is a state every purge, every removed source and every interrupted drain
// passes through. Those rows are still in both FTS tables and still in the
// vocabulary, and nothing here may return one.
//
// A hit is a session row, and the ranked list is loaded `FROM sessions`, so the
// route ladder below cannot surface an orphan even if a join were loosened —
// those cases are a ratchet over the shape, not the proof. The two reads that
// can leak one are pinned separately and each is a real oracle: the snippet,
// which is handed a rowid and asked for its text, and the typo repair, whose
// dictionary is the FTS b-tree and lists an orphan's terms like any other.
const ORPHAN_SESSION_ROW = 99
const ORPHAN_TEXT = 'orphaned marmoset secret'
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
/** Two rows in the FTS table and the vocabulary, and no session row for them. */
function plantOrphans(db: SyncDatabase, text: string = ORPHAN_TEXT): number[] {
const rowids: number[] = []
for (let n = 0; n < 2; n++) {
const rowid = Number(
db
.prepare("INSERT INTO messages(session_row_id,role,ts) VALUES (?,'user',?)")
.run(ORPHAN_SESSION_ROW, '2026-09-10T00:00:00.000Z').lastInsertRowid
)
db.prepare(
'INSERT INTO messages_fts(rowid,user_text,assistant_text,tool_text,identifiers) VALUES (?,?,?,?,?)'
).run(rowid, text, '', '', identifierShadowText(text))
rowids.push(rowid)
}
return rowids
}
async function withOrphans(): Promise<{ harness: SessionSearchHarness; rowids: number[] }> {
harness = await openSessionSearchHarness('ss-orphan-rows')
addSyntheticSession(harness.db, { id: 1, text: 'the haystack line here' })
const rowids = plantOrphans(harness.db)
// The oracle only means anything if the rows are really there to be found.
expect(
harness.db
.prepare("SELECT count(*) AS c FROM messages_fts WHERE messages_fts MATCH 'marmoset'")
.get()
).toEqual({ c: 2 })
expect(
harness.db.prepare("SELECT doc FROM messages_vocab WHERE term = 'marmoset'").get()
).toEqual({ doc: 2 })
return { harness, rowids }
}
it.each([
['phrase', '"orphaned marmoset"'],
['and', 'orphaned secret'],
['single-token literal', 'marmoset'],
['or', 'marmoset haystack orphaned'],
['typo repair', 'marmosett'],
['operator only', 'repo:app']
])('returns no orphaned row on the %s route', async (_route, query) => {
const { harness: open } = await withOrphans()
for (const scope of ['all', 'conversation'] as const) {
const hits = open.engine.search({ query, scope }).hits
expect(hits.map((hit) => hit.sessionId)).not.toContain(String(ORPHAN_SESSION_ROW))
expect(hits.filter((hit) => hit.evidence?.snippet.includes('marmoset'))).toEqual([])
}
})
it('never repairs a term onto a spelling only orphaned rows carry', async () => {
const { harness: open } = await withOrphans()
// `marmoset` is in the vocabulary twice, which is what would make it the
// repair for `marmosett` if the repair trusted the vocabulary alone.
expect(new SessionSearchTypoRepair(open.db).correct('marmosett', 'all')).toBeNull()
expect(open.engine.search({ query: 'marmosett' }).planner.repairedTerms).toBeUndefined()
})
it('snippets nothing for an orphaned row, even asked for it by rowid', async () => {
const { harness: open, rowids } = await withOrphans()
const plan = planSessionSearchQuery('marmoset')
for (const scope of ['all', 'conversation'] as const) {
expect(sessionSearchSnippet(open.db, scope, rowids[0]!, plan)).toEqual({
text: '',
truncated: false
})
}
})
it('still answers for the live session beside them', async () => {
const { harness: open } = await withOrphans()
expect(open.engine.search({ query: 'haystack' }).hits.map((hit) => hit.sessionId)).toEqual(['1'])
})
// Reclaiming those rows is the other half. The drain deletes only from
// `messages`, so for a long time it was argued to change no answer and left
// outside the generation fence. Retrieval never saw them, but the typo repair's
// dictionary is `messages_vocab`, a view over the FTS b-tree that lists a term
// whether or not a reader can reach the rows carrying it — so the drain moved
// which word a query was repaired to, under a cursor that was still honoured.
describe('a purge reclaiming rows nothing can reach', () => {
/** A live session and a purged one that both carry `text`. */
async function withReclaimable(): Promise<SessionSearchHarness> {
harness = await openSessionSearchHarness('ss-orphan-drain')
// Two live rows, which is what makes `marmoset` eligible as a repair at all.
addSyntheticSession(harness.db, { id: 1, text: 'the marmoset lives here', rows: 2 })
plantOrphans(harness.db)
return harness
}
it('answers the same before and after, because the repair counts live rows', async () => {
const open = await withReclaimable()
const before = open.engine.search({ query: 'marmosett' })
expect(before.planner.repairedTerms).toEqual(['marmoset'])
expect(before.hits.map((hit) => hit.sessionId)).toEqual(['1'])
await open.store.purgeOlderThan(null)
expect(open.db.prepare('SELECT count(*) AS c FROM messages').get()).toEqual({ c: 2 })
const after = open.engine.search({ query: 'marmosett' })
expect(after.planner.repairedTerms).toEqual(before.planner.repairedTerms)
expect(after.hits.map((hit) => hit.sessionId)).toEqual(before.hits.map((hit) => hit.sessionId))
})
it('moves the generation anyway, so no cursor spans it', async () => {
// The repair counting live rows fixes the common case. It does not make the
// drain provably inert: `messages_vocab` still decides which candidates
// survive its scan limit, and reclaiming a term's last row changes where
// that limit cuts. The fence is what covers the rest, at the price of
// refusing a cursor once per batch while a purge runs.
const open = await withReclaimable()
// A second live session, so page one has a page two to be refused.
addSyntheticSession(open.db, { id: 2, text: 'the marmoset again', rows: 2 })
const page = open.engine.search({ query: 'marmoset', limit: 1 })
expect(page.page.cursor).not.toBeNull()
const before = readIndexGeneration(open.db)
await open.store.purgeOlderThan(null)
expect(readIndexGeneration(open.db)).toBeGreaterThan(before)
try {
open.engine.search({ query: 'marmoset', limit: 1, cursor: page.page.cursor! })
expect.unreachable('a cursor must not span a purge')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('stale-generation')
}
})
it('picks the same repair when an unreachable spelling was the more common one', async () => {
// Two candidates equally close to the query. `marmosetx` led on the old
// ranking only because two of its rows belonged to a session retention had
// already cut loose, so the drain swapped the repair under a live cursor.
harness = await openSessionSearchHarness('ss-orphan-drain-tie')
const db = harness.db
for (let id = 1; id <= 4; id++) {
addSyntheticSession(db, { id, text: `marmosetx session${id}` })
}
for (let id = 5; id <= 9; id++) {
addSyntheticSession(db, { id, text: `marmosetq session${id}` })
}
plantOrphans(db, 'marmosetx')
const before = harness.engine.search({ query: 'marmosett' })
expect(before.planner.repairedTerms).toEqual(['marmosetq'])
await harness.store.purgeOlderThan(null)
expect(harness.engine.search({ query: 'marmosett' }).planner.repairedTerms).toEqual(
before.planner.repairedTerms
)
})
})
src/main/ai-vault-search/session-search-page-cursor.ts
-108
View File
@@ -1,108 +0,0 @@
import { createHash } from 'node:crypto'
import type { SessionSearchRequest } from './session-search-engine-types'
export type SessionSearchCursorRejection = 'stale-generation' | 'different-query' | 'malformed'
/**
* A cursor the engine refuses to honour. Typed, and thrown rather than
* swallowed: silently restarting at page one hands the caller a page it has
* already shown as if it were the next one, and silently re-running against a
* newer index hands it a slice of a list it never saw.
*/
export class SessionSearchCursorError extends Error {
constructor(
readonly rejection: SessionSearchCursorRejection,
/**
* The generation the index is at now. Always present: the engine knows it
* before it looks at the cursor at all.
*/
readonly actualGeneration: number,
/**
* The generation the cursor claims it was minted in. Absent only when the
* cursor could not be decoded far enough to carry a number, which is one of
* the `malformed` cases.
*/
readonly expectedGeneration?: number
) {
super(`Search cursor rejected: ${rejection}`)
this.name = 'SessionSearchCursorError'
}
}
type CursorPayload = {
/** Index generation. */
g: number
/**
* Offset into the ranked list, not a session id. Ids are not in a cursor at
* all, so nothing here depends on `sessions.id` being unique over time —
* though it is, because PR 2 made the column AUTOINCREMENT so a purged
* session's id is never reissued to a live one.
*/
o: number
/** Query identity; see `sessionSearchPageKey`. */
k: string
}
/**
* Everything a page's ranking depends on except the limit. Two requests with
* the same key produce the same ranked list within one generation, so a cursor
* minted by one is meaningful to the other; the limit is left out on purpose so
* a caller may change its page size mid-pagination.
*/
export function sessionSearchPageKey(request: SessionSearchRequest): string {
const filters = request.filters ?? {}
const identity = JSON.stringify([
request.query,
request.scope ?? 'all',
filters.sort ?? 'relevance',
filters.since ?? null,
[...(filters.agents ?? [])].sort(),
[...(filters.scopePaths ?? [])].sort()
])
return createHash('sha256').update(identity).digest('base64url').slice(0, 16)
}
export function encodeSessionSearchCursor(generation: number, offset: number, key: string): string {
const payload: CursorPayload = { g: generation, o: offset, k: key }
return Buffer.from(JSON.stringify(payload), 'utf-8').toString('base64url')
}
/**
* The offset this cursor points at, or a typed rejection.
*
* Every rejection carries `actualGeneration`, and every one that could read a
* generation out of the cursor carries `expectedGeneration` too, so a caller
* can tell "the index moved under you, ask for page one" from "this cursor is
* not ours" and act on the first without showing anyone an error.
*/
export function decodeSessionSearchCursor(cursor: string, generation: number, key: string): number {
let payload: CursorPayload
try {
payload = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf-8')) as CursorPayload
} catch {
throw new SessionSearchCursorError('malformed', generation)
}
// A generation that survived parsing is worth reporting even when the rest of
// the payload is unusable: it is what tells the caller which snapshot the
// cursor thought it was walking.
const claimed =
typeof payload?.g === 'number' && Number.isFinite(payload.g) ? payload.g : undefined
if (
claimed === undefined ||
!Number.isInteger(payload?.o) ||
payload.o < 0 ||
typeof payload?.k !== 'string'
) {
throw new SessionSearchCursorError('malformed', generation, claimed)
}
// Generation first: a caller who changed the query AND waited through a
// publish should hear about the index moving, which is the condition it
// cannot fix by paging again.
if (claimed !== generation) {
throw new SessionSearchCursorError('stale-generation', generation, claimed)
}
if (payload.k !== key) {
throw new SessionSearchCursorError('different-query', generation, claimed)
}
return payload.o
}
src/main/ai-vault-search/session-search-paging.test.ts
-309
View File
@@ -1,309 +0,0 @@
import { afterEach, describe, expect, it } from 'vitest'
import type { SessionSearchRequest } from './session-search-engine-types'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
import { readIndexGeneration } from './session-search-index-generation'
import {
decodeSessionSearchCursor,
encodeSessionSearchCursor,
SessionSearchCursorError,
sessionSearchPageKey
} from './session-search-page-cursor'
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
async function open(name: string, options = {}): Promise<SessionSearchHarness> {
harness = await openSessionSearchHarness(name, options)
return harness
}
async function withSessions(count: number, options = {}): Promise<SessionSearchHarness> {
harness = await openSessionSearchHarness('ss-engine-paging', options)
for (let id = 1; id <= count; id++) {
addSyntheticSession(harness.db, {
id,
text: `needle padding ${'word '.repeat(id % 5)}`,
updatedAt: `2026-09-${String(id).padStart(2, '0')}T00:00:00.000Z`
})
}
return harness
}
describe('a cursor walks one ranked list', () => {
it('pages through every session exactly once, in one stable order', async () => {
const { engine } = await withSessions(25)
const request: SessionSearchRequest = { query: 'needle', limit: 10 }
const seen: string[] = []
let cursor: string | null = null
let pages = 0
do {
const page = engine.search(cursor ? { ...request, cursor } : request)
seen.push(...page.hits.map((hit) => hit.sessionId))
cursor = page.page.cursor
pages++
expect(pages).toBeLessThan(10)
} while (cursor !== null)
expect(pages).toBe(3)
expect(seen).toHaveLength(25)
expect(new Set(seen).size).toBe(25)
// The same walk, run again against the same generation, is the same walk.
expect(engine.search(request).hits.map((hit) => hit.sessionId)).toEqual(seen.slice(0, 10))
})
it('closes the page when the last hit has been handed out', async () => {
const { engine } = await withSessions(3)
const page = engine.search({ query: 'needle', limit: 10 })
expect(page.hits).toHaveLength(3)
expect(page.page.hasMore).toBe(false)
expect(page.page.cursor).toBeNull()
})
it('lets a caller change page size mid-walk', async () => {
const { engine } = await withSessions(12)
const first = engine.search({ query: 'needle', limit: 5 })
const rest = engine.search({ query: 'needle', limit: 20, cursor: first.page.cursor! })
expect(rest.hits).toHaveLength(7)
expect(rest.page.hasMore).toBe(false)
})
it('breaks a tie by session, so two entries cannot swap between pages', async () => {
// Same text, same timestamp: every ranking key is equal, which is exactly
// where an unstable sort would hand one session out twice and lose another.
harness = await openSessionSearchHarness('ss-engine-ties')
for (let id = 1; id <= 6; id++) {
addSyntheticSession(harness.db, { id, text: 'needle', updatedAt: '2026-09-01T00:00:00.000Z' })
}
const first = harness.engine.search({ query: 'needle', limit: 3 })
const second = harness.engine.search({ query: 'needle', limit: 3, cursor: first.page.cursor! })
const seen = [...first.hits, ...second.hits].map((hit) => hit.sessionId)
expect(seen).toEqual(['1', '2', '3', '4', '5', '6'])
})
})
describe('a cursor is refused rather than reinterpreted', () => {
it('rejects a cursor minted before the index moved', async () => {
const { engine, store } = await withSessions(25)
const first = engine.search({ query: 'needle', limit: 10 })
// A proven deletion of a path this index really held hides a session, which
// is exactly the change a cursor must not be allowed to page across.
store.removeFile('/synthetic/1.jsonl')
expect(() => engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! })).toThrow(
SessionSearchCursorError
)
try {
engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! })
expect.unreachable('a stale cursor must not be silently re-run')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('stale-generation')
}
})
it('names both generations, so a caller can tell a moved index from a bad cursor', async () => {
// What a caller does about it differs: a moved index means quietly ask for
// page one again, a bad cursor means something is wrong with the caller.
const { engine, store } = await withSessions(25)
const first = engine.search({ query: 'needle', limit: 10 })
const minted = readIndexGeneration(harness!.db)
// Any published read moves the generation, including one for a file this
// page never mentioned. That is the fence working, not a defect.
store.removeFile('/synthetic/9.jsonl')
try {
engine.search({ query: 'needle', limit: 10, cursor: first.page.cursor! })
expect.unreachable('the index moved')
} catch (error) {
const rejected = error as SessionSearchCursorError
expect(rejected.rejection).toBe('stale-generation')
expect(rejected.expectedGeneration).toBe(minted)
expect(rejected.actualGeneration).toBe(readIndexGeneration(harness!.db))
expect(rejected.actualGeneration).toBeGreaterThan(rejected.expectedGeneration!)
}
})
it('rejects a cursor carried over to a different query', async () => {
const { engine } = await withSessions(25)
const first = engine.search({ query: 'needle', limit: 10 })
try {
engine.search({ query: 'padding', limit: 10, cursor: first.page.cursor! })
expect.unreachable('a cursor indexes into one ranked list, not any list')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('different-query')
}
})
it('rejects a cursor whose filters changed, which reranks the list', async () => {
const { engine } = await withSessions(25)
const first = engine.search({ query: 'needle', limit: 10 })
try {
engine.search({
query: 'needle',
limit: 10,
cursor: first.page.cursor!,
filters: { sort: 'newest' }
})
expect.unreachable('a different sort is a different ranked list')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('different-query')
}
})
// Every field the ranked list depends on has to be in the key, and a field
// that is in the key but never pinned is a field a refactor can drop while
// the suite stays green. One case each, through the engine, so the assertion
// is about a refused page and not about a hash.
it.each([
['scope', { scope: 'conversation' as const }],
['sort', { filters: { sort: 'newest' as const } }],
['agents', { filters: { agents: ['codex' as const] } }],
['scopePaths', { filters: { scopePaths: ['/repo/app'] } }],
['since', { filters: { since: '2026-09-01T00:00:00.000Z' } }]
])('rejects a cursor presented with a different %s', async (_field, changed) => {
const { engine } = await withSessions(25)
const request: SessionSearchRequest = {
query: 'needle',
limit: 10,
scope: 'all',
filters: { sort: 'relevance', agents: ['claude'], scopePaths: ['/'], since: undefined }
}
const first = engine.search(request)
expect(first.page.cursor).not.toBeNull()
try {
engine.search({
...request,
...changed,
filters: { ...request.filters, ...('filters' in changed ? changed.filters : {}) },
cursor: first.page.cursor!
})
expect.unreachable('a narrowing the ranked list depends on must invalidate the cursor')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('different-query')
}
})
it('rejects a cursor that is not one of ours', async () => {
const { engine } = await withSessions(3)
try {
engine.search({ query: 'needle', cursor: 'not-a-cursor' })
expect.unreachable('a malformed cursor is not an empty one')
} catch (error) {
expect((error as SessionSearchCursorError).rejection).toBe('malformed')
}
})
})
describe('cursor encoding', () => {
const request: SessionSearchRequest = { query: 'needle', filters: { scopePaths: ['/a'] } }
it('round-trips an offset within its own generation and query', () => {
const key = sessionSearchPageKey(request)
expect(decodeSessionSearchCursor(encodeSessionSearchCursor(7, 40, key), 7, key)).toBe(40)
})
it('keys a request by what changes its ranking, and not by its page size', () => {
expect(sessionSearchPageKey({ ...request, limit: 5 })).toBe(
sessionSearchPageKey({ ...request, limit: 50 })
)
expect(sessionSearchPageKey({ ...request, scope: 'conversation' })).not.toBe(
sessionSearchPageKey(request)
)
})
it('reads a filter list in any order as the same request', () => {
expect(sessionSearchPageKey({ query: 'a', filters: { agents: ['claude', 'codex'] } })).toBe(
sessionSearchPageKey({ query: 'a', filters: { agents: ['codex', 'claude'] } })
)
})
it.each([
['a negative offset', encodeSessionSearchCursor(1, -1, 'k'), 1],
['a non-integer offset', Buffer.from('{"g":1,"o":1.5,"k":"k"}').toString('base64url'), 1],
['a payload that is not an object', Buffer.from('"nope"').toString('base64url'), undefined],
['text that is not base64url JSON', 'zzz!!', undefined]
])('rejects %s as malformed, still naming the index generation', (_name, cursor, claimed) => {
// The caller has to know which snapshot it was refused against whatever was
// wrong with the cursor, and the generation it claimed whenever that
// survived parsing.
try {
decodeSessionSearchCursor(cursor, 7, 'k')
expect.unreachable('a malformed cursor is not an empty one')
} catch (error) {
const rejected = error as SessionSearchCursorError
expect(rejected.rejection).toBe('malformed')
expect(rejected.actualGeneration).toBe(7)
expect(rejected.expectedGeneration).toBe(claimed)
}
})
})
describe('the candidate limit is a tunable default, and says when it cut', () => {
it('does not claim truncation when every session fits', async () => {
const { engine } = await withSessions(5, { sessionCandidateLimit: 600 })
expect(engine.search({ query: 'needle' }).truncated.candidates).toBe(false)
})
it('claims truncation, and ranks only what it retrieved, at the limit', async () => {
const { engine } = await withSessions(10, { sessionCandidateLimit: 4 })
const result = engine.search({ query: 'needle', limit: 100 })
expect(result.truncated.candidates).toBe(true)
expect(result.hits).toHaveLength(4)
})
it('applies the same limit to an operator-only page', async () => {
const { engine } = await withSessions(10, { sessionCandidateLimit: 4 })
const result = engine.search({ query: 'repo:app', limit: 100 })
expect(result.truncated.candidates).toBe(true)
expect(result.hits).toHaveLength(4)
})
it('says it gave up when the operator walk stopped scanning, not that it is done', async () => {
// The shape that reads as a confident empty answer: the only match sits
// past the walk's ceiling, so the walk stops having found nothing. Zero
// hits and `truncated.candidates` false would tell a caller there is
// nothing to find, which is a different claim from "I stopped looking".
// The walk reads a page at a time and gives up past a ceiling of
// `candidateLimit` x 20, so the corpus has to be deeper than one page for
// the ceiling to be what ends it. The only match is the oldest session.
const deep = 600
const { db, engine } = await open('ss-engine-sparse-deep', { sessionCandidateLimit: 2 })
for (let id = 1; id <= deep; id++) {
addSyntheticSession(db, {
id,
cwd: id === deep ? '/repo/needleonly' : '/repo/app',
updatedAt: new Date(Date.UTC(2026, 8, 9) - id * 60_000).toISOString()
})
}
const result = engine.search({ query: 'repo:needleonly' })
expect(result.hits).toHaveLength(0)
expect(result.truncated.candidates).toBe(true)
})
it('does not claim it gave up when the walk really did read everything', async () => {
const { db, engine } = await open('ss-engine-sparse-shallow', { sessionCandidateLimit: 600 })
addSyntheticSession(db, { id: 1, cwd: '/repo/app' })
const result = engine.search({ query: 'repo:nothing-here' })
expect(result.hits).toHaveLength(0)
expect(result.truncated.candidates).toBe(false)
})
})
describe('the response carries the snapshot it was built from', () => {
it('reports the index generation on every result', async () => {
const { db, engine, store } = await withSessions(3)
const before = engine.search({ query: 'needle' }).generation
expect(before).toBe(readIndexGeneration(db))
store.removeFile('/synthetic/1.jsonl')
const after = engine.search({ query: 'needle' }).generation
expect(after).toBe(readIndexGeneration(db))
expect(after).toBeGreaterThan(before)
})
})
src/main/ai-vault-search/session-search-pass.ts
+216
View File
@@ -0,0 +1,216 @@
import type { AiVaultScanIssue } from '../../shared/ai-vault-types'
import { ensureSessionParseCacheLoaded } from '../ai-vault/session-parse-cache-persistence'
import {
cursorChatMetaRefusals,
withCursorChatMetaScan
} from '../ai-vault/session-scanner-cursor-chat-meta'
import { recordSessionScanIssue } from '../ai-vault/session-scan-issues'
import {
mergeDegradedRoots,
scanIssueDegradedRoots,
unreadableRoots,
type SessionSearchDegradedRoot
} from './session-search-degraded-roots'
import { retireDeletedSessionSearchSources } from './session-search-deleted-sources'
import type { SessionSearchDirectoryReader } from './session-search-directory-listings'
import { runSessionSearchIndexPass } from './session-search-index-pass'
import {
discoverSessionSearchCandidates,
isUnderScanRoot,
sessionSearchEmptiedRoots,
sessionSearchRootListings,
type SessionSearchScanRoots
} from './session-search-scan-roots'
import type { SessionSearchFileRow, SessionSearchStore } from './session-search-store'
import { sessionSearchEnumeratedContainers } from './session-search-synthetic-sources'
/**
* Rows a cycle proves present or gone, newest first.
*
* Why bounded and why newest first: a cycle lists the newest N per agent, so
* every older row it holds is undiscovered and would otherwise be walked every
* twenty seconds. Newest first is what makes the guarantee hold — a transcript
* recent enough for the window to cover is recent enough to be in this slice,
* so its deletion is proven on the very next cycle whenever it happened.
*/
const RETIREMENT_ROWS_PER_CYCLE = 512
/**
* Directories either pass may read proving deletions.
*
* The bound on the walk is readdirs, not rows: rows sharing a directory are one
* read and then map lookups, and a directory that answers an error answers it
* once for every row under it. Counting rows instead let one unreadable
* directory hold the whole walk for as long as it stayed unreadable.
*/
const RETIREMENT_DIRECTORIES_PER_PASS = 512
export type SessionSearchPassArgs = {
store: SessionSearchStore
roots: SessionSearchScanRoots
/** A sweep lists every root; a cycle lists the newest N per agent. */
full: boolean
recentPerAgent: number
/** Real roots that listed transcripts on the previous pass; undefined before the first. */
previousRootsWithFiles?: ReadonlySet<string>
/** True once the pass is out of wall time; reads stop, everything else finishes. */
overdue?: () => boolean
/** One readdir per directory for the whole pass, shared by every step. */
listings: SessionSearchDirectoryReader
signal?: AbortSignal
}
export type SessionSearchPassResult = {
/** Real roots this pass listed transcripts under, for the next pass to compare against. */
rootsWithFiles: Set<string>
degradedRoots: SessionSearchDegradedRoot[]
/** False when the pass was cut short; its conclusions are not to be recorded. */
completed: boolean
/**
* True when the deadline stopped the reads with candidates still owed.
*
* The caller's one use for it: a cycle lists the newest N per agent, so a
* backlog outside that window is only *visible* to a sweep. Without this a
* first run would index the recency window in its opening pass and then crawl,
* making progress only on the periodic sweep every five minutes.
*/
outOfTime: boolean
}
/**
* One pass. Four steps, the same four whether it sweeps or cycles.
*
* 1. **Discover.** The only filesystem walk: every root on a sweep, the newest
* N per agent on a cycle. Everything below is decided from what it returns.
* 2. **Decide and read.** Per candidate, its stat against its row. Reads stop
* at the deadline and nothing is recorded about what was left, because being
* owed is a fact about the row and not an entry in a queue.
* 3. **Retire.** Candidates are the rows this pass's discovery did not return,
* inside the scope that discovery covered. The stateless walk proves each
* one gone, present or unverifiable; only `gone` deletes.
* 4. **Report.** Root health for this pass. The counts are a query, made by the
* caller against the same rows, so nothing here is tallied.
*
* The pass keeps nothing. Everything it learns is either on a row or in the
* result the caller compares against the next pass.
*/
export async function runSessionSearchPass(
args: SessionSearchPassArgs
): Promise<SessionSearchPassResult> {
const { store, signal } = args
if (args.full) {
// Every sweep opens with the purge, so a window narrower than the last
// instance held is applied by the first sweep of this one.
await store.purgeOlderThan(store.retentionCutoff, signal)
}
await ensureSessionParseCacheLoaded()
return withCursorChatMetaScan(async () => {
const swept = await discoverSessionSearchCandidates(args.roots, {
limitPerAgent: args.full ? Number.POSITIVE_INFINITY : args.recentPerAgent,
signal
})
const issues: AiVaultScanIssue[] = [...swept.issues]
let completed = true
let outOfTime = false
const rows = new Map(store.files().map((row) => [row.path, row]))
try {
const read = await runSessionSearchIndexPass(store, swept.candidates, {
signal,
rows,
overdue: args.overdue
})
outOfTime = read.outOfTime
} catch (error) {
if (!signal?.aborted) {
throw error
}
completed = false
}
const listings = sessionSearchRootListings(args.roots, swept.discoveries)
const roots = listings.map((listing) => listing.root)
const rootsWithFiles = new Set(
listings.filter((listing) => listing.files > 0).map((listing) => listing.root)
)
// Undefined, not empty, before any pass has recorded one: an empty set is a
// real observation and this is the absence of one.
const previousRootsWithFiles = args.previousRootsWithFiles
// A pass cut short saw part of the machine, so its silence about a path is
// not evidence; it retires nothing and publishes no verdicts.
const retirement = completed
? await retireDeletedSessionSearchSources({
store,
paths: retirementCandidates(rows, swept, roots, args.full),
roots,
// Only a sweep enumerates without a per-agent limit, so only a sweep
// may prove a synthetic row's container holds it no longer.
enumeratedContainers: args.full
? sessionSearchEnumeratedContainers(swept.candidates, issues)
: undefined,
emptiedRoots: previousRootsWithFiles
? sessionSearchEmptiedRoots(previousRootsWithFiles, rootsWithFiles)
: new Set(),
listings: args.listings,
directoryLimit: RETIREMENT_DIRECTORIES_PER_PASS,
signal
})
: { retired: [], unverifiable: [], unchecked: [], degradedRoots: [] }
for (const refusal of cursorChatMetaRefusals()) {
// One issue per refused chats root, not one per Cursor transcript.
recordSessionScanIssue(issues, {
agent: 'cursor',
path: refusal.chatsRoot,
message: refusal.message
})
}
// Roots that listed no transcripts and cannot be listed either: the walker
// swallows a readdir failure, so this is the only place it surfaces.
const unlistable = completed
? await unreadableRoots(
roots.filter((root) => !rootsWithFiles.has(root)),
args.listings,
signal
)
: []
return {
rootsWithFiles,
degradedRoots: mergeDegradedRoots(
scanIssueDegradedRoots(roots, issues),
retirement.degradedRoots,
unlistable
),
completed,
outOfTime
}
})
}
/**
* Rows this pass's discovery did not return, inside the scope it covered.
*
* A sweep covers everything, so every undiscovered row is a candidate. A cycle
* covers the newest N per agent, so it may only judge rows under a root it
* actually listed, and it takes the newest of those: an older row is not
* evidence of anything a cycle looked for, and the next sweep is what reaches
* it. This is the whole of what used to be a watch set carried between passes.
*/
function retirementCandidates(
rows: ReadonlyMap<string, SessionSearchFileRow>,
swept: { candidates: readonly { file: { path: string } }[] },
roots: readonly string[],
full: boolean
): string[] {
const discovered = new Set(swept.candidates.map((candidate) => candidate.file.path))
const undiscovered = [...rows.values()].filter((row) => !discovered.has(row.path))
if (full) {
return undiscovered.map((row) => row.path)
}
return undiscovered
.filter((row) => roots.some((root) => isUnderScanRoot(row.path, root)))
.sort((left, right) => right.mtimeMs - left.mtimeMs)
.slice(0, RETIREMENT_ROWS_PER_CYCLE)
.map((row) => row.path)
}
src/main/ai-vault-search/session-search-query-log.test.ts
-61
View File
@@ -1,61 +0,0 @@
import { afterEach, expect, it } from 'vitest'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
import { logSessionSearchQuery } from './session-search-query-log'
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
async function open(options = {}): Promise<SessionSearchHarness> {
harness = await openSessionSearchHarness('ss-query-log', options)
addSyntheticSession(harness.db, { id: 1, text: 'needle' })
return harness
}
function loggedQueries(harness: SessionSearchHarness): string[] {
return (
harness.db.prepare('SELECT query FROM search_log ORDER BY id').all() as { query: string }[]
).map((row) => row.query)
}
it('writes nothing on the query path unless the caller asked for a log', async () => {
const opened = await open()
opened.engine.search({ query: 'needle' })
expect(loggedQueries(opened)).toEqual([])
})
it('records the query and its route when logging is on', async () => {
const opened = await open({ logQueries: true })
opened.engine.search({ query: 'needle' })
const rows = opened.db.prepare('SELECT query, route, hits FROM search_log').all() as {
query: string
route: string
hits: number
}[]
expect(rows).toEqual([{ query: 'needle', route: 'or', hits: 1 }])
})
it('stores the query as typed, the way the index stores content as written', async () => {
// PR 2 decided the index does not redact: it is a second copy of plaintext
// the user already holds under their own home directory. The same holds for
// what they typed into the search box.
const opened = await open({ logQueries: true })
opened.engine.search({ query: 'Bearer abcdefghijklmnopqrstuvwxyz012345' })
expect(loggedQueries(opened)[0]).toBe('Bearer abcdefghijklmnopqrstuvwxyz012345')
})
it('keeps the newest N and drops the rest, so the log cannot grow with use', async () => {
const opened = await open()
// The real ceiling is 5,000; the trim is the same statement at any size.
for (let n = 0; n < 12; n++) {
logSessionSearchQuery(opened.db, { query: `q${n}`, route: 'or', hits: 0, durationMs: 1 }, 5)
}
expect(loggedQueries(opened)).toEqual(['q7', 'q8', 'q9', 'q10', 'q11'])
})
src/main/ai-vault-search/session-search-query-log.ts
-30
View File
@@ -1,30 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
export const SEARCH_LOG_LIMIT = 5000
/**
* Local-only telemetry the eval set is rebuilt from.
*
* The query is stored as typed, for the reason PR 2 gives for not redacting
* transcript content: this file sits beside an index that already holds the
* user's own plaintext, so a second copy of what they typed is not a new
* exposure. What may leave the machine is a transport policy and belongs where
* the wire is.
*
* Nothing enables this by default: the engine writes a row only when its caller
* asked for it, because a log write on the query path is a write on what is
* otherwise a read-only lane. Who turns it on is PR 3b's settings decision.
*/
export function logSessionSearchQuery(
db: SyncDatabase,
entry: { query: string; route: string; hits: number; durationMs: number },
limit: number = SEARCH_LOG_LIMIT
): void {
db.prepare(
'INSERT INTO search_log(ts, query, route, hits, duration_ms) VALUES (?, ?, ?, ?, ?)'
).run(new Date().toISOString(), entry.query, entry.route, entry.hits, entry.durationMs)
db.prepare(
`DELETE FROM search_log WHERE id <= (
SELECT id FROM search_log ORDER BY id DESC LIMIT 1 OFFSET ?)`
).run(limit)
}
src/main/ai-vault-search/session-search-query-planner.test.ts
-84
View File
@@ -1,84 +0,0 @@
import { describe, expect, it } from 'vitest'
import {
andExpression,
isLiteralQuery,
orExpression,
phraseExpression,
planSessionSearchQuery,
quoteFtsTerm
} from './session-search-query-planner'
describe('literal shape decides whether the phrase route is even tried', () => {
it.each([
'resolveTerminalPath',
'src/main/foo-bar.ts',
'MAX_RETRY_COUNT',
'kern.tty.ptmx_max',
'#19687',
'STA-4850',
'"exact words here"',
'TypeError: undefined',
'foo() {'
])('treats %s as quoting something from a transcript', (query) => {
expect(isLiteralQuery(query)).toBe(true)
})
it.each(['why is the terminal slow', 'how do I resume a session', 'relay capacity'])(
'treats %s as prose',
(query) => {
expect(isLiteralQuery(query)).toBe(false)
}
)
})
describe('the body is what the phrase and AND routes see', () => {
it('drops stop words from prose so the AND route is not defeated by "the"', () => {
expect(planSessionSearchQuery('why is the relay dropping frames').body).toEqual([
'relay',
'dropping',
'frames'
])
})
it('keeps stop words inside a literal, where they are part of what was quoted', () => {
// The literal shape is `foo.ts`; dropping `the` would change what was typed.
expect(planSessionSearchQuery('the foo.ts file').body).toEqual(['the', 'foo.ts', 'file'])
})
it('keeps a query that is nothing but stop words rather than answering nothing', () => {
expect(planSessionSearchQuery('how do I').body).toEqual(['how', 'do', 'I'])
})
it('has no terms for a query with no searchable token', () => {
expect(planSessionSearchQuery(' ... ').terms).toEqual([])
})
})
describe('the OR fallback fans an identifier out into its pieces', () => {
it('adds the split pieces after the whole term, never in place of it', () => {
const plan = planSessionSearchQuery('resolveTerminalPath')
expect(plan.terms[0]).toBe('resolveTerminalPath')
expect(plan.terms).toContain('terminal')
expect(plan.terms).toContain('path')
// `resolve` is not a stop word, so the whole identifier is reachable by piece.
expect(plan.terms).toContain('resolve')
})
it('leaves an ordinary word alone', () => {
expect(planSessionSearchQuery('relay').terms).toEqual(['relay'])
})
})
describe('FTS5 expressions quote every term', () => {
it('quotes punctuation that would otherwise be syntax', () => {
expect(quoteFtsTerm('cli.mjs')).toBe('"cli.mjs"')
expect(quoteFtsTerm('C++')).toBe('"C++"')
expect(quoteFtsTerm('say "hi"')).toBe('"say ""hi"""')
})
it('builds one phrase, an AND chain, and an OR chain from the same terms', () => {
expect(phraseExpression(['alpha', 'beta'])).toBe('"alpha beta"')
expect(andExpression(['alpha', 'beta'])).toBe('"alpha" AND "beta"')
expect(orExpression(['alpha', 'beta'])).toBe('"alpha" OR "beta"')
})
})
src/main/ai-vault-search/session-search-query-planner.ts
-140
View File
@@ -1,140 +0,0 @@
import type { SessionSearchScope } from './session-search-engine-types'
import { identifierShadowTerms } from './session-search-identifier-split'
// Tokens exactly as the unicode61 tokenizer with `_ . - / +` tokenchars emits them.
const INDEX_TOKEN = /[\p{L}\p{N}\p{M}\p{Co}_./+-]+/gu
const STOP_WORDS = new Set(
(
'a an and are as at be but by for from how i if in into is it its of on or that the this to ' +
'was were what when where which who why with you your we my me do does did not no can could ' +
'should would about our us they them there their has have had been being so such then than ' +
"these those there's im ive dont"
).split(' ')
)
const MAX_BODY_TERMS = 48
const MAX_TERMS = 64
// A query that quotes something from a transcript: camelCase, SCREAMING_SNAKE,
// a dotted or snake_case name, a path, a filename, a PR number, a ticket, code
// punctuation, or an error word.
const LITERAL_SHAPE =
/[A-Za-z0-9_]*[a-z][A-Z][A-Za-z0-9_]*|\b[A-Z][A-Z0-9]{2,}(_[A-Z0-9]+)+\b|\b\w{2,}[._]\w{2,}\b|\b[\w.-]+\/[\w/.-]+\b|\b\w+\.(ts|tsx|js|jsx|py|rs|go|json|md|sh|yml|yaml|toml|c|cc|h|java|sql)\b|#\d{3,}|\b[A-Z]{2,6}-\d{2,}\b|[(){};=]|::|->|--\w|\b(Error|Exception|Traceback|error:|warning:)\b/
const QUOTED = /"[^"]{3,}"|'[^']{3,}'/
export type SessionSearchQueryPlan = {
literal: boolean
/**
* The query had more terms than the planner will search. What is dropped is
* the tail, so a match that only the last term would have found is missed;
* the caller is told rather than handed a confident empty answer.
*/
truncated: boolean
/** Deduplicated index-faithful terms for the OR fallback, incl. identifier pieces. */
terms: string[]
/** Query-order tokens minus stop words: the phrase / AND candidate. */
body: string[]
}
export function isLiteralQuery(query: string): boolean {
return QUOTED.test(query) || LITERAL_SHAPE.test(query)
}
/**
* The tokenizer contract, unfolded: the same boundaries FTS5 draws for
* `unicode61 tokenchars '_.-/+'`. Pinned against real `fts5vocab` output in
* session-search-fts5-contract.test.ts, which is what makes it safe to plan a
* query without asking SQLite.
*/
export function indexTokens(query: string, limit = Number.POSITIVE_INFINITY): string[] {
const out: string[] = []
for (const match of query.matchAll(INDEX_TOKEN)) {
const token = match[0]
// Separators alone (`--`, `...`) are a token to FTS5 but never a search term.
if (/[\p{L}\p{N}\p{Co}]/u.test(token)) {
out.push(token)
if (out.length >= limit) {
break
}
}
}
return out
}
/**
* `literal` overrides the shape test. Typo repair re-plans the query it
* corrected, and a corrected spelling can look like ordinary prose even though
* what was typed was a literal: `parseJsonn(the, data)` has the punctuation that
* makes it literal, `parsejson the data` does not. Without the override the
* re-plan would drop `the` as a stop word, so the repaired query would search
* for less than the original asked for and `repairedTerms` would report a body
* the user never typed.
*/
export function planSessionSearchQuery(
query: string,
literal = isLiteralQuery(query)
): SessionSearchQueryPlan {
// One past the cap, so the plan can tell a query that just fits from one that
// was cut. `indexTokens` stops at its limit, so it cannot be asked afterwards.
const overCap = indexTokens(query, MAX_BODY_TERMS + 1)
const truncated = overCap.length > MAX_BODY_TERMS
const raw = overCap.slice(0, MAX_BODY_TERMS)
let body = literal ? raw : raw.filter((token) => !STOP_WORDS.has(token.toLowerCase()))
if (body.length < 2) {
body = raw
}
const terms = [...new Set(body)]
const extra: string[] = []
for (const term of terms) {
for (const piece of identifierShadowTerms(term, 12)) {
if (!terms.includes(piece) && !STOP_WORDS.has(piece) && !extra.includes(piece)) {
extra.push(piece)
}
}
}
return {
literal,
truncated,
terms: [...terms, ...extra].slice(0, MAX_TERMS),
body: body.slice(0, MAX_BODY_TERMS)
}
}
// Why: `cli.mjs`, `foo-bar`, and `C++` are all FTS5 syntax errors unquoted.
export function quoteFtsTerm(term: string): string {
return `"${term.replaceAll('"', '""')}"`
}
export function phraseExpression(terms: readonly string[]): string {
return quoteFtsTerm(terms.join(' '))
}
export function andExpression(terms: readonly string[]): string {
return terms.map(quoteFtsTerm).join(' AND ')
}
export function orExpression(terms: readonly string[]): string {
return terms.map(quoteFtsTerm).join(' OR ')
}
/**
* What a scope is, now that there is one FTS table.
*
* `conversation` used to be a second table holding a copy of the two prose
* columns. It is a column filter instead: PR 2 measured the filter at
* 1.16-1.36x the p95 of the dedicated table on a 105 MB corpus, against a 2x
* bar, and the table cost a tenth of the index to maintain.
*
* It lives beside the other expression builders, and not with the retrieval
* that uses it, because the typo repair has to ask the same question of the
* same scope and importing it from there is a cycle.
*
* The filter binds to the whole expression, so it is applied here and nowhere
* else — `{cols}: (a AND b)` filters both terms, while a prefix pasted in front
* of a bare `a AND b` would filter only `a` and quietly search tool output for
* the rest.
*/
const CONVERSATION_COLUMNS = '{user_text assistant_text}'
export function scopedExpression(scope: SessionSearchScope, expression: string): string {
return scope === 'all' ? expression : `${CONVERSATION_COLUMNS}: (${expression})`
}
src/main/ai-vault-search/session-search-query-schema.ts
-79
View File
@@ -1,79 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import {
SESSION_SEARCH_GENERATION_SQL,
SESSION_SEARCH_GENERATION_TRIGGERS
} from './session-search-index-generation'
/** An engine feature the index on disk cannot serve. */
export type SessionSearchUnavailableFeature = 'typo-repair'
const QUERY_SCHEMA_SQL = `
-- The typo repair's whole dictionary. Why the index's own vocabulary and not a
-- word list: it can never suggest a term this index does not hold, and it needs
-- no model. fts5vocab is a view over the FTS5 b-tree, so it costs no extra rows.
CREATE VIRTUAL TABLE IF NOT EXISTS messages_vocab USING fts5vocab(messages_fts, 'row');
-- Locally logged queries, stored as typed, bounded. Nothing writes here unless a
-- caller opts in; the eval set is rebuilt from it (see session-search-query-log).
CREATE TABLE IF NOT EXISTS search_log(
id INTEGER PRIMARY KEY,
ts TEXT NOT NULL,
query TEXT NOT NULL,
route TEXT NOT NULL,
hits INTEGER NOT NULL,
duration_ms REAL NOT NULL
);
${SESSION_SEARCH_GENERATION_SQL}`
/** Everything the SQL above creates, so a missing one is what triggers a re-run. */
const OWNED = ['messages_vocab', 'search_log', ...SESSION_SEARCH_GENERATION_TRIGGERS]
/**
* The vocabulary's target. Creating a fts5vocab table over a missing FTS table
* succeeds and every query against it then fails, so the feature's health is
* this name's presence rather than the vocabulary's own.
*/
const VOCABULARY_SOURCE = 'messages_fts'
const PROBED = [...OWNED, VOCABULARY_SOURCE]
/**
* Creates whatever of the engine's own schema is missing, and reports what it
* still cannot serve.
*
* These objects are the query engine's, not the store's. Nothing on the write
* path reads any of them, so under the stack's YAGNI rule they do not belong in
* PR 2's schema, and an index built by a process that never opens an engine
* carries none of their cost. None of them needs a schema version either: every
* one is derived from what PR 2 already holds, so re-creating them over any of
* its files is correct, while a version bump would throw a whole index away to
* add a view over its own b-tree.
*
* Run per search, not once per engine. A capability is a fact about the file
* rather than about this object: another handle can rebuild the index under a
* live connection, so a verdict taken in a constructor is wrong for the rest of
* the engine's life in both directions — it would keep reaching for a table
* that went away and never pick one back up when it returned. The steady-state
* cost is the single indexed `sqlite_master` lookup below.
*
* A create that throws is not caught. The only way to reach one is an index
* whose `files` table is gone, which is a rebuild in flight — and an engine
* over that cannot report a hit's source either, so there is nothing to degrade
* to. Losing only the vocabulary's source is the case worth surviving, and that
* one is reported rather than thrown.
*/
export function ensureSessionSearchQuerySchema(
db: SyncDatabase
): readonly SessionSearchUnavailableFeature[] {
const present = presentNames(db)
if (OWNED.some((name) => !present.has(name))) {
db.exec(QUERY_SCHEMA_SQL)
}
return present.has(VOCABULARY_SOURCE) ? [] : ['typo-repair']
}
function presentNames(db: SyncDatabase): Set<string> {
const rows = db
.prepare(`SELECT name FROM sqlite_master WHERE name IN (${PROBED.map(() => '?').join(',')})`)
.all(...PROBED) as { name: string }[]
return new Set(rows.map((row) => row.name))
}
src/main/ai-vault-search/session-search-read-decision.test.ts
+117
View File
@@ -0,0 +1,117 @@
import { expect, it } from 'vitest'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import type { SessionSearchIndexedFile } from './session-search-file-cursor'
import {
SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT,
sessionSearchReadDecision
} from './session-search-read-decision'
import type { SessionSearchFileRow } from './session-search-store'
const PATH = '/transcripts/one.jsonl'
const MTIME = 1_740_000_000_000
function candidate(overrides: Partial<SessionFileCandidate['file']> = {}): SessionFileCandidate {
return {
agent: 'claude',
codexHome: null,
file: {
path: PATH,
mtimeMs: MTIME,
modifiedAt: new Date(MTIME).toISOString(),
sizeBytes: 100,
...overrides
}
}
}
function row(overrides: Partial<SessionSearchFileRow> = {}): SessionSearchFileRow {
return {
path: PATH,
identity: null,
mtimeMs: MTIME,
sizeBytes: 100,
state: 'current',
failCount: 0,
failedMtimeMs: null,
...overrides
}
}
const cursor: SessionSearchIndexedFile = { byteOffset: 100, mtimeMs: MTIME, sizeBytes: 100 }
function decide(args: {
file?: Partial<SessionFileCandidate['file']>
row?: SessionSearchFileRow | undefined
cursor?: SessionSearchIndexedFile | null
cutoffMs?: number | null
}) {
return sessionSearchReadDecision({
candidate: candidate(args.file),
row: 'row' in args ? args.row : row(),
cursor: 'cursor' in args ? (args.cursor ?? null) : cursor,
cutoffMs: args.cutoffMs ?? null
})
}
it('reads a path the index holds nothing for, and lets the reader continue where it can', () => {
// Not `whole`: there is no span this index has to reach past, and the first
// enablement inside a running app has a warm list cursor to make use of.
expect(decide({ row: undefined })).toBe('any')
})
it('skips a file the index already covers at this stat', () => {
expect(decide({})).toBe('skip')
})
it('reads a file whose stat moved, however it moved', () => {
expect(decide({ file: { mtimeMs: MTIME + 1 } })).toBe('any')
// Grown without its mtime moving: a same-second append, or a restored stamp.
expect(decide({ file: { sizeBytes: 200 } })).toBe('any')
})
it('reads a file outside the retention window not at all', () => {
expect(decide({ row: undefined, cutoffMs: MTIME + 1 })).toBe('skip')
// And retention wins over everything else that would have asked for a read.
expect(decide({ row: row({ state: 'due' }), cutoffMs: MTIME + 1 })).toBe('skip')
})
it('reads a row owed a whole read from the start', () => {
expect(decide({ row: row({ state: 'due' }) })).toBe('whole')
})
it('reads whole rather than appending onto a cursor that continues nothing', () => {
// A different file at the same name: the identity check hands back no cursor.
expect(decide({ cursor: null })).toBe('whole')
// A chunked read that committed a prefix and no offset any append continues.
expect(decide({ cursor: { byteOffset: null, mtimeMs: MTIME, sizeBytes: 100 } })).toBe('whole')
// Shorter than the index read to, so this is not that file any more.
expect(decide({ file: { sizeBytes: 40 }, cursor })).toBe('whole')
})
it('retries a failed read until it has failed enough times at one stat', () => {
for (let failures = 1; failures < SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT; failures++) {
expect(
decide({ row: row({ state: 'failed', failCount: failures, failedMtimeMs: MTIME }) })
).toBe('any')
}
expect(
decide({
row: row({
state: 'failed',
failCount: SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT,
failedMtimeMs: MTIME
})
})
).toBe('skip')
})
it('starts trying again the moment a held-out file changes', () => {
// The stat is the whole release condition, so nothing has to remember when
// the failures happened or schedule a retry.
expect(
decide({
file: { mtimeMs: MTIME + 1 },
row: row({ state: 'failed', failCount: 9, failedMtimeMs: MTIME })
})
).toBe('any')
})
src/main/ai-vault-search/session-search-read-decision.ts
+100
View File
@@ -0,0 +1,100 @@
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import type { SessionParseReadRequirement } from '../ai-vault/session-scanner-parse-cache'
import { requiresWholeRead, type SessionSearchIndexedFile } from './session-search-file-cursor'
import type { SessionSearchFileRow } from './session-search-store'
/**
* Failures at one unchanged stat before a file is left alone.
*
* Three rather than one, because a single failure is often a transcript being
* rewritten under the read; three at the same mtime is not. The retry policy is
* the stat itself: an edit, a restore, or a `touch` after a `chmod` all move it,
* and nothing else does, so no timer is needed and none is kept.
*/
export const SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT = 3
/**
* What a pass owes one candidate: nothing, a read, or a read from the start.
*
* `any` and `whole` are the reader's own lanes. `whole` drops the session
* list's resume point, which is the only way to reach a span this index never
* saw; `any` asks for some bytes and lets the reader continue where it can,
* which is what the first enablement inside a running app needs — a warm list
* cursor sitting at the file's current stat would otherwise open nothing.
*/
export type SessionSearchReadDecision = 'skip' | SessionParseReadRequirement
/**
* The whole of the indexer's decide step, as a function of the candidate's stat
* and the row the store holds for it. No pass state, no queue, no memory: the
* same inputs give the same answer on the first pass after a restart as on the
* hundredth of a long-running process, which is what lets a deadline cut a pass
* short with nothing to record. What did not get read is still owed, because
* being owed is a fact about the row.
*/
export function sessionSearchReadDecision(args: {
candidate: SessionFileCandidate
/** The file table's row, or undefined when the index holds nothing for it. */
row: SessionSearchFileRow | undefined
/** The cursor for this candidate's identity; null when it is not continuable. */
cursor: SessionSearchIndexedFile | null
/** Oldest transcript mtime worth holding rows for, or null for all history. */
cutoffMs: number | null
}): SessionSearchReadDecision {
const { candidate, row, cursor, cutoffMs } = args
const file = candidate.file
// Retention first: a file outside the window is not worth reading whatever
// else is true of it, and the purge is what removes any row it still has.
if (cutoffMs !== null && file.mtimeMs < cutoffMs) {
return 'skip'
}
if (!row) {
// Nothing held for this path. Not `whole`, because the reader can continue
// from wherever it likes: there is no span this index has to reach past.
return 'any'
}
if (heldOut(row, file.mtimeMs)) {
return 'skip'
}
if (row.state === 'due') {
// The index is behind on a span no append reaches: a declined append, or a
// window that widened to admit this file.
return 'whole'
}
if (cursor === null || requiresWholeRead(cursor)) {
// A different file at the same name, or a chunked read that left a prefix
// and no cursor. Appending onto either would splice two spans together.
return 'whole'
}
const size = file.sizeBytes
if (typeof size === 'number' && cursor.byteOffset !== null && cursor.byteOffset > size) {
// Shorter than the index read to: this is not the file that cursor came from.
return 'whole'
}
if (row.state === 'failed') {
// Still within its retries, or the stat moved since it last failed.
return 'any'
}
return statMatches(row, file) ? 'skip' : 'any'
}
/**
* True when this file has failed enough times at exactly this stat to stop
* trying. The stat is the whole release condition, so a file nobody touches is
* never read again and one that changes is read on the next pass that sees it.
*/
function heldOut(row: SessionSearchFileRow, mtimeMs: number): boolean {
return (
row.state === 'failed' &&
row.failCount >= SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT &&
row.failedMtimeMs === mtimeMs
)
}
/** The row already describes the file as it is now. */
function statMatches(row: SessionSearchFileRow, file: SessionFileCandidate['file']): boolean {
return (
row.mtimeMs === file.mtimeMs &&
(row.sizeBytes === null || file.sizeBytes === undefined || row.sizeBytes === file.sizeBytes)
)
}
src/main/ai-vault-search/session-search-retention-policy.test.ts
+26
View File
@@ -0,0 +1,26 @@
import { expect, it } from 'vitest'
import { sessionSearchHistoryCutoffMs } from './session-search-retention-policy'
const NOW = 1_740_000_000_000
it('treats a fractional or non-positive day count as no bound at all', () => {
// A day count that floors to zero would read as "all history" in one place
// and "cutoff is now" in the other; both sides answer null.
expect(sessionSearchHistoryCutoffMs(0.4, NOW)).toBeNull()
expect(sessionSearchHistoryCutoffMs(0, NOW)).toBeNull()
expect(sessionSearchHistoryCutoffMs(-30, NOW)).toBeNull()
expect(sessionSearchHistoryCutoffMs(30, NOW)).toBe(NOW - 30 * 86_400_000)
// Clamped rather than unbounded: a caller asking for three thousand years of
// history gets the ceiling, not an mtime before the epoch.
expect(sessionSearchHistoryCutoffMs(999_999, NOW)).toBe(NOW - 3_650 * 86_400_000)
})
// The cutoff is read from the clock on every pass, not frozen at construction:
// a purge and the accept check that follows it must not disagree about where
// the window is, or the sweep deletes rows the next candidate re-indexes.
it('moves the cutoff with the clock', () => {
const later = NOW + 86_400_000
expect(sessionSearchHistoryCutoffMs(30, later)).toBe(
(sessionSearchHistoryCutoffMs(30, NOW) ?? 0) + 86_400_000
)
})
src/main/ai-vault-search/session-search-retention-policy.ts
+25
View File
@@ -0,0 +1,25 @@
const DAY_MS = 86_400_000
const HISTORY_DAYS_MAX = 3_650
/**
* The retention window, as the indexer's callers state it and as the store
* consumes it. Settings storage is PR 3b's problem; this is the arithmetic.
*/
function normalizeSessionSearchHistoryDays(value: number | null): number | null {
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
return null
}
// Why floor then re-check: a fractional day floors to 0, which reads as "all
// history" on one side and "now" on the other; make the two agree.
const days = Math.floor(value)
return days <= 0 ? null : Math.min(HISTORY_DAYS_MAX, days)
}
/** The oldest transcript mtime worth indexing; null means no bound. */
export function sessionSearchHistoryCutoffMs(
historyDays: number | null,
nowMs: number
): number | null {
const days = normalizeSessionSearchHistoryDays(historyDays)
return days === null ? null : nowMs - days * DAY_MS
}
src/main/ai-vault-search/session-search-retrieval.ts
-251
View File
@@ -1,251 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import type { SessionSearchRoute, SessionSearchScope } from './session-search-engine-types'
import type { MessageRow, SessionRow } from './session-search-hit-ranking'
import {
andExpression,
orExpression,
phraseExpression,
planSessionSearchQuery,
scopedExpression,
type SessionSearchQueryPlan
} from './session-search-query-planner'
import type { SessionRowFilter } from './session-search-row-filter'
import { SessionSearchTypoRepair } from './session-search-typo-repair'
// The operator-only walk: rows per page, and how far past a full candidate set
// it will read before giving up on finding more matches.
const RECENT_PAGE_ROWS = 512
// Ids per `loadSessions` statement, with room to spare for the filter's own
// bound values beside them.
const SESSION_ID_BATCH = 500
const RECENT_SCAN_FACTOR = 20
// Measured: user 3 / assistant 2 / tool 1 / identifiers 1 (MRR 0.503 vs 0.475 flat).
const FULL_WEIGHTS = '3.0, 2.0, 1.0, 1.0'
// The conversation scope zeroes the two columns its filter already excludes.
// Measured, and stated because it is easy to over-read: these zeros change no
// score. FTS5's bm25 sums over the columns the query matched, and the filter
// has already kept the match out of those two, so the same rows come back with
// `1.0, 1.0` here. They are a statement of what the scope means, not the fence
// that enforces it — `scopedExpression` is the fence.
const CONVERSATION_WEIGHTS = '3.0, 2.0, 0.0, 0.0'
export type RetrievalScope = {
scope: SessionSearchScope
sort: 'relevance' | 'newest'
filter: SessionRowFilter
/**
* `repo:` / `path:`, which SQL cannot express. Applied over retrieved rows;
* see session-search-row-filter for why it cannot be pushed down.
*/
matchesOperators: (session: SessionRow) => boolean
/**
* Sessions retrieved before ranking cuts the page. See
* docs/reference/agent-session-search-query-tuning.md for the measurements
* behind the default; it is an option because the right value depends on how
* large an index is and no single number is right for every host.
*/
candidateLimit: number
}
export type Retrieved = {
rows: MessageRow[]
route: SessionSearchRoute
/** The plan the rows were actually retrieved by; snippets highlight from it. */
plan: SessionSearchQueryPlan
repairedTerms?: string[]
}
/**
* The bm25 weights a scope ranks with. The conversation pair stays here rather
* than beside `scopedExpression`, because weights are a property of this SQL
* and nothing else asks for them.
*/
export function scopedWeights(scope: SessionSearchScope): string {
return scope === 'all' ? FULL_WEIGHTS : CONVERSATION_WEIGHTS
}
/** The FTS half of a search: the route ladder and the SQL each rung runs. */
export class SessionSearchRetrieval {
/** Null when this index has no vocabulary to repair against; the rung is skipped. */
private readonly typoRepair: SessionSearchTypoRepair | null
constructor(
private readonly db: SyncDatabase,
canRepairTypos = true
) {
this.typoRepair = canRepairTypos ? new SessionSearchTypoRepair(db) : null
}
/**
* The route ladder: phrase, then AND for a literal-looking query, then typo
* repair, then OR.
*
* Repair runs before the OR fallback rather than after it fails. A typo next
* to a common word would otherwise be masked: the common word alone retrieves
* plenty of rows over OR, so nothing would ever look like a miss worth
* repairing.
*/
run(plan: SessionSearchQueryPlan, scope: RetrievalScope): Retrieved {
const exact = this.literal(plan, scope)
if (exact) {
return { ...exact, plan }
}
const repaired = this.repair(plan, scope.scope)
const effective = repaired ?? plan
const literal = repaired ? this.literal(repaired, scope) : null
const found = literal ?? {
rows: this.match(orExpression(effective.terms), scope),
route: 'or' as const
}
return {
rows: found.rows,
route: repaired ? (`typo+${found.route}` as SessionSearchRoute) : found.route,
plan: effective,
...(repaired ? { repairedTerms: repaired.body } : {})
}
}
/**
* Newest sessions the constraints allow: what an operator-only query names.
*
* Walked in pages rather than taken in one `LIMIT`, because the operators are
* applied in JS. A single cut of the newest N would hand ranking whatever
* happened to be recent and then throw most of it away, so `repo:x` on a busy
* index could answer with nothing while plenty matched. The walk is bounded
* both ways: it stops at a full candidate set, and at a ceiling on rows read.
*/
recent(scope: RetrievalScope): { sessions: SessionRow[]; incomplete: boolean } {
const { conditions, values } = scope.filter
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
const page = this.db.prepare(
`SELECT * FROM sessions ${where}
ORDER BY updated_at DESC, id DESC LIMIT ? OFFSET ?`
)
const ceiling = scope.candidateLimit * RECENT_SCAN_FACTOR
const sessions: SessionRow[] = []
let scanned = 0
// Why the flag and not a count: both caps mean the same thing to a caller —
// a session it never saw may have matched — and only the loop knows which
// of them ended it. Reporting rows read instead let the engine infer
// completeness from a full candidate set alone, so giving up at the ceiling
// with nothing found looked exactly like a search that found nothing.
let incomplete = false
while (sessions.length < scope.candidateLimit) {
if (scanned >= ceiling) {
incomplete = true
break
}
const rows = page.all(...values, RECENT_PAGE_ROWS, scanned) as SessionRow[]
if (rows.length === 0) {
break
}
scanned += rows.length
for (const row of rows) {
if (sessions.length < scope.candidateLimit && scope.matchesOperators(row)) {
sessions.push(row)
}
}
}
return { sessions, incomplete: incomplete || sessions.length >= scope.candidateLimit }
}
/**
* Read in batches, because the id list is as long as the candidate limit and
* every id is a bound parameter, so a single statement scales with a knob the
* tuning doc invites a host to raise.
*
* Not a fix for a reachable failure, and worth saying so: SQLite has bound
* `SQLITE_MAX_VARIABLE_NUMBER` at 32,766 since 3.32, every runtime this stack
* supports is past that, and the measured limit on this one is higher still.
* A candidate limit that large is not a configuration anyone would choose.
* The batch is here so the ceiling belongs to this file rather than to
* whichever SQLite the process happened to link.
*/
loadSessions(ids: readonly number[], scope: RetrievalScope): SessionRow[] {
const rows: SessionRow[] = []
for (let start = 0; start < ids.length; start += SESSION_ID_BATCH) {
const batch = ids.slice(start, start + SESSION_ID_BATCH)
const conditions = [`id IN (${batch.map(() => '?').join(',')})`, ...scope.filter.conditions]
rows.push(
...(this.db
.prepare(`SELECT * FROM sessions WHERE ${conditions.join(' AND ')}`)
.all(...batch, ...scope.filter.values) as SessionRow[])
)
}
return rows.filter((row) => scope.matchesOperators(row))
}
private repair(
plan: SessionSearchQueryPlan,
scope: SessionSearchScope
): SessionSearchQueryPlan | null {
if (!this.typoRepair) {
return null
}
const typoRepair = this.typoRepair
let changed = false
const body = plan.body.map((term) => {
// Repaired inside the scope the search will run in, so a spelling only
// tool output carries neither suppresses a repair nor becomes one.
const fix = typoRepair.correct(term, scope)
if (fix && fix !== term.toLowerCase()) {
changed = true
return fix
}
return term
})
// The repair changes spellings, not the query's character: the re-plan is
// told what the original decided so a corrected literal keeps every term it
// was typed with.
return changed ? planSessionSearchQuery(body.join(' '), plan.literal) : null
}
/** Phrase, then AND, for literal-looking queries; null when neither matches. */
private literal(
plan: SessionSearchQueryPlan,
scope: RetrievalScope
): { rows: MessageRow[]; route: 'phrase' | 'and' } | null {
if (!plan.literal || plan.body.length === 0) {
return null
}
// A one-token literal (`resolveTerminalPath`, `src/a/b.ts`) is its own
// phrase: the tokenizer keeps it whole, so the exact token is the cheap,
// precise first try before the identifier pieces fan out over OR.
const phrase = this.match(phraseExpression(plan.body), scope)
if (phrase.length > 0) {
return { rows: phrase, route: 'phrase' }
}
if (plan.body.length < 2) {
return null
}
const and = this.match(andExpression(plan.body), scope)
return and.length > 0 ? { rows: and, route: 'and' } : null
}
private match(expression: string, scope: RetrievalScope): MessageRow[] {
const { filter, sort, candidateLimit } = scope
const eligible = filter.conditions.length
? ` AND m.session_row_id IN (SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')})`
: ''
const matched = `SELECT messages_fts.rowid AS rowid,
-bm25(messages_fts, ${scopedWeights(scope.scope)}) AS score,
m.session_row_id, m.role, m.ts, s.updated_at
FROM messages_fts JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id WHERE messages_fts MATCH ?${eligible}`
// Why: collapse to one row per session BEFORE the candidate limit, on both
// sort orders, so a single long session cannot occupy the whole page.
// `max(score)` makes SQLite pick that session's best row for the bare columns.
// Cost of grouping instead of a bounded top-N sorter, measured: ~1.75x
// (49.6 vs 28.6 ms at 80k matching rows, 183.6 vs 104.1 ms at 240k) and a
// temp b-tree over every match. No inner LIMIT can bound it: the CTE has no
// order, so any cut drops whole sessions rather than their surplus rows.
const order = sort === 'newest' ? 'updated_at DESC, score DESC' : 'score DESC'
const sql = `WITH matched AS MATERIALIZED (${matched})
SELECT rowid, max(score) AS score, session_row_id, role, ts FROM matched
GROUP BY session_row_id ORDER BY ${order} LIMIT ${candidateLimit}`
return this.db
.prepare(sql)
.all(scopedExpression(scope.scope, expression), ...filter.values) as MessageRow[]
}
}
src/main/ai-vault-search/session-search-row-filter.test.ts
-137
View File
@@ -1,137 +0,0 @@
import { afterEach, describe, expect, it } from 'vitest'
import type SyncDatabase from '../sqlite/sync-database'
import type { SessionSearchFilters } from './session-search-engine-types'
import { cwdKey } from './session-search-file-records'
import { sessionRowFilter } from './session-search-row-filter'
import {
openSessionSearchIndexFile,
type SessionSearchIndexFile
} from './session-search-index-test-fixture'
let index: SessionSearchIndexFile | null = null
afterEach(async () => {
await index?.close()
index = null
})
async function openIndex(): Promise<SyncDatabase> {
index = await openSessionSearchIndexFile('ss-row-filter')
return index.db
}
function addSession(
db: SyncDatabase,
id: number,
cwd: string | null,
overrides: { agent?: string; updatedAt?: string } = {}
): void {
db.prepare(
`INSERT INTO sessions(id,agent,session_id,file_path,title,cwd,cwd_key,updated_at,resume_command)
VALUES (?,?,?,?,'fixture',?,?,?,'')`
).run(
id,
overrides.agent ?? 'claude',
String(id),
`/synthetic/${id}`,
cwd,
cwdKey(cwd),
overrides.updatedAt ?? '2026-09-01T00:00:00.000Z'
)
}
function selected(db: SyncDatabase, filters: SessionSearchFilters = {}): number[] {
const filter = sessionRowFilter(filters)
const where = filter.conditions.length > 0 ? `WHERE ${filter.conditions.join(' AND ')}` : ''
return (
db.prepare(`SELECT id FROM sessions ${where} ORDER BY id`).all(...filter.values) as {
id: number
}[]
).map((row) => row.id)
}
describe('a cwd scope is the sidebar key, or anything below it', () => {
it.each([
['C:\\Work\\App', 'c:/work/app', true],
['C:\\Work\\App\\src', 'c:/work/app', true],
['/work/APP/src', '/work/app', false],
['/work/caf\u00e9', '/work/cafe\u0301', true],
['/work/app-other', '/work/app', false],
['/work/a_b/src', '/work/a_b', true],
['/work/axb/src', '/work/a_b', false],
// Roots: `/` is the one key that is already a separator, which is where a
// range bound is easiest to get wrong. A Windows key is not under POSIX `/`.
['/', '/', true],
['/work/app', '/', true],
['C:\\Work\\App', '/', false],
['C:\\', 'C:\\', true],
['C:\\Work\\App', 'C:\\', true]
])('scopes %s under %s: %s', async (cwd, scope, expected) => {
const db = await openIndex()
addSession(db, 1, cwd)
expect(selected(db, { scopePaths: [scope] })).toEqual(expected ? [1] : [])
})
it('never matches a session whose transcript recorded no cwd', async () => {
const db = await openIndex()
addSession(db, 1, null)
expect(selected(db, { scopePaths: ['/work'] })).toEqual([])
expect(selected(db)).toEqual([1])
})
it('keeps a WSL UNC workspace distinct from the bare Linux spelling', async () => {
// PR 2 decided cwd_key does not qualify a Linux path with its distro: the
// collision is real but every SSH host has it too, and the fix is a column
// naming the execution host, not a key only some hosts spell differently.
const db = await openIndex()
addSession(db, 1, '\\\\wsl.localhost\\Ubuntu\\home\\ada\\app')
addSession(db, 2, '/home/ada/app')
expect(selected(db, { scopePaths: ['\\\\wsl$\\Ubuntu\\home\\ada'] })).toEqual([1])
expect(selected(db, { scopePaths: ['/home/ada/app'] })).toEqual([2])
expect(selected(db, { scopePaths: ['\\\\wsl$\\Debian\\home\\ada\\app'] })).toEqual([])
})
})
describe('caller filters', () => {
it('narrows by agent, and by updated-at floor', async () => {
const db = await openIndex()
addSession(db, 1, '/work/app', { agent: 'claude', updatedAt: '2026-09-01T00:00:00.000Z' })
addSession(db, 2, '/work/app', { agent: 'codex', updatedAt: '2026-09-05T00:00:00.000Z' })
expect(selected(db, { agents: ['codex'] })).toEqual([2])
expect(selected(db, { since: '2026-09-03T00:00:00.000Z' })).toEqual([2])
expect(selected(db, { agents: ['claude'], since: '2026-09-03T00:00:00.000Z' })).toEqual([])
})
it('applies the retention cutoff through the files table', async () => {
const db = await openIndex()
addSession(db, 1, '/work/app')
addSession(db, 2, '/work/app')
db.prepare(
"INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES ('a',0,100,1)"
).run()
db.prepare(
"INSERT INTO files(path,byte_offset,mtime_ms,session_row_id) VALUES ('b',0,500,2)"
).run()
const filter = sessionRowFilter({}, 300)
const rows = db
.prepare(`SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')}`)
.all(...filter.values) as { id: number }[]
expect(rows.map((row) => row.id)).toEqual([2])
})
})
it('plans a cwd scope as a seek on sessions_cwd_key, never a scan', async () => {
const db = await openIndex()
const filter = sessionRowFilter({ scopePaths: ['/work/app'] })
const plan = (
db
.prepare(
`EXPLAIN QUERY PLAN SELECT id FROM sessions WHERE ${filter.conditions.join(' AND ')}`
)
.all(...filter.values) as { detail: string }[]
).map((row) => row.detail)
expect(plan.join(' | ')).toContain('sessions_cwd_key')
expect(plan.some((detail) => detail.startsWith('SEARCH'))).toBe(true)
expect(plan.some((detail) => detail.startsWith('SCAN sessions'))).toBe(false)
})
src/main/ai-vault-search/session-search-row-filter.ts
-90
View File
@@ -1,90 +0,0 @@
import { cwdKey } from './session-search-file-records'
import type { SessionSearchFilters } from './session-search-engine-types'
/** SQL fragments for the `sessions` WHERE clause; every condition is ANDed. */
export type SessionRowFilter = {
conditions: string[]
values: (string | number)[]
}
// Stored identity: `cwdKey` is the sidebar's `folderGroupKey` without its prefix,
// so a scope term and an indexed session are keyed by one function, never two.
const CWD = 'cwd_key'
/**
* The narrowings SQL can express exactly, in one place, so retrieval, the
* operator-only page and the session load cannot drift apart. These conditions
* run over `sessions` itself. Reachability is not here and is not a condition:
* it is the INNER JOIN to `sessions` that every retrieval carries, which is
* what makes a message row a purge has not reclaimed yet unreadable.
*
* `repo:` and `path:` are deliberately absent. What they mean is the predicate
* the sessions panel applies (`matchesAiVaultQueryOperators`), and SQL cannot
* express it: LIKE folds ASCII and nothing else, so `path:CAFÉ` would miss
* `café`; `path:` searches the transcript path as well as the working
* directory, so `path:jsonl` would miss every session; and `repo:` compares the
* last two path segments, not one. A second spelling that came close would be a
* query meaning different things in the list and in the index, so the engine
* applies the panel's own predicate over the rows it retrieves instead.
*
* `scopePaths` stays here because it is exact: a prefix range over the key
* `cwdKey` produces, which folds exactly where the execution host folds —
* Windows drives, never a POSIX directory name.
*/
export function sessionRowFilter(
filters: SessionSearchFilters,
cutoffMs: number | null = null
): SessionRowFilter {
const filter: SessionRowFilter = { conditions: [], values: [] }
if (cutoffMs !== null) {
filter.conditions.push('id IN (SELECT session_row_id FROM files WHERE mtime_ms >= ?)')
filter.values.push(cutoffMs)
}
if (filters.agents && filters.agents.length > 0) {
filter.conditions.push(`agent IN (${filters.agents.map(() => '?').join(',')})`)
filter.values.push(...filters.agents)
}
if (filters.since) {
filter.conditions.push('updated_at >= ?')
filter.values.push(filters.since)
}
if (filters.scopePaths && filters.scopePaths.length > 0) {
// Several scopes mean any of them; every other narrowing is ANDed on.
const present = filters.scopePaths
.map((scope) => scopeCondition(filter, scope))
.filter((condition) => condition !== null)
if (present.length > 0) {
filter.conditions.push(`(${present.join(' OR ')})`)
}
}
return filter
}
/** A scope the caller could not key is a scope nothing is inside of. */
function scopeCondition(filter: SessionRowFilter, scope: string): string | null {
const key = cwdKey(scope)
return key === null ? null : insideCondition(filter, key)
}
/**
* `key` itself, or anything below it. Why a half-open range and not
* `substr(key, 1, length(?)) = ?`: only `>=`/`<` can seek `sessions_cwd_key`;
* the substr form scans it. The bound is the child prefix with its last byte
* incremented, so it stops at the end of that prefix and nowhere else. The two
* arms cannot merge: one range over the bare key would also swallow a sibling
* like `/work/app-other`. No wildcards, so `%`/`_` in a folder name are literal.
*
* The filesystem root is the one key that already ends in a separator, and
* appending a second one would bound the range at `//`, which sorts below every
* real child; `cwdKey` keeps it as `/` for exactly this reason.
*/
function insideCondition(filter: SessionRowFilter, key: string): string {
const children = key.endsWith('/') ? key : `${key}/`
filter.values.push(key, children, nextAfterPrefix(children))
return `(${CWD} = ? OR (${CWD} >= ? AND ${CWD} < ?))`
}
/** The first string that sorts after every string starting with `prefix`. */
function nextAfterPrefix(prefix: string): string {
return prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1)
}
src/main/ai-vault-search/session-search-scan-roots.test.ts
+58
View File
@@ -0,0 +1,58 @@
import { expect, it } from 'vitest'
import { delimiter, join } from 'node:path'
import type { SessionFileDiscovery } from '../ai-vault/session-scanner-types'
import { sessionSearchRootListings } from './session-search-scan-roots'
const STATE = '/tmp/ss-roots/openclaw-state'
const LEGACY = '/tmp/ss-roots/openclaw-legacy'
function file(path: string): SessionFileDiscovery['files'][number] {
return { path, mtimeMs: 0, modifiedAt: new Date(0).toISOString() }
}
it('splits a merged discovery into the real directories behind it', () => {
const current = join(STATE, 'agents')
const legacy = join(LEGACY, 'agents')
const listings = sessionSearchRootListings(
{ openclawStateDir: STATE, openclawLegacyStateDir: LEGACY },
[
{
agent: 'openclaw',
// What discovery reports for an agent whose roots are alternates.
rootDir: [current, legacy].join(delimiter),
files: [
file(join(current, 'a', 'sessions', 'one.jsonl')),
file(join(current, 'a', 'sessions', 'two.jsonl')),
file(join(legacy, 'b', 'sessions', 'three.jsonl'))
]
}
]
)
const byRoot = Object.fromEntries(listings.map((one) => [one.root, one.files]))
expect(byRoot[current]).toBe(2)
expect(byRoot[legacy]).toBe(1)
// The joined string is never reported as a directory.
expect(listings.every((one) => !one.root.includes(delimiter))).toBe(true)
})
it('attributes a file by path segment, not by string prefix', () => {
const agents = join(STATE, 'agents')
const legacy = join(LEGACY, 'agents')
const listings = sessionSearchRootListings(
{ openclawStateDir: STATE, openclawLegacyStateDir: LEGACY },
[
{
agent: 'openclaw',
rootDir: [agents, legacy].join(delimiter),
// A sibling directory whose name merely starts with a root's name. It
// is under no root, so it belongs to none of them.
files: [file(join(`${agents}-old`, 'b', 'sessions', 'two.jsonl'))]
}
]
)
const byRoot = Object.fromEntries(listings.map((one) => [one.root, one.files]))
expect(byRoot[agents]).toBe(0)
expect(byRoot[legacy]).toBe(0)
})
src/main/ai-vault-search/session-search-scan-roots.ts
+135
View File
@@ -0,0 +1,135 @@
import type { AiVaultScanIssue } from '../../shared/ai-vault-types'
import { AI_VAULT_AGENT_SOURCES } from '../ai-vault/session-scanner-agent-sources'
import { normalizedWslHomeDirs } from '../ai-vault/session-scanner-roots'
import { sessionCandidatesFromDiscoveries } from '../ai-vault/session-scanner-candidates'
import { discoverAiVaultSessionSources } from '../ai-vault/session-scanner-source-discovery'
import type {
AiVaultScanOptions,
SessionFileCandidate,
SessionFileDiscovery
} from '../ai-vault/session-scanner-types'
/** One real directory a scan walked, and what it listed there. */
export type SessionSearchRootListing = { root: string; files: number }
/**
* Where the indexer looks. The caller resolves these so the index enumerates
* exactly the trees the session list does; the indexer owns the bounds
* (`limit`, `limitPerAgent`, `unlimited`) and its own cancellation, so those
* are not the caller's to set.
*/
export type SessionSearchScanRoots = Omit<
AiVaultScanOptions,
'signal' | 'limit' | 'unlimited' | 'limitPerAgent' | 'scopePaths'
>
export type SessionSearchDiscovery = {
/** Newest first, Codex hardlink aliases collapsed, exactly as a list scan sees them. */
candidates: SessionFileCandidate[]
discoveries: SessionFileDiscovery[]
issues: AiVaultScanIssue[]
}
/**
* The discovery half of a list scan, without the parse. `limitPerAgent` is the
* sidebar's own recency rule (`SessionNewestFiles` keeps the newest N per root);
* passing Infinity is what makes a sweep whole.
*/
export async function discoverSessionSearchCandidates(
roots: SessionSearchScanRoots,
args: { limitPerAgent: number; signal?: AbortSignal }
): Promise<SessionSearchDiscovery> {
const issues: AiVaultScanIssue[] = []
const options: AiVaultScanOptions = { ...roots, signal: args.signal }
const discoveries = await discoverAiVaultSessionSources({
options,
limitPerAgent: args.limitPerAgent,
issues
})
const candidates = await sessionCandidatesFromDiscoveries(discoveries, options)
return { candidates, discoveries, issues }
}
/**
* Containment on path segments, not on string prefix, and on both separators:
* discovery joins with the platform's, a configured root can arrive spelled
* with the other, and `/a/agents-old` is not inside `/a/agents`.
*/
export function isUnderScanRoot(path: string, root: string): boolean {
return root.length > 0 && (path.startsWith(`${root}/`) || path.startsWith(`${root}\\`))
}
/**
* The real directories behind a scan's discoveries, with their file counts.
*
* Why this exists: an agent whose roots are alternates for one install reports
* them as a single discovery whose `rootDir` is every path joined by the
* platform's path delimiter. That string is not a directory. Health probes
* readdir it and get ENOENT, a containment check never matches a file under it,
* and a scan issue recorded against a real root never equals it — so the fence
* meant to protect an unmounted tree is inert for exactly the agent most likely
* to have one. Splitting the joined string back apart would be worse: a
* directory may legally contain the delimiter. The constituent paths come from
* the same source table discovery read.
*/
export function sessionSearchRootListings(
roots: SessionSearchScanRoots,
discoveries: readonly SessionFileDiscovery[]
): SessionSearchRootListing[] {
const wslHomeDirs = normalizedWslHomeDirs(roots.wslHomeDirs)
const counts = new Map<string, number>()
for (const discovery of discoveries) {
const constituents = constituentRoots(roots, wslHomeDirs, discovery)
for (const root of constituents) {
counts.set(root, counts.get(root) ?? 0)
}
for (const file of discovery.files) {
const owner = owningRoot(constituents, file.path)
if (owner !== null) {
counts.set(owner, (counts.get(owner) ?? 0) + 1)
}
}
}
return [...counts].map(([root, files]) => ({ root, files }))
}
function constituentRoots(
roots: SessionSearchScanRoots,
wslHomeDirs: readonly string[],
discovery: SessionFileDiscovery
): string[] {
const declared = AI_VAULT_AGENT_SOURCES[discovery.agent]?.rootDirs(roots, wslHomeDirs) ?? []
if (declared.includes(discovery.rootDir)) {
return [discovery.rootDir]
}
// Either a merged discovery, whose rootDir is the joined string, or a source
// that builds its own discoveries (OpenCode, Antigravity) and reports a real
// directory that this table does not list.
return declared.length > 0 ? declared : [discovery.rootDir]
}
function owningRoot(constituents: readonly string[], path: string): string | null {
let owner: string | null = null
for (const root of constituents) {
if (isUnderScanRoot(path, root) && (owner === null || root.length > owner.length)) {
owner = root
}
}
return owner
}
/**
* Roots that listed transcripts on the previous pass and list none on this one.
*
* The one bit of memory the retirement walk gets, and what it buys: a root that
* blinks empty for a single pass is unverifiable rather than proven gone, so a
* sync client swapping a directory out cannot retire a tree. It is deliberately
* not evidence that survives the process — see the invariant block in
* `session-search-deleted-sources.ts` for what that costs and why.
*/
export function sessionSearchEmptiedRoots(
previous: ReadonlySet<string>,
current: ReadonlySet<string>
): Set<string> {
return new Set([...previous].filter((root) => !current.has(root)))
}
src/main/ai-vault-search/session-search-schema.ts
+12 -1
View File
@@ -57,7 +57,18 @@ CREATE TABLE IF NOT EXISTS files(
byte_offset INTEGER NOT NULL,
mtime_ms REAL NOT NULL,
size_bytes INTEGER,
session_row_id INTEGER
session_row_id INTEGER,
-- What this row still owes a reader, so that nothing has to be remembered
-- between passes. 'current': the rows match the file at the stat recorded
-- here. 'due': the index is behind on content it cannot reach by appending,
-- so the next pass reads the file whole. 'failed': the last read did not
-- commit, and the two columns below are what stop it being retried for ever.
state TEXT NOT NULL DEFAULT 'current',
fail_count INTEGER NOT NULL DEFAULT 0,
-- The mtime the failures were observed at. A file that fails at one stat is
-- left alone once it has failed enough times, and only a change to this stat
-- can mean the file itself changed, so it is the whole retry policy.
failed_mtime_ms REAL
);
-- Retention walks the expiring end of this column; without it that is a full scan and a sort.
CREATE INDEX IF NOT EXISTS files_mtime ON files(mtime_ms);
src/main/ai-vault-search/session-search-sidebar-parity.test.ts
-137
View File
@@ -1,137 +0,0 @@
import { afterEach, expect, it } from 'vitest'
import type { AiVaultSession } from '../../shared/ai-vault-types'
import { filterAiVaultSessions } from '../../shared/ai-vault-session-filters'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
// `repo:` and `path:` have to mean one thing. The sessions panel and the index
// answer from different stores by different mechanisms, so the only way to keep
// them equal is for both to run the same predicate; this asserts they do, over
// the shapes where a second SQL spelling went wrong.
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
type Fixture = { id: number; cwd: string; filePath: string; text: string }
const SESSIONS: Fixture[] = [
{
id: 1,
cwd: '/Users/Ada/orca/session-search',
filePath: '/Users/Ada/.claude/projects/a/one.jsonl',
text: 'harbor pilot manifest'
},
{
id: 2,
cwd: '/Users/ada/work/café',
filePath: '/Users/ada/.codex/sessions/two.jsonl',
text: 'harbor dock crane'
},
{
id: 3,
cwd: '/srv/other/service',
filePath: '/srv/.claude/projects/b/three.jsonl',
text: 'harbor manifest beta'
},
{
id: 4,
cwd: 'C:\\Work\\Orca\\App',
filePath: 'C:\\Users\\Ada\\.claude\\four.jsonl',
text: 'harbor windows lane'
}
]
// Each of these matched in the panel and missed in the index while the engine
// tried to say `repo:` / `path:` in SQL.
const QUERIES = [
'harbor path:jsonl',
'harbor repo:orca/session-search',
'harbor path:CAFÉ',
'harbor path:/Users/Ada/orca',
'harbor repo:app',
'harbor repo:Orca/App',
'harbor path:.codex',
'harbor path:/srv repo:other/service',
'harbor repo:session-search path:jsonl',
'harbor path:"/Users/ada/work"',
'harbor repo:nothing-here',
'harbor path:one.jsonl path:two.jsonl',
'harbor'
]
function asSession(fixture: Fixture): AiVaultSession {
const at = '2026-09-01T00:00:00.000Z'
return {
id: String(fixture.id),
executionHostId: 'local',
agent: 'claude',
sessionId: String(fixture.id),
title: 'fixture',
cwd: fixture.cwd,
branch: null,
model: null,
filePath: fixture.filePath,
codexHome: null,
createdAt: at,
updatedAt: at,
modifiedAt: at,
messageCount: 1,
totalTokens: 0,
previewMessages: [{ role: 'user', text: fixture.text }],
queuedMessageCount: 0,
subagentTranscriptCount: 0,
resumeCommand: '',
subagent: null
} as AiVaultSession
}
/** The panel's own answer, operators only: free text is FTS in the index. */
function sidebarIds(query: string): string[] {
const operatorsOnly = query
.split(/\s+/)
.filter((token) => /^(repo|path):/i.test(token))
.join(' ')
return filterAiVaultSessions(SESSIONS.map(asSession), {
query: operatorsOnly,
agents: ['claude'],
scope: 'all',
sort: 'updated',
activeWorktreePaths: [],
hideEmptySessions: false
})
.map((session) => session.sessionId)
.sort()
}
it.each(QUERIES)('answers %s the way the sessions panel does', async (query) => {
harness = await openSessionSearchHarness('ss-sidebar-parity')
for (const fixture of SESSIONS) {
addSyntheticSession(harness.db, {
id: fixture.id,
cwd: fixture.cwd,
text: fixture.text,
filePath: fixture.filePath,
sessionFilePath: fixture.filePath
})
}
const engineIds = harness.engine
.search({ query, limit: 100 })
.hits.map((hit) => hit.sessionId)
.sort()
expect(engineIds).toEqual(sidebarIds(query))
})
it('is not vacuous: these queries do select, and reject, real sessions', () => {
// A parity suite where every query matched everything, or nothing, would pass
// against any predicate at all.
const answers = QUERIES.map((query) => sidebarIds(query).length)
expect(answers.some((count) => count > 0 && count < SESSIONS.length)).toBe(true)
expect(answers.some((count) => count === 0)).toBe(true)
})
src/main/ai-vault-search/session-search-snippet-marks.test.ts
-112
View File
@@ -1,112 +0,0 @@
import { afterEach, expect, it } from 'vitest'
import {
SESSION_SEARCH_SNIPPET_MARK_CLOSE,
SESSION_SEARCH_SNIPPET_MARK_OPEN
} from './session-search-engine-types'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
// A snippet has to name which of a row's four columns matched, and the marks
// FTS5 wraps a match in are the only signal. Searching the marked text for the
// public `[[` reads a transcript's own brackets as a highlight — and transcripts
// are full of them, because a bash `[[ -f x ]]` and numpy's `[[1, 2]]` are
// exactly the sort of thing an agent session holds. Whether a column matched is
// the difference between two renderings of the same text instead.
let harness: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
harness = null
})
const BASH = 'run this: if [[ -f /home/me/.aws/credentials ]]; then cat it; fi'
const TOOL = 'zebrafish appears only in the tool output here'
it('shows the column that matched, not the one that happens to contain brackets', async () => {
harness = await openSessionSearchHarness('ss-snippet-marks')
// Session 1's match is in tool output while its user turn holds a bash test
// expression; session 2 is the same match with no brackets anywhere.
addSyntheticSession(harness.db, { id: 1, text: BASH, toolText: TOOL })
addSyntheticSession(harness.db, { id: 2, text: 'run this script please', toolText: TOOL })
const hits = harness.engine.search({ query: 'zebrafish' }).hits
expect(hits).toHaveLength(2)
for (const hit of hits) {
expect(hit.evidence?.snippet).toContain(
`${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebrafish${SESSION_SEARCH_SNIPPET_MARK_CLOSE}`
)
expect(hit.evidence?.snippet).not.toContain('credentials')
}
})
it('falls back to any column for an identifier-only match, brackets or not', async () => {
// `zebra` reaches this row only through the identifier shadow column, which is
// what column -1 exists for. The user turn holds numpy output, so a bracket
// scan would have stopped at it and shown a column with no match in it.
harness = await openSessionSearchHarness('ss-snippet-marks-fallback')
addSyntheticSession(harness.db, {
id: 1,
text: 'numpy printed [[1, 2], [3, 4]] before the call',
toolText: 'zebra-fish-count = 4'
})
const [hit] = harness.engine.search({ query: 'zebra' }).hits
expect(hit?.evidence?.snippet).toContain(
`${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebra${SESSION_SEARCH_SNIPPET_MARK_CLOSE}`
)
expect(hit?.evidence?.snippet).not.toContain('numpy')
})
it('leaves a transcript’s own brackets in the text it shows', async () => {
// The marks are rewritten from private-use code points at the very end, so a
// row that both matches and contains `[[` keeps its own characters.
harness = await openSessionSearchHarness('ss-snippet-marks-literal')
addSyntheticSession(harness.db, { id: 1, text: `zebrafish ${BASH}` })
const [hit] = harness.engine.search({ query: 'zebrafish' }).hits
expect(hit?.evidence?.snippet).toContain(
`${SESSION_SEARCH_SNIPPET_MARK_OPEN}zebrafish${SESSION_SEARCH_SNIPPET_MARK_CLOSE}`
)
expect(hit?.evidence?.snippet).toContain('[[ -f')
})
it('picks by comparison, so a private-use code point in content cannot pose as a mark', async () => {
// The marks are private-use code points, and a transcript may hold one:
// agent output carries Nerd Font glyphs, which live in the same block. So the
// column is chosen by comparing a marked rendering against an unmarked one,
// not by looking for a mark in the text.
harness = await openSessionSearchHarness('ss-snippet-marks-private-use')
addSyntheticSession(harness.db, {
id: 1,
text: 'the \uE000 glyph a font printed here',
toolText: TOOL
})
const [hit] = harness.engine.search({ query: 'zebrafish' }).hits
expect(hit?.evidence?.snippet).toContain('zebrafish')
expect(hit?.evidence?.snippet).not.toContain('glyph')
})
it('truncates on the last real mark, not on a bracket the transcript wrote', async () => {
// Over the character ceiling the snippet is cut, and it must not cut between
// an open mark and its close. Finding that open mark by searching for `[[`
// stops at the transcript's own bracket instead and throws away everything
// after it.
harness = await openSessionSearchHarness('ss-snippet-marks-truncation')
const long = (letter: string): string =>
Array.from({ length: 5 }, () => `${letter.repeat(55)}/tail`).join(' ')
addSyntheticSession(harness.db, {
id: 1,
text: `zebrafish ${long('p')} [[ ${long('q')}`
})
const snippet = harness.engine.search({ query: 'zebrafish' }).hits[0]?.evidence?.snippet ?? ''
expect(snippet).toContain('[[zebrafish]]')
// The cut is the character ceiling, so the text after the transcript's own
// bracket survives up to it.
expect(snippet).toContain('qqqqq')
})
src/main/ai-vault-search/session-search-snippet.ts
-126
View File
@@ -1,126 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import {
SESSION_SEARCH_SNIPPET_MARK_CLOSE,
SESSION_SEARCH_SNIPPET_MARK_OPEN
} from './session-search-engine-types'
import {
orExpression,
scopedExpression,
type SessionSearchQueryPlan
} from './session-search-query-planner'
import type { SessionSearchScope } from './session-search-engine-types'
// What FTS5 wraps a match in before this module rewrites it to the public
// marks. Private-use code points, and not `[[`, because two different jobs here
// have to tell a mark from content: choosing the column to show, and refusing
// to cut a snippet between an open mark and its close. Transcripts contain
// `[[` — a bash `[[ -f x ]]`, numpy's `[[1, 2]]` — and a mark the content can
// forge makes both of those decisions wrong on real text.
const MARK_OPEN = '\uE000'
const MARK_CLOSE = '\uE001'
const SNIPPET_TOKENS = 12
// Why a ceiling on top of the token count: a transcript chunk can be 8000
// characters with no separator in it, which FTS5 reports as one token, so
// "twelve tokens" is not by itself a bound on what a hit carries.
const SNIPPET_MAX_CHARS = 512
export type SessionSearchSnippet = {
text: string
truncated: boolean
}
export const EMPTY_SNIPPET: SessionSearchSnippet = { text: '', truncated: false }
/**
* The window of one message that shows why it matched.
*
* The expression is the plan's OR form rather than the route's, so a hit found
* through typo repair is marked with the repaired terms it was actually
* retrieved by, and a phrase hit still marks each of its words.
*/
export function sessionSearchSnippet(
db: SyncDatabase,
scope: SessionSearchScope,
rowid: number,
plan: SessionSearchQueryPlan
): SessionSearchSnippet {
// Why: the identifier shadow column is word soup; a hit that also matches in a
// prose column should be shown from there. Column -1 (any column) is the
// fallback for rows that only matched through the shadow column.
//
// The same four for every scope, because the scope is already in the
// expression below. A conversation snippet cannot come out of `tool_text` for
// the reason the search could not: the row has to match
// `{user_text assistant_text}: …` before any of these columns is read, and a
// row that matches under that filter carries its mark in column 0 or 1. A
// second list here would be a guard with nothing left to guard, and the two
// would mask each other's mistakes.
const columns = [0, 1, 2, -1]
// Each column twice: once marked, once with empty marks. Whether a column
// matched is then the difference between two renderings of the same text,
// which content cannot forge — searching the marked one for a mark reads a
// transcript's own `[[` as a highlight and shows a column that matched
// nothing.
const select = columns
.flatMap((column, index) => [
`snippet(messages_fts, ${column}, '${MARK_OPEN}', '${MARK_CLOSE}', '…', ${SNIPPET_TOKENS}) AS c${index}`,
`snippet(messages_fts, ${column}, '', '', '…', ${SNIPPET_TOKENS}) AS p${index}`
])
.join(', ')
try {
// Why the subselect: a bound `rowid = ?` or `rowid IN (?)` next to MATCH is
// silently ignored by the FTS5 planner, which then returns the first match
// in the table. Why the join to `sessions`: retrieval proved this rowid
// belonged to a live session, but a purge can commit between that statement
// and this one, and a message row outlives its session row until the drain
// reaches it. INNER, never LEFT — this is the last read before content is
// returned to a caller.
const row = db
.prepare(
`SELECT ${select} FROM messages_fts
JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id
WHERE messages_fts MATCH ? AND messages_fts.rowid IN (SELECT ?)`
)
.get(scopedExpression(scope, orExpression(plan.terms)), rowid) as
| Record<string, string>
| undefined
if (!row) {
return EMPTY_SNIPPET
}
// A snippet with nothing highlighted tells the user nothing; omit it.
const marked = columns
.map((_column, index) => row[`c${index}`])
.find((text, index) => text !== undefined && text !== row[`p${index}`])
return marked === undefined ? EMPTY_SNIPPET : publicMarks(truncateSnippet(marked))
} catch {
return EMPTY_SNIPPET
}
}
/** The internal marks, swapped for the ones a caller sees, once and at the end. */
function publicMarks(snippet: SessionSearchSnippet): SessionSearchSnippet {
return {
...snippet,
text: snippet.text
.replaceAll(MARK_OPEN, SESSION_SEARCH_SNIPPET_MARK_OPEN)
.replaceAll(MARK_CLOSE, SESSION_SEARCH_SNIPPET_MARK_CLOSE)
}
}
/** Cut on a code-point boundary, and never between a mark and its close. */
export function truncateSnippet(text: string): SessionSearchSnippet {
if (text.length <= SNIPPET_MAX_CHARS) {
return { text, truncated: false }
}
const points = [...text]
if (points.length <= SNIPPET_MAX_CHARS) {
return { text, truncated: false }
}
const cut = points.slice(0, SNIPPET_MAX_CHARS).join('')
const opened = cut.lastIndexOf(MARK_OPEN)
// An open mark with no close hands the renderer something it can never close.
const balanced = opened !== -1 && !cut.includes(MARK_CLOSE, opened) ? cut.slice(0, opened) : cut
return { text: balanced, truncated: true }
}
src/main/ai-vault-search/session-search-source-presence.ts
-40
View File
@@ -1,40 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import type { SessionSearchSourcePresence } from './session-search-engine-types'
/**
* Where each session's source stands, read from the index's own `files` table.
*
* Why not a stat: a search page of 20 hits would be 20 filesystem round trips
* on the query path, and on an SSH or WSL host each one can block for as long
* as the connection takes to answer — the reviewer's F11. The index already
* records what discovery last proved about every file it read, so the query
* path reads that instead of asking the disk again.
*
* The vocabulary is deliberately short of `missing`. A row here means the index
* holds a live file record for the session, which is `present`. No row means
* this read cannot tell whether the source is gone or merely unrecorded, and
* loss of contact is never evidence of absence
* (docs/reference/ssh-execution-boundary.md), so it is `unverifiable`. Proving
* a deletion is the indexer's job and it retires the session's rows outright.
*/
export function sessionSourcePresence(
db: SyncDatabase,
sessionRowIds: readonly number[]
): Map<number, SessionSearchSourcePresence> {
const presence = new Map<number, SessionSearchSourcePresence>(
sessionRowIds.map((id) => [id, 'unverifiable' as const])
)
if (sessionRowIds.length === 0) {
return presence
}
const rows = db
.prepare(
`SELECT DISTINCT session_row_id FROM files
WHERE session_row_id IN (${sessionRowIds.map(() => '?').join(',')})`
)
.all(...sessionRowIds) as { session_row_id: number }[]
for (const row of rows) {
presence.set(row.session_row_id, 'present')
}
return presence
}
src/main/ai-vault-search/session-search-store-is-memory.test.ts
+265
View File
@@ -0,0 +1,265 @@
import { chmod, rm, utimes } from 'node:fs/promises'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
import type SyncDatabase from '../sqlite/sync-database'
import { SessionSearchIndexer } from './session-search-indexer'
import {
FakeSessionSearchClock,
openSessionSearchIndexerHarness,
writeClaudeTranscript,
type SessionSearchIndexerHarness
} from './session-search-indexer-test-fixture'
/*
* S1-S5: the store is the only memory.
*
* Every question the indexer answers between passes -- what is owed a read,
* what has failed and how often, what it holds and therefore what may have been
* deleted, what to report -- is a row in the `files` table. These tests check
* that from outside the object: a second connection, hand-written SQL, and the
* clock. Two things outlive a pass and are not rows, and both are named here:
* the timer, and one bit per root for the retirement walk's grace.
*/
const INTERVAL_MS = 20_000
const CAN_DENY_READ = process.platform !== 'win32' && process.getuid?.() !== 0
const FIRST = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
const SECOND = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff'
const THIRD = 'cccccccc-dddd-4eee-8fff-000000000000'
let harness: SessionSearchIndexerHarness
let clock: FakeSessionSearchClock
let indexer: SessionSearchIndexer | null
let errors: unknown[]
beforeEach(async () => {
resetSessionParseCacheForTests()
resetTranscriptConsumersForTests()
errors = []
clock = new FakeSessionSearchClock()
harness = await openSessionSearchIndexerHarness('ss-memory')
indexer = null
})
afterEach(async () => {
indexer?.close()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await harness.cleanup()
})
function newIndexer(
overrides: Partial<ConstructorParameters<typeof SessionSearchIndexer>[0]> = {}
): SessionSearchIndexer {
indexer = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
reconcileIntervalMs: INTERVAL_MS,
onError: (error) => errors.push(error),
...overrides
})
return indexer
}
function transcriptPath(name: string): string {
return join(harness.claudeProjectDir, `${name}.jsonl`)
}
async function nextCycle(): Promise<void> {
clock.advance(INTERVAL_MS)
await indexer?.settled()
}
/** The whole `files` table as a second connection sees it, ordered for comparison. */
function fileTable(): unknown[] {
return harness.read((db: SyncDatabase) =>
db
.prepare(
`SELECT path, dev, ino, byte_offset, mtime_ms, size_bytes, session_row_id,
state, fail_count, failed_mtime_ms
FROM files ORDER BY path`
)
.all()
)
}
// S1. The status is a query. A counter kept beside the rows is what needs a
// rule about when to reset, and every such rule this feature grew was wrong.
it('S1: reports exactly what a hand-written query over the rows reports', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['one'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['two'], SECOND)
await newIndexer().start()
const bySql = (): Record<string, number> =>
Object.fromEntries(
(
harness.read((db: SyncDatabase) =>
db.prepare('SELECT state, count(*) AS n FROM files GROUP BY state').all()
) as { state: string; n: number }[]
).map((row) => [row.state, Number(row.n)])
)
const reported = indexer?.status()
const counted = bySql()
expect(reported?.filesIndexed).toBe(counted.current ?? 0)
expect(reported?.filesDue).toBe(counted.due ?? 0)
expect(reported?.filesFailed).toBe(counted.failed ?? 0)
expect(reported?.filesIndexed).toBe(2)
// And it stays a query: delete a row behind the indexer's back and the very
// next call reports the table, not a number it remembered.
harness.write((db: SyncDatabase) =>
db.prepare('DELETE FROM files WHERE path = ?').run(transcriptPath(FIRST))
)
expect(indexer?.status().filesIndexed).toBe(1)
})
// S2. A deletion is proven by comparing the rows against what discovery
// returned, so the moment it happened does not matter. Every boundary a pass
// has is a moment a file can go.
it('S2: retires a file deleted right after the opening sweep', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['going'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['staying'], SECOND)
await newIndexer().start()
await rm(transcriptPath(FIRST))
await nextCycle()
expect(fileTable()).toHaveLength(1)
})
it('S2: retires a file deleted right after a cycle', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['going'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['staying'], SECOND)
await newIndexer().start()
await nextCycle()
await rm(transcriptPath(FIRST))
await nextCycle()
expect(fileTable()).toHaveLength(1)
})
it('S2: retires a file deleted right after a periodic sweep', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['going'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['staying'], SECOND)
await newIndexer({ fullSweepEveryCycles: 2 }).start()
await nextCycle()
await nextCycle()
// The third pass is the periodic sweep; the file goes the moment it ends.
await nextCycle()
await rm(transcriptPath(FIRST))
await nextCycle()
expect(fileTable()).toHaveLength(1)
})
it('S2: retires a file deleted while a pass was out of time', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['going'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['staying'], SECOND)
await writeClaudeTranscript(transcriptPath(THIRD), ['also staying'], THIRD)
// One transcript a pass: the opening sweep leaves two of the three unread.
clock.costPerNowMs = 1_000
await newIndexer({ passDeadlineMs: 1_000 }).start()
expect(fileTable()).toHaveLength(1)
await rm(transcriptPath(FIRST))
await nextCycle()
await nextCycle()
// Read what it could, and proved the deletion in the same pass it was still
// catching up in: retirement is not what the deadline bounds.
expect((fileTable() as { path: string }[]).map((row) => row.path)).not.toContain(
transcriptPath(FIRST)
)
})
// S3. The stat is the whole retry policy: a file that fails at one stat stops
// being read, and only a change to that stat starts it again.
it.skipIf(!CAN_DENY_READ)(
'S3: stops reading a file that fails three times at one stat',
async () => {
const path = transcriptPath(FIRST)
await writeClaudeTranscript(path, ['behind the wrong mode bits'], FIRST)
await chmod(path, 0o000)
try {
await newIndexer().start()
for (let cycle = 0; cycle < 4; cycle++) {
await nextCycle()
}
const row = harness.read((db: SyncDatabase) =>
db.prepare('SELECT state, fail_count AS failCount FROM files WHERE path = ?').get(path)
) as { state: string; failCount: number }
// Three, not four and not seven: the pass after the third costs nothing.
expect(row).toEqual({ state: 'failed', failCount: 3 })
expect(indexer?.status()).toMatchObject({ filesFailed: 1, phase: 'degraded' })
// Only the stat releases it.
await chmod(path, 0o644)
const later = new Date(Date.now() + 60_000)
await utimes(path, later, later)
await nextCycle()
expect(indexer?.status()).toMatchObject({ filesIndexed: 1, filesFailed: 0 })
} finally {
await chmod(path, 0o644)
}
}
)
// S4. Two passes over an unchanged filesystem leave the table byte for byte as
// they found it. Anything that drifted would be state the rows do not hold.
it('S4: leaves the file table identical across passes with no change on disk', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['one'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['two'], SECOND)
await newIndexer({ fullSweepEveryCycles: 2 }).start()
const afterSweep = fileTable()
await nextCycle()
expect(fileTable()).toEqual(afterSweep)
await nextCycle()
expect(fileTable()).toEqual(afterSweep)
// Including across the periodic sweep, which reads the same rows again.
await nextCycle()
expect(fileTable()).toEqual(afterSweep)
expect(errors).toEqual([])
})
// S5. Nothing a close interrupts needs repairing: the next instance reads the
// rows as they stand and decides from them alone.
it('S5: leaves the store consistent when a close interrupts a pass', async () => {
await writeClaudeTranscript(transcriptPath(FIRST), ['one'], FIRST)
await writeClaudeTranscript(transcriptPath(SECOND), ['two'], SECOND)
await writeClaudeTranscript(transcriptPath(THIRD), ['three'], THIRD)
newIndexer()
let closed = false
clock.onNow = () => {
if (closed || fileTable().length === 0) {
return
}
closed = true
indexer?.close()
}
await indexer?.start()
await indexer?.settled()
clock.onNow = null
const interrupted = fileTable()
expect(interrupted.length).toBeGreaterThan(0)
expect(interrupted.length).toBeLessThan(3)
expect(errors).toEqual([])
// A new instance over the same database: no repair pass, no recovery, just
// the rows and what they say is owed.
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
await newIndexer().start()
expect(indexer?.status()).toMatchObject({ filesIndexed: 3, filesDue: 0, filesFailed: 0 })
})
src/main/ai-vault-search/session-search-store.ts
+130 -69
View File
@@ -13,10 +13,36 @@ import {
import { deleteExpiredSearchFiles, drainOrphanedMessages } from './session-search-retention-delete'
import { openSessionSearchDatabase } from './session-search-schema'
// A paused store keeps recording what it declined, so the set needs a ceiling.
// Above it the oldest record goes and the drop is counted, because a re-read set
// that silently forgets is worse than one that says it is incomplete.
export const STALE_PATH_LIMIT = 20_000
/**
* What a row still owes a reader.
*
* `current`: the rows match the file at the stat this row records.
* `due`: the index is behind on a span it cannot reach by appending, so the
* next pass must read the file whole.
* `failed`: the last read did not commit; `failCount` and `failedMtimeMs` are
* what stop it being retried for ever.
*/
export type SessionSearchFileState = 'current' | 'due' | 'failed'
/**
* One row of the index's own file table.
*
* This is the indexer's whole memory between passes: what it holds, at what
* stat, and what each row still owes. Nothing it decides is answered from
* anywhere else, which is why a second connection can check its status.
*/
export type SessionSearchFileRow = {
path: string
identity: SessionSearchFileIdentity
mtimeMs: number
sizeBytes: number | null
state: SessionSearchFileState
failCount: number
failedMtimeMs: number | null
}
/** How many rows are in each state; the whole of the indexer's progress report. */
export type SessionSearchStateCounts = { current: number; due: number; failed: number }
/**
* Owns the index database. PR 2 scope: the write half only — the transcript
@@ -27,12 +53,7 @@ export class SessionSearchStore {
private readonly db: SyncDatabase
private readonly writer: SessionSearchIndexWriter
private closed = false
private acceptingWrites = true
private retentionCutoffMs: number | null = null
// Files this index knows it is behind on. Filled by a declined or abandoned
// read; PR 3's indexer drains it. Nothing here schedules the re-read.
private readonly stale = new Map<string, SessionFileCandidate>()
private droppedStalePaths = 0
// One drain at a time. A replace that commits while one is running asks for
// another pass rather than starting a second walk of the same rows.
private draining = false
@@ -104,23 +125,26 @@ export class SessionSearchStore {
return this.db
}
setAcceptingWrites(accept: boolean): void {
this.acceptingWrites = accept
}
/** The oldest transcript mtime worth indexing; PR 3 derives it from the retention setting. */
setRetentionCutoffMs(cutoffMs: number | null): void {
this.retentionCutoffMs = cutoffMs
}
/** Whether this candidate is new enough to be worth holding rows for at all. */
private withinRetention(candidate: SessionFileCandidate): boolean {
return this.retentionCutoffMs === null || candidate.file.mtimeMs >= this.retentionCutoffMs
/** The cutoff a caller's own decide step compares a candidate's mtime against. */
get retentionCutoff(): number | null {
return this.retentionCutoffMs
}
/** Whether a write for this candidate may start right now. */
acceptsCandidate(candidate: SessionFileCandidate): boolean {
return !this.closed && this.acceptingWrites && this.withinRetention(candidate)
/**
* Whether this candidate is new enough to hold rows for.
*
* Enforced here as well as in the indexer's decide step, and not only there:
* the consumer observes every read the session list makes, not only the ones
* the index asked for, so a sidebar scan of a transcript outside the window
* would otherwise index rows the next purge deletes again.
*/
private withinRetention(candidate: SessionFileCandidate): boolean {
return this.retentionCutoffMs === null || candidate.file.mtimeMs >= this.retentionCutoffMs
}
indexedFile(path: string, identity: SessionSearchFileIdentity): SessionSearchIndexedFile | null {
@@ -139,7 +163,7 @@ export class SessionSearchStore {
previousByteOffset: number,
identity?: () => TranscriptSessionIdentity | null
): SessionSearchFileWrite | null {
if (!this.acceptsCandidate(candidate)) {
if (this.closed || !this.withinRetention(candidate)) {
return null
}
try {
@@ -150,10 +174,14 @@ export class SessionSearchStore {
}
}
/**
* A read that landed. Written after the commit rather than inside it: the
* transaction owns the rows and the cursor, and a crash between the two
* leaves a row that says `failed` over content that is in fact current, which
* the next pass fixes by reading a file it did not have to.
*/
writeCommitted(candidate: SessionFileCandidate): void {
// Why: a list scan queues every file the backfill has not reached yet; once
// one lands, a later pass must not re-read the whole queue.
this.stale.delete(candidate.file.path)
this.setFileState(candidate.file.path, 'current')
}
reportWriteFailure(error: unknown): void {
@@ -161,55 +189,89 @@ export class SessionSearchStore {
}
/**
* Records a file whose content the index is behind on, for a later whole
* re-read. Recorded while paused too: a pause is exactly the window in which
* reads are declined, so refusing to remember them would lose every file the
* pause covered.
*/
markStale(candidate: SessionFileCandidate): void {
if (this.closed || !this.withinRetention(candidate)) {
return
}
// Re-inserting moves the path to the end, so the oldest record is the one
// dropped when a long pause overruns the bound.
this.stale.delete(candidate.file.path)
this.stale.set(candidate.file.path, candidate)
while (this.stale.size > STALE_PATH_LIMIT) {
const oldest = this.stale.keys().next()
if (oldest.done) {
break
}
this.stale.delete(oldest.value)
this.droppedStalePaths += 1
}
}
/**
* Files the index knew it was behind on and could not keep a record of. A
* non-zero count means the re-read set is incomplete, so coverage cannot be
* reported as whole until a full pass runs.
*/
get droppedPendingFileCount(): number {
return this.droppedStalePaths
}
/**
* Hands the re-read set to its scheduler and clears it.
* Every row this index holds. The candidate list for retirement and the whole
* of the status, read in one query so that no pass has to carry either.
*
* These paths are behind, not merely dirty: the index declined their last read
* because it covered a span the index never saw. Re-dispatching a scan is not
* enough on its own, because the reader picks `append` from the session list's
* resume point and the consumer will decline again. The caller must pass each
* path to `requestWholeTranscriptRead` first.
* The cursor is deliberately not here: whether a row can be continued is
* `indexedFile`'s question, and one spelling of the half-written sentinel is
* enough.
*/
takeStale(): SessionFileCandidate[] {
const candidates = [...this.stale.values()]
this.stale.clear()
return candidates
files(): SessionSearchFileRow[] {
return (
this.db
.prepare(
`SELECT path, dev, ino, mtime_ms AS mtimeMs, size_bytes AS sizeBytes,
state, fail_count AS failCount, failed_mtime_ms AS failedMtimeMs
FROM files`
)
.all() as (Omit<SessionSearchFileRow, 'identity'> & {
dev: number | null
ino: number | null
})[]
).map((row) => ({
path: row.path,
identity:
typeof row.dev === 'number' && typeof row.ino === 'number'
? { dev: row.dev, ino: row.ino }
: null,
mtimeMs: row.mtimeMs,
sizeBytes: row.sizeBytes,
state: row.state,
failCount: row.failCount,
failedMtimeMs: row.failedMtimeMs
}))
}
get pendingFileCount(): number {
return this.stale.size
/**
* Moves a row's read state.
*
* `failed` also counts the failure and records the stat it happened at, which
* is what lets the next pass tell "this file has never worked" from "this
* file has changed since it last failed". A path with no row is a no-op: the
* next pass reads it because the index holds nothing for it.
*/
setFileState(path: string, state: SessionSearchFileState, atMtimeMs?: number): void {
try {
if (state === 'failed') {
// Inserted when there is no row, because the common unreadable file is
// one the index never managed to hold: a transcript behind the wrong
// mode bits fails on its very first read, and with nowhere to write the
// count it would be read again on every pass for the life of the
// process. The cursor is zero and there is no session, which is what
// "the index holds nothing for this file" already looks like.
this.db
.prepare(
`INSERT INTO files(path, byte_offset, mtime_ms, state, fail_count, failed_mtime_ms)
VALUES (?, 0, ?, 'failed', 1, ?)
ON CONFLICT(path) DO UPDATE SET
state = 'failed',
fail_count = files.fail_count + 1,
failed_mtime_ms = excluded.failed_mtime_ms`
)
.run(path, atMtimeMs ?? 0, atMtimeMs ?? null)
return
}
this.db
.prepare(
'UPDATE files SET state = ?, fail_count = 0, failed_mtime_ms = NULL WHERE path = ?'
)
.run(state, path)
} catch (error) {
this.onError(error)
}
}
/** Rows per state. The status is this query and the pass's own degraded roots. */
stateCounts(): SessionSearchStateCounts {
const rows = this.db.prepare('SELECT state, count(*) AS n FROM files GROUP BY state').all() as {
state: SessionSearchFileState
n: number
}[]
const counts: SessionSearchStateCounts = { current: 0, due: 0, failed: 0 }
for (const row of rows) {
counts[row.state] = Number(row.n)
}
return counts
}
/**
@@ -218,7 +280,6 @@ export class SessionSearchStore {
* (docs/reference/ssh-execution-boundary.md).
*/
removeFile(path: string): void {
this.stale.delete(path)
try {
this.writer.removeFile(path)
} catch (error) {
src/main/ai-vault-search/session-search-synthetic-sources.ts
+56
View File
@@ -0,0 +1,56 @@
import type { AiVaultScanIssue } from '../../shared/ai-vault-types'
import { splitOpenCodeSqliteCandidate } from '../ai-vault/session-scanner-opencode-sqlite-paths'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
/**
* A row whose path names a container and an entry inside it rather than a file
* of its own. OpenCode's SQLite sessions are the one shape today
* (`<opencode.db>#<sessionId>`), which is why this reads through that source's
* own splitter rather than reinventing the encoding.
*/
export type SessionSearchSyntheticSource = { container: string; id: string }
export function splitSyntheticSessionSource(path: string): SessionSearchSyntheticSource | null {
const openCode = splitOpenCodeSqliteCandidate(path)
return openCode ? { container: openCode.dbPath, id: openCode.sessionId } : null
}
/**
* Which containers a pass enumerated in full, and every id each of them held.
*
* This is the synthetic equivalent of a directory listing, and it has to meet
* the same bar before the retirement walk may prove anything from it:
*
* - **Exhaustive.** Only a sweep enumerates without a per-agent limit. A cycle
* asks for the newest N, so an id it did not return may simply be the N+1th.
* Callers that are not a census do not build this at all.
* - **Successful.** A container a scan issue names could not be read, and a
* read that failed returns no ids rather than an error the walk can see. A
* named container is left out, so its rows stay unverifiable.
* - **Non-empty.** A container that returned nothing is not evidence that it
* holds nothing: a database whose schema this scanner no longer recognises
* returns an empty list with no error at all, and believing it would retire
* every session in one pass. The cost is one stale row per container whose
* last entry the user deletes, until the container gains an entry or goes.
*/
export function sessionSearchEnumeratedContainers(
candidates: readonly SessionFileCandidate[],
issues: readonly AiVaultScanIssue[]
): Map<string, Set<string>> {
const containers = new Map<string, Set<string>>()
for (const candidate of candidates) {
const synthetic = splitSyntheticSessionSource(candidate.file.path)
if (!synthetic) {
continue
}
const ids = containers.get(synthetic.container) ?? new Set<string>()
ids.add(synthetic.id)
containers.set(synthetic.container, ids)
}
for (const issue of issues) {
if (issue.kind !== 'notice') {
containers.delete(issue.path)
}
}
return containers
}
src/main/ai-vault-search/session-search-typo-policy.test.ts
-80
View File
@@ -1,80 +0,0 @@
import { describe, expect, it } from 'vitest'
import type SyncDatabase from '../sqlite/sync-database'
import { openSessionSearchIndexFile } from './session-search-index-test-fixture'
import { ensureSessionSearchQuerySchema } from './session-search-query-schema'
import { SessionSearchTypoRepair } from './session-search-typo-repair'
/** A session row the planted messages below hang off, so a repair can see them. */
function addSession(db: SyncDatabase, id: number): void {
db.prepare(
`INSERT INTO sessions(id,agent,session_id,file_path,title,resume_command)
VALUES (?, 'claude', ?, '/synthetic/fixture', 'typo fixture', '')`
).run(id, String(id))
}
function addTerm(db: SyncDatabase, sessionRowId: number, term: string): void {
const rowid = db
.prepare("INSERT INTO messages(session_row_id, role) VALUES (?, 'user')")
.run(sessionRowId).lastInsertRowid
db.prepare('INSERT INTO messages_fts(rowid, user_text) VALUES (?, ?)').run(Number(rowid), term)
}
describe('typo repair policy', () => {
it.each([
{ input: 'coalesces', candidate: 'coalesced', copies: 2, exact: true, expected: null },
{ input: 'coalescs', candidate: 'coalesces', copies: 1, exact: false, expected: null },
{ input: 'coalescs', candidate: 'coalesces', copies: 2, exact: false, expected: 'coalesces' },
{ input: 'café', candidate: 'cafe', copies: 1, exact: false, expected: null },
{ input: 'car', candidate: 'cars', copies: 2, exact: false, expected: null },
{ input: 'calm', candidate: 'clam', copies: 2, exact: false, expected: null }
])(
'repairs $input to $expected with $copies postings (exact=$exact)',
async ({ input, candidate, copies, exact, expected }) => {
const index = await openSessionSearchIndexFile('ss-typo-policy')
try {
ensureSessionSearchQuerySchema(index.db)
addSession(index.db, 1)
for (let i = 0; i < copies; i++) {
addTerm(index.db, 1, candidate)
}
if (exact) {
addTerm(index.db, 1, input)
}
expect(new SessionSearchTypoRepair(index.db).correct(input, 'all')).toBe(expected)
} finally {
await index.close()
}
}
)
// A purge cuts a session loose in one transaction and reclaims its rows over
// many, so the vocabulary can still list a term whose only rows nothing can
// reach. Abandoning the prefix at that term would lose a repair the rest of
// the index can already serve.
it('falls through to the best candidate a reader can still reach', async () => {
const index = await openSessionSearchIndexFile('ss-typo-orphaned')
try {
const { db } = index
ensureSessionSearchQuerySchema(db)
addSession(db, 1)
// `coalesces` scores higher against `coalescs` than `coalesced` does, and
// shares its prefix, so only the fall-through can reach the reachable one.
// Session 2 is never created: these rows are what an unfinished purge
// leaves behind, and the vocabulary counts them all the same.
for (const [term, session] of [
['coalesces', 2],
['coalesces', 2],
['coalesced', 1],
['coalesced', 1]
] as const) {
addTerm(db, session, term)
}
expect(db.prepare("SELECT doc FROM messages_vocab WHERE term='coalesces'").get()).toEqual({
doc: 2
})
expect(new SessionSearchTypoRepair(db).correct('coalescs', 'all')).toBe('coalesced')
} finally {
await index.close()
}
})
})
src/main/ai-vault-search/session-search-typo-repair.ts
-163
View File
@@ -1,163 +0,0 @@
import type SyncDatabase from '../sqlite/sync-database'
import type { SessionSearchScope } from './session-search-engine-types'
import { quoteFtsTerm, scopedExpression } from './session-search-query-planner'
// Why: a query term with zero postings is usually a typo. The index's own
// vocabulary (fts5vocab) is the dictionary, so repair needs no model and can
// never suggest a word the index does not contain. Measured MRR 0.553 → 0.566.
const MIN_TERM_LENGTH = 4
const MAX_TERM_LENGTH = 40
const LENGTH_SLACK = 2
const MIN_DOC_FREQUENCY = 2
const MIN_SIMILARITY = 0.82
const MAX_CANDIDATES = 4000
// Candidates counted against live rows per prefix before giving up on it. Only
// reached for a term the scope has no posting for, which is the rare case.
const MAX_VISIBILITY_PROBES = 8
// How far a live count walks before it stops caring. It exists to break ties
// between candidates of equal similarity, and the difference between a term in
// sixty-four rows and one in six thousand does not change which is the better
// repair — but reading either in full would.
const MAX_COUNTED_ROWS = 64
// Longest common subsequence length; the indel distance is len(a)+len(b)-2·LCS.
function commonSubsequenceLength(a: string, b: string): number {
let previous = Array.from<number>({ length: b.length + 1 }).fill(0)
let current = Array.from<number>({ length: b.length + 1 }).fill(0)
for (let i = 1; i <= a.length; i += 1) {
for (let j = 1; j <= b.length; j += 1) {
current[j] =
a.charCodeAt(i - 1) === b.charCodeAt(j - 1)
? previous[j - 1] + 1
: Math.max(previous[j], current[j - 1])
}
;[previous, current] = [current, previous]
}
return previous[b.length]
}
/** Normalized indel similarity in [0, 1], the scale rapidfuzz's `fuzz.ratio` uses. */
function similarity(a: string, b: string): number {
const total = a.length + b.length
return total === 0 ? 1 : (2 * commonSubsequenceLength(a, b)) / total
}
/**
* Spelling repair over the index's own vocabulary.
*
* The vocabulary proposes and a scoped count disposes. `messages_vocab` is a
* view over the whole FTS b-tree: it has no column filter, because fts5vocab is
* per table, and it counts rows whose session a purge already cut loose. So
* every decision that reaches the plan — whether a term is already spelled
* right, whether a candidate is eligible, and which of two equally close
* candidates wins — is taken from a `messages_fts MATCH` under the same column
* filter retrieval uses, joined to `sessions`.
*
* That is not tidiness. Reading the vocabulary directly made the repair depend
* on rows the search could never return: tool output suppressed a
* conversation-scope repair and supplied suggestions the scope would never
* show, and retention's orphan drain silently changed which word a query was
* repaired to.
*
* The cost is one bounded count per candidate examined, at most
* `MAX_VISIBILITY_PROBES` per prefix, and only for a term the scope has no
* posting for. See docs/reference/agent-session-search-query-tuning.md.
*/
export class SessionSearchTypoRepair {
private readonly liveRows: ReturnType<SyncDatabase['prepare']>
private readonly candidatesByPrefix: ReturnType<SyncDatabase['prepare']>
constructor(db: SyncDatabase) {
this.liveRows = db.prepare(
`SELECT count(*) AS rows FROM (
SELECT m.id FROM messages_fts
JOIN messages m ON m.id = messages_fts.rowid
JOIN sessions s ON s.id = m.session_row_id
WHERE messages_fts MATCH ? LIMIT ${MAX_COUNTED_ROWS})`
)
// fts5vocab is ordered by term, so a prefix range plus a length band is a
// bounded scan and no sort. Ordered by term rather than by `doc`: the
// ordering decides which candidates survive the limit, and `doc` counts
// rows no reader can see, so the drain reclaiming them moved the cut.
this.candidatesByPrefix = db.prepare(
`SELECT term FROM messages_vocab
WHERE term >= ? AND term < ? AND length(term) BETWEEN ? AND ?
ORDER BY term LIMIT ?`
)
}
/** Live rows carrying this term inside `scope`, counted no further than it matters. */
private countRows(term: string, scope: SessionSearchScope): number {
const row = this.liveRows.get(scopedExpression(scope, quoteFtsTerm(term))) as { rows: number }
return row.rows
}
/** Whether a live row inside `scope` holds this term. */
hasPostings(term: string, scope: SessionSearchScope): boolean {
return this.countRows(term, scope) > 0
}
/** Returns the closest indexed term, or null when `term` exists or nothing is close enough. */
correct(term: string, scope: SessionSearchScope): string | null {
const lowered = term.toLowerCase()
if (lowered.length < MIN_TERM_LENGTH || lowered.length > MAX_TERM_LENGTH) {
return null
}
if (this.hasPostings(lowered, scope)) {
return null
}
// Two-letter prefix first (a typo rarely hits both), then the transposed
// pair, then the bare first letter as the wide fallback.
const prefixes = [lowered.slice(0, 2), lowered[1] + lowered[0], lowered[0]]
for (const prefix of prefixes) {
const best = this.bestVisible(lowered, prefix, scope)
if (best) {
return best
}
}
return null
}
/**
* The closest candidate at `prefix` that this scope can actually answer with.
*
* Ranking is pure CPU, so the walk is bounded rather than the count: the
* closest term can be one the scope never shows, and abandoning the prefix
* there would lose a repair the rest of the index can serve. Ties on
* similarity go to the more common word, which is the same prior the
* vocabulary's `doc` used to supply — counted live here so the answer does
* not move when a purge reclaims rows nothing could reach.
*/
private bestVisible(lowered: string, prefix: string, scope: SessionSearchScope): string | null {
const counted = this.ranked(lowered, prefix)
.slice(0, MAX_VISIBILITY_PROBES)
.map((candidate) => ({ ...candidate, rows: this.countRows(candidate.term, scope) }))
.filter((candidate) => candidate.rows >= MIN_DOC_FREQUENCY)
if (counted.length === 0) {
return null
}
// Already sorted by similarity; a stable sort keeps that and orders the ties.
return counted.sort((left, right) => right.score - left.score || right.rows - left.rows)[0]!
.term
}
/** Candidates similar enough to be a repair, closest first. */
private ranked(lowered: string, prefix: string): { term: string; score: number }[] {
return this.candidates(prefix, lowered.length)
.map((row) => ({ term: row.term, score: similarity(lowered, row.term) }))
.filter((candidate) => candidate.score >= MIN_SIMILARITY)
.sort((left, right) => right.score - left.score || (left.term < right.term ? -1 : 1))
}
private candidates(prefix: string, length: number): { term: string }[] {
const last = prefix.charCodeAt(prefix.length - 1)
const upper = prefix.slice(0, -1) + String.fromCharCode(last + 1)
return this.candidatesByPrefix.all(
prefix,
upper,
Math.max(MIN_TERM_LENGTH - 1, length - LENGTH_SLACK),
length + LENGTH_SLACK,
MAX_CANDIDATES
) as { term: string }[]
}
}
src/main/ai-vault-search/session-search-typo-scope.test.ts
-71
View File
@@ -1,71 +0,0 @@
import { afterEach, expect, it } from 'vitest'
import {
addSyntheticSession,
openSessionSearchHarness,
type SessionSearchHarness
} from './session-search-engine-test-fixture'
// Typo repair used to read `messages_vocab` and probe `messages_fts` with no
// column filter, so tool output decided whether a conversation-scoped query was
// repaired — in both directions. A tool row carrying the misspelling made the
// query look correctly spelled and suppressed the repair; a tool row carrying a
// rare word offered it as the suggestion, naming in `repairedTerms` a string
// from a column the scope will never show.
let harness: SessionSearchHarness | null = null
let control: SessionSearchHarness | null = null
afterEach(async () => {
await harness?.close()
await control?.close()
harness = null
control = null
})
it('repairs a conversation query the same way with or without a tool row', async () => {
harness = await openSessionSearchHarness('ss-typo-scope-suppress')
addSyntheticSession(harness.db, { id: 1, text: 'we changed resolveTerminalPath today', rows: 2 })
// A second session whose tool output happens to contain the misspelling.
addSyntheticSession(harness.db, {
id: 2,
text: 'ran the linter',
toolText: 'warning: unknown symbol resolveterminalpth in build log',
rows: 2,
role: 'assistant'
})
// The same index without that one tool row.
control = await openSessionSearchHarness('ss-typo-scope-control')
addSyntheticSession(control.db, { id: 1, text: 'we changed resolveTerminalPath today', rows: 2 })
addSyntheticSession(control.db, { id: 2, text: 'ran the linter' })
const request = { query: 'resolveterminalpth', scope: 'conversation' } as const
const withTool = harness.engine.search(request)
const clean = control.engine.search(request)
expect(clean.planner.repairedTerms).toEqual(['resolveterminalpath'])
expect(clean.hits.map((hit) => hit.sessionId)).toEqual(['1'])
expect(withTool.planner.repairedTerms).toEqual(clean.planner.repairedTerms)
expect(withTool.hits.map((hit) => hit.sessionId)).toEqual(clean.hits.map((hit) => hit.sessionId))
})
it('never repairs a conversation query onto a word only tool output holds', async () => {
harness = await openSessionSearchHarness('ss-typo-scope-leak')
addSyntheticSession(harness.db, {
id: 1,
text: 'ran the deploy',
toolText: 'AWS_SESSION_TOKEN=quicksilverfox expired',
rows: 2,
role: 'assistant'
})
addSyntheticSession(harness.db, { id: 2, text: 'ordinary prose about nothing' })
const narrowed = harness.engine.search({ query: 'quicksilverfx', scope: 'conversation' })
expect(narrowed.planner.repairedTerms).toBeUndefined()
expect(narrowed.hits).toEqual([])
// The same query over the whole corpus still finds it, which is the scope
// doing its job rather than the repair being broken.
const wide = harness.engine.search({ query: 'quicksilverfx', scope: 'all' })
expect(wide.planner.repairedTerms).toEqual(['quicksilverfox'])
expect(wide.hits.map((hit) => hit.sessionId)).toEqual(['1'])
})
src/main/ai-vault-search/session-search-work-loop.ts
+87
View File
@@ -0,0 +1,87 @@
import type { SessionSearchClock, SessionSearchTimerHandle } from './session-search-clock'
export type SessionSearchWorkLoopOptions = {
clock: SessionSearchClock
intervalMs: number
/** A task that threw for a reason other than its own abort. */
onFailure: (error: unknown) => void
}
/**
* Runs the indexer's passes one at a time, on an interval, until it is closed.
*
* Separate from the indexer because it is the part with no opinion about
* transcripts: a task chain that never overlaps itself, a timer that only ever
* has one pending tick, and a close that cancels both. Arming inside the chain
* rather than beside it is what makes `settled` mean "everything queued so far
* has finished, including the re-arm", which is what a fake-clock test needs.
*/
export class SessionSearchWorkLoop {
private timer: SessionSearchTimerHandle | null = null
private controller: AbortController | null = null
private chain: Promise<void> = Promise.resolve()
private closed = false
constructor(private readonly options: SessionSearchWorkLoopOptions) {}
/** Everything queued so far. Never rejects: a task's failure is reported, not thrown. */
get settled(): Promise<void> {
return this.chain
}
/** Queues `work` behind whatever is running, then re-arms the interval. */
queue(work: (signal: AbortSignal) => Promise<void>, tick: () => void): Promise<void> {
const chained = this.chain
.then(
() => this.run(work),
() => this.run(work)
)
.then(() => this.arm(tick))
this.chain = chained
return chained
}
/**
* Stops the timer, the task in flight and everything queued behind it. Nothing
* queued before this call may run afterwards: that is what lets the indexer
* close its store here and know no pass will reach for it.
*/
close(): void {
this.closed = true
if (this.timer !== null) {
this.options.clock.clearTimeout(this.timer)
this.timer = null
}
this.controller?.abort()
}
private arm(tick: () => void): void {
if (this.closed || this.timer !== null) {
return
}
this.timer = this.options.clock.setTimeout(() => {
this.timer = null
tick()
}, this.options.intervalMs)
}
private async run(work: (signal: AbortSignal) => Promise<void>): Promise<void> {
if (this.closed) {
return
}
const controller = new AbortController()
this.controller = controller
try {
await work(controller.signal)
} catch (error) {
// An aborted task is a close, never a failure.
if (!controller.signal.aborted) {
this.options.onFailure(error)
}
} finally {
if (this.controller === controller) {
this.controller = null
}
}
}
}
src/main/ai-vault/session-scanner-parse-cache.ts
+48 -9
View File
@@ -26,6 +26,7 @@ import {
import {
readResumableTranscript,
readWholeTranscript,
requestWholeTranscriptRead,
type TranscriptReadStats
} from './session-transcript-reader'
@@ -104,29 +105,67 @@ export function createSessionParseStats(): SessionParseStats {
export async function parseAgentSessionFileCached(
candidate: SessionFileCandidate,
platform: NodeJS.Platform,
stats?: SessionParseStats
stats?: SessionParseStats,
requireRead?: SessionParseReadRequirement
): Promise<AiVaultSession | null> {
// The whole lookup-read-store sequence runs in the lane: a concurrent parse of
// the same path shares this entry's resume point and its message channel.
return inSessionParseFileLane(candidate.file.path, () =>
parseCachedInLane(candidate, platform, stats)
parseCachedInLane(candidate, platform, stats, requireRead)
)
}
/**
* What a caller other than the session list needs out of this parse.
*
* `any`: some bytes must be read. A cursor already at the file's current stat
* is dropped so the reader opens it; one that is merely behind is left alone,
* because an append is a read.
*
* `whole`: the file must be re-read from zero, for a consumer whose own cursor
* covers a span this one does not.
*
* Why it is a parameter and not two calls around this one: the decision reads
* cache state and then changes it, so outside the per-path lane an overlapping
* list parse can store its entry in between and the forced read silently
* degrades to a reuse.
*/
export type SessionParseReadRequirement = 'any' | 'whole'
/**
* True when this cursor already sits at the transcript's current stat, so a
* parse would reuse the cached fold and read no bytes at all.
*/
function sessionParseCacheCoversTranscript(
candidate: SessionFileCandidate,
platform: NodeJS.Platform
): boolean {
const { file } = candidate
const entry = getSessionParseCacheEntry(file.path)
return (
entry !== undefined &&
entry.platform === platform &&
entry.mtimeMs === file.mtimeMs &&
(entry.sizeBytes === null || file.sizeBytes === undefined || entry.sizeBytes === file.sizeBytes)
)
}
async function parseCachedInLane(
candidate: SessionFileCandidate,
platform: NodeJS.Platform,
stats?: SessionParseStats
stats?: SessionParseStats,
requireRead?: SessionParseReadRequirement
): Promise<AiVaultSession | null> {
const { file } = candidate
if (
requireRead === 'whole' ||
(requireRead === 'any' && sessionParseCacheCoversTranscript(candidate, platform))
) {
requestWholeTranscriptRead(file.path)
}
const entry = getSessionParseCacheEntry(file.path)
const transcriptUnchanged =
entry !== undefined &&
entry.platform === platform &&
entry.mtimeMs === file.mtimeMs &&
(entry.sizeBytes === null || file.sizeBytes === undefined || entry.sizeBytes === file.sizeBytes)
if (transcriptUnchanged) {
if (entry !== undefined && sessionParseCacheCoversTranscript(candidate, platform)) {
if (sidecarUnchanged(entry.sidecar, file.sidecar)) {
return reuseCachedSession(candidate, entry, stats)
}
src/main/runtime/push/push-registration-rpc.test.ts
+2 -2
View File
@@ -2,14 +2,14 @@ import { mkdtempSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
import type { RpcContext, RpcMethod } from '../rpc/core'
import { eraseRpcMethods, type RpcContext, type RpcMethod } from '../rpc/core'
import { NOTIFICATION_METHODS } from '../rpc/methods/notifications'
import { DeviceRegistry } from '../device-registry'
import { OrcaRuntimeRpcServer } from '../runtime-rpc'
import { OrcaRuntimeService } from '../orca-runtime'
function method(name: string): RpcMethod {
const found = NOTIFICATION_METHODS.find((candidate) => candidate.name === name)
const found = eraseRpcMethods(NOTIFICATION_METHODS).find((candidate) => candidate.name === name)
if (!found || 'stream' in found) {
throw new Error(`${name} is not a one-shot RPC method`)
}
src/main/runtime/rpc/core-typed-method-contract.test.ts
+114
View File
@@ -0,0 +1,114 @@
// The preserved types are the whole point of defineMethod, so they are asserted here: if a name
// widens to `string` or a result to `unknown`, these assertions fail at typecheck, not at runtime.
import { describe, expect, expectTypeOf, it } from 'vitest'
import { z } from 'zod'
import {
buildRegistry,
defineMethod,
defineStreamingMethod,
eraseRpcMethods,
isStreamingMethod,
type RpcContext,
type RpcMethod,
type RpcStreamingMethod
} from './core'
import type { ALL_RPC_METHODS } from './methods'
import { STATUS_METHODS } from './methods/status'
import type { HOST_CAPABILITY_METHODS } from './methods/host-capabilities'
const ProbeParams = z.object({ id: z.string(), count: z.number().optional() })
const probe = defineMethod({
name: 'test.typedProbe',
params: ProbeParams,
handler: (params) => ({ id: params.id, count: params.count ?? 0 })
})
const schemalessProbe = defineMethod({
name: 'test.schemalessProbe',
params: null,
handler: () => ['a', 'b']
})
const streamingProbe = defineStreamingMethod({
name: 'test.streamingProbe',
params: ProbeParams,
handler: async (params, _ctx, emit) => {
emit(params.id)
}
})
type ByName<TMethods, TName extends string> = Extract<TMethods, { name: TName }>
describe('defineMethod preserves the declared contract', () => {
it('keeps the literal method name', () => {
expectTypeOf(probe.name).toEqualTypeOf<'test.typedProbe'>()
expectTypeOf(streamingProbe.name).toEqualTypeOf<'test.streamingProbe'>()
expect(probe.name).toBe('test.typedProbe')
})
it('keeps the producer result type', () => {
expectTypeOf(probe.handler).returns.toEqualTypeOf<{ id: string; count: number }>()
expectTypeOf(schemalessProbe.handler).returns.toEqualTypeOf<string[]>()
})
it('infers parsed params from the schema, and `void` without one', () => {
expectTypeOf(probe.handler)
.parameter(0)
.toEqualTypeOf<{ id: string; count?: number | undefined }>()
expectTypeOf(schemalessProbe.handler).parameter(0).toEqualTypeOf<void>()
expectTypeOf(streamingProbe.handler)
.parameter(0)
.toEqualTypeOf<{ id: string; count?: number | undefined }>()
expectTypeOf(probe.params).toEqualTypeOf<typeof ProbeParams>()
})
it('keeps a registered method addressable by its literal name', () => {
type StatusGet = ByName<(typeof STATUS_METHODS)[number], 'status.get'>
type ListDistros = ByName<(typeof HOST_CAPABILITY_METHODS)[number], 'host.wsl.listDistros'>
expectTypeOf<StatusGet>().not.toBeNever()
expectTypeOf<StatusGet['handler']>().returns.toExtend<{ runtimeId: string }>()
expectTypeOf<ListDistros['handler']>().returns.toEqualTypeOf<Promise<string[]>>()
// The manifest is the erasure boundary's input, so the literal names have to survive it too.
expectTypeOf<ByName<(typeof ALL_RPC_METHODS)[number], 'status.get'>>().not.toBeNever()
})
})
describe('eraseRpcMethods is the registry boundary', () => {
it('erases to the shape the dispatcher calls, keeping the streaming split', () => {
expectTypeOf(eraseRpcMethods([probe])).toEqualTypeOf<readonly RpcMethod[]>()
expectTypeOf(eraseRpcMethods([streamingProbe])).toEqualTypeOf<readonly RpcStreamingMethod[]>()
expectTypeOf(eraseRpcMethods(STATUS_METHODS)).toEqualTypeOf<readonly RpcMethod[]>()
expectTypeOf(eraseRpcMethods([probe])[0]!.handler)
.parameter(0)
.toEqualTypeOf<unknown>()
})
it('returns the same methods, so nothing about the runtime value changes', () => {
const erased = eraseRpcMethods([probe, streamingProbe])
expect(erased[0]).toBe(probe)
expect(erased[1]).toBe(streamingProbe)
})
it('produces methods the registry accepts and the dispatcher can invoke', async () => {
const registry = buildRegistry([probe, streamingProbe, ...STATUS_METHODS])
const registered = registry.get('test.typedProbe')
expect(registered).toBe(probe)
expect(registry.get('status.get')).toBe(STATUS_METHODS[0])
expect(isStreamingMethod(registry.get('test.streamingProbe')!)).toBe(true)
expect(registered && isStreamingMethod(registered)).toBe(false)
// The dispatcher parses params itself and then calls the erased handler with `unknown`.
const parsed: unknown = probe.params.parse({ id: 'a' })
expect(
registered && !isStreamingMethod(registered)
? await registered.handler(parsed, {} as RpcContext)
: undefined
).toEqual({ id: 'a', count: 0 })
})
it('rejects a duplicate name before erasure hides it', () => {
expect(() => buildRegistry([probe, probe])).toThrow('duplicate_rpc_method:test.typedProbe')
})
})
src/main/runtime/rpc/core.ts
+80 -35
View File
@@ -119,28 +119,27 @@ export type RpcContext = {
) => () => void
}
export type RpcHandler<TParams> = (params: TParams, ctx: RpcContext) => unknown
export type RpcHandler<TParams, TResult> = (params: TParams, ctx: RpcContext) => TResult
// Why: RpcMethod erases the param type; centralizing the cast in defineMethod sidesteps RpcHandler's contravariance.
export type RpcMethod = {
readonly name: string
readonly params: ZodType | null
readonly handler: (params: unknown, ctx: RpcContext) => unknown
// Why: a schema-less method takes no params, so its handler must not be able to read the first argument.
type RpcParsedParams<TSchema extends ZodType | null> = TSchema extends ZodType
? TSchema['_output']
: void
// Why: the authored shape — literal name, params schema, and producer result all survive for compile-time contracts.
export type RpcTypedMethod<TName extends string, TSchema extends ZodType | null, TResult> = {
readonly name: TName
readonly params: TSchema
readonly handler: RpcHandler<RpcParsedParams<TSchema>, TResult>
}
type DefineMethodSpec<TSchema extends ZodType | null> = {
name: string
params: TSchema
handler: RpcHandler<TSchema extends ZodType ? TSchema['_output'] : void>
}
export function defineMethod<TSchema extends ZodType | null>(
spec: DefineMethodSpec<TSchema>
): RpcMethod {
export function defineMethod<TName extends string, TSchema extends ZodType | null, TResult>(
spec: RpcTypedMethod<TName, TSchema, TResult>
): RpcTypedMethod<TName, TSchema, TResult> {
return {
name: spec.name,
params: spec.params,
handler: spec.handler as RpcMethod['handler']
handler: spec.handler
}
}
@@ -150,6 +149,53 @@ export type RpcStreamingHandler<TParams> = (
emit: (result: unknown) => void
) => Promise<void>
// Why: emitted values stay `unknown` — the emit callback is an input, so there is no return position to infer them from.
export type RpcTypedStreamingMethod<TName extends string, TSchema extends ZodType | null> = {
readonly name: TName
readonly params: TSchema
readonly stream: true
readonly handler: RpcStreamingHandler<RpcParsedParams<TSchema>>
}
export function defineStreamingMethod<TName extends string, TSchema extends ZodType | null>(
spec: Omit<RpcTypedStreamingMethod<TName, TSchema>, 'stream'>
): RpcTypedStreamingMethod<TName, TSchema> {
return {
name: spec.name,
params: spec.params,
stream: true,
handler: spec.handler
}
}
// Why `never` params: it makes the declaration a supertype of every parsed-params handler, so typed methods
// travel to the registry boundary — and only there get erased — without a cast in each methods module.
export type RpcMethodDeclaration = {
readonly name: string
readonly params: ZodType | null
readonly handler: (params: never, ctx: RpcContext) => unknown
}
export type RpcStreamingMethodDeclaration = {
readonly name: string
readonly params: ZodType | null
readonly stream: true
readonly handler: (
params: never,
ctx: RpcContext,
emit: (result: unknown) => void
) => Promise<void>
}
export type RpcAnyMethodDeclaration = RpcMethodDeclaration | RpcStreamingMethodDeclaration
// Why: RpcMethod is the registry's erased view; the dispatcher parses params itself and hands handlers `unknown`.
export type RpcMethod = {
readonly name: string
readonly params: ZodType | null
readonly handler: (params: unknown, ctx: RpcContext) => unknown
}
// Why: the `stream` flag lets the dispatcher route these to the emit-based path instead of the one-shot Promise path.
export type RpcStreamingMethod = {
readonly name: string
@@ -162,34 +208,33 @@ export type RpcStreamingMethod = {
) => Promise<void>
}
type DefineStreamingMethodSpec<TSchema extends ZodType | null> = {
name: string
params: TSchema
handler: RpcStreamingHandler<TSchema extends ZodType ? TSchema['_output'] : void>
}
export function defineStreamingMethod<TSchema extends ZodType | null>(
spec: DefineStreamingMethodSpec<TSchema>
): RpcStreamingMethod {
return {
name: spec.name,
params: spec.params,
stream: true,
handler: spec.handler as RpcStreamingMethod['handler']
}
}
export type RpcAnyMethod = RpcMethod | RpcStreamingMethod
// Why the overloads: erasure drops the parsed-params type, not the one-shot/streaming split the dispatcher routes on.
export function eraseRpcMethods(methods: readonly RpcMethodDeclaration[]): readonly RpcMethod[]
export function eraseRpcMethods(
methods: readonly RpcStreamingMethodDeclaration[]
): readonly RpcStreamingMethod[]
export function eraseRpcMethods(
methods: readonly RpcAnyMethodDeclaration[]
): readonly RpcAnyMethod[]
// Why: the one place the parsed-params type is dropped — contravariance makes it uncastable by assignment, and
// the dispatcher only ever calls a handler with an already-parsed `unknown`. Runtime value is untouched.
export function eraseRpcMethods(
methods: readonly RpcAnyMethodDeclaration[]
): readonly RpcAnyMethod[] {
return methods as readonly RpcAnyMethod[]
}
export function isStreamingMethod(method: RpcAnyMethod): method is RpcStreamingMethod {
return 'stream' in method && method.stream === true
}
export type RpcRegistry = ReadonlyMap<string, RpcAnyMethod>
export function buildRegistry(methods: readonly RpcAnyMethod[]): RpcRegistry {
export function buildRegistry(methods: readonly RpcAnyMethodDeclaration[]): RpcRegistry {
const registry = new Map<string, RpcAnyMethod>()
for (const method of methods) {
for (const method of eraseRpcMethods(methods)) {
if (registry.has(method.name)) {
throw new Error(`duplicate_rpc_method:${method.name}`)
}
src/main/runtime/rpc/dispatcher-request-parsing.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { compile, type ZodType } from 'zod'
import {
formatZodError,
type RpcAnyMethod,
type RpcAnyMethodDeclaration,
type RpcEnvelopeMeta,
type RpcRequest,
type RpcResponse
@@ -12,7 +12,7 @@ const compiledParams = new WeakMap<ZodType, ZodType>()
export function parseRpcRequestParams(
request: RpcRequest,
method: RpcAnyMethod,
method: RpcAnyMethodDeclaration,
meta: RpcEnvelopeMeta
): { value: unknown; error?: undefined } | { value?: undefined; error: RpcResponse } {
if (method.params === null) {
src/main/runtime/rpc/dispatcher.ts
+5 -2
View File
@@ -1,7 +1,7 @@
import {
buildRegistry,
isStreamingMethod,
type RpcAnyMethod,
type RpcAnyMethodDeclaration,
type RpcEnvelopeMeta,
type RpcRegistry,
type RpcRequest,
@@ -24,7 +24,10 @@ import { parseRpcRequestParams } from './dispatcher-request-parsing'
import { RpcStreamingDispatcher } from './rpc-streaming-dispatcher'
import { invokeDispatcherUnaryMethod } from './dispatcher-unary-method-invocation'
export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] }
export type DispatcherOptions = {
runtime: OrcaRuntimeService
methods?: readonly RpcAnyMethodDeclaration[]
}
type DispatchCallOptions = RpcDispatchStreamingOptions
src/main/runtime/rpc/methods/accounts.test.ts
+2 -2
View File
@@ -2,11 +2,11 @@ import { describe, expect, it, vi } from 'vitest'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { isStreamingMethod } from '../core'
import { eraseRpcMethods, isStreamingMethod } from '../core'
import { ACCOUNT_METHODS } from './accounts'
function method(name: string) {
const found = ACCOUNT_METHODS.find((candidate) => candidate.name === name)
const found = eraseRpcMethods(ACCOUNT_METHODS).find((candidate) => candidate.name === name)
if (!found) {
throw new Error(`Missing method ${name}`)
}
src/main/runtime/rpc/methods/accounts.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineMethod, defineStreamingMethod } from '../core'
import {
AccountsUnsubscribeParams,
AddClaudeFromConfigDirParams,
@@ -24,7 +24,7 @@ let accountsSubscriptionSeq = 0
// captures an already-authenticated CLAUDE_CONFIG_DIR (no PTY) so the local
// `orca account add` CLI can register accounts on a headless host; it is gated
// to the local runtime connection, never a mobile device token. See #1438.
export const ACCOUNT_METHODS: readonly RpcAnyMethod[] = [
export const ACCOUNT_METHODS = [
defineMethod({
name: 'accounts.list',
params: ListAccountsParams,
src/main/runtime/rpc/methods/agent-hooks.test.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { isStreamingMethod, type RpcContext } from '../core'
import { eraseRpcMethods, isStreamingMethod, type RpcContext } from '../core'
const { installForRuntimeHomeSerializedMock, realpathMock } = vi.hoisted(() => ({
installForRuntimeHomeSerializedMock: vi.fn(),
@@ -23,7 +23,7 @@ const RUNTIME_HOME =
'\\\\wsl.localhost\\Ubuntu-24.04\\home\\jin\\.local\\share\\orca\\codex-runtime-home\\home'
function prepareMethod() {
const method = AGENT_HOOK_METHODS.find(
const method = eraseRpcMethods(AGENT_HOOK_METHODS).find(
(candidate) => candidate.name === 'agentHooks.prepareCodexForWslPane'
)
if (!method || isStreamingMethod(method)) {
src/main/runtime/rpc/methods/agent-hooks.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { prepareManagedWslCodexHomeBeforeShellLaunch } from '../../../codex/managed-wsl-home-shell-preflight'
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { PrepareCodexForWslPaneParams } from '../../../../shared/rpc-contract/agent-hooks-params'
export const AGENT_HOOK_METHODS: readonly RpcMethod[] = [
export const AGENT_HOOK_METHODS = [
defineMethod({
name: 'agentHooks.prepareCodexForWslPane',
params: PrepareCodexForWslPaneParams,
src/main/runtime/rpc/methods/agent-session.ts
+2 -2
View File
@@ -10,7 +10,7 @@ import {
parseAgentSessionOperationTimestamp
} from '../../../../shared/agent-session-host-authority'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { defineMethod, type RpcAnyMethod } from '../core'
import { defineMethod } from '../core'
import {
CreateAgentSessionParams,
EnsureAgentSessionParams
@@ -58,7 +58,7 @@ function assertOperationTimestampWithinFutureSkew(clientOperationId: string): vo
}
}
export const AGENT_SESSION_METHODS: RpcAnyMethod[] = [
export const AGENT_SESSION_METHODS = [
defineMethod({
name: 'terminal.ensureAgentSession',
params: EnsureAgentSessionParams,
src/main/runtime/rpc/methods/ai-vault.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { restampAiVaultListResult } from '../../../ai-vault/session-list-results'
import type { AiVaultPrepareSessionResumeArgs } from '../../../../shared/ai-vault-resume-preparation'
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
@@ -15,7 +15,7 @@ import {
} from '../../../../shared/rpc-contract/ai-vault-params'
export { AiVaultListSessionsParams, AiVaultPrepareSessionResumeParams, AiVaultSessionTitlesParams }
export const AI_VAULT_METHODS: RpcMethod[] = [
export const AI_VAULT_METHODS = [
defineMethod({
name: 'aiVault.resolveSessionTitles',
params: AiVaultSessionTitlesParams,
src/main/runtime/rpc/methods/artifacts.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcAnyMethod } from '../core'
import { defineMethod } from '../core'
import {
ArtifactsDeleteParams,
ListOptions,
@@ -6,7 +6,7 @@ import {
WriteRequest
} from '../../../../shared/rpc-contract/artifacts-params'
export const ARTIFACT_METHODS: readonly RpcAnyMethod[] = [
export const ARTIFACT_METHODS = [
defineMethod({
name: 'artifacts.list',
params: ListOptions,
src/main/runtime/rpc/methods/automation-scoped-list-methods.test.ts
+2 -2
View File
@@ -4,14 +4,14 @@
* current callers also receive owner metadata.
*/
import { describe, expect, it, vi } from 'vitest'
import type { RpcContext, RpcRequest } from '../core'
import { eraseRpcMethods, type RpcContext, type RpcRequest } from '../core'
import { RpcDispatcher } from '../dispatcher'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { AUTOMATION_METHODS } from './automations'
import { AUTOMATION_OWNER_FENCING_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version'
function method(name: string) {
const found = AUTOMATION_METHODS.find((entry) => entry.name === name)
const found = eraseRpcMethods(AUTOMATION_METHODS).find((entry) => entry.name === name)
if (!found?.params) {
throw new Error(`missing method ${name}`)
}
src/main/runtime/rpc/methods/automations.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { AUTOMATION_OWNER_FENCING_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version'
import type { AutomationOwnerPrecondition } from '../../../../shared/automation-owner-precondition'
import { defineMethod, type RpcContext, type RpcMethod } from '../core'
import { defineMethod, type RpcContext } from '../core'
import {
AutomationCreate,
AutomationId,
@@ -25,7 +25,7 @@ function mutationOwner(
return context.runtime.automationOwnerPrecondition(id) ?? undefined
}
export const AUTOMATION_METHODS: RpcMethod[] = [
export const AUTOMATION_METHODS = [
defineMethod({
name: 'automation.list',
params: AutomationList,
src/main/runtime/rpc/methods/browser-client-file-channel.ts
+2 -2
View File
@@ -7,7 +7,7 @@ import { BROWSER_CLIENT_HOST_RUNTIME_CAPABILITY } from '../../../../shared/proto
import { getBrowserClientDownloadTransferStore } from '../../browser-client-download-transfer-store'
import { getBrowserHostLeaseRegistry } from '../../browser-host-lease-registry-instance'
import { getRuntimeBrowserPageRegistry } from '../../runtime-browser-page-registry'
import { defineMethod, type RpcAnyMethod, type RpcContext } from '../core'
import { defineMethod, type RpcContext } from '../core'
type FileChannelAuthorityParams = {
browserHostClientId: string
@@ -64,7 +64,7 @@ function requireFileChannelPage(
return page
}
export const BROWSER_CLIENT_FILE_CHANNEL_METHODS: RpcAnyMethod[] = [
export const BROWSER_CLIENT_FILE_CHANNEL_METHODS = [
defineMethod({
name: 'browser.clientHost.fileChannel.read',
params: BrowserClientFileChannelReadParams,
src/main/runtime/rpc/methods/browser-client-host.ts
+2 -2
View File
@@ -14,9 +14,9 @@ import { getRuntimeBrowserPageRegistry } from '../../runtime-browser-page-regist
import { adoptRuntimeBrowserClientPagesFromInventory } from '../../runtime-browser-client-page-adoption'
import { recoverUnavailableRuntimeBrowserClientPages } from '../../runtime-browser-client-page-recovery'
import { releaseRuntimeBrowserClientPageRecord } from '../../runtime-browser-client-page-release'
import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineMethod, defineStreamingMethod } from '../core'
export const BROWSER_CLIENT_HOST_METHODS: RpcAnyMethod[] = [
export const BROWSER_CLIENT_HOST_METHODS = [
defineStreamingMethod({
name: 'browser.clientHost.attach',
params: BrowserClientHostAttachParams,
src/main/runtime/rpc/methods/browser-core.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { BrowserTarget } from '../schemas'
import {
Check,
@@ -35,7 +35,7 @@ import { BrowserOpenUrlParams, BrowserTabCreateParams } from './browser-tab-crea
import { BROWSER_TEXT_METHODS } from './browser-text-rpc-methods'
import { CertificateProceed } from '../../../../shared/rpc-contract/browser-core-params'
export const BROWSER_CORE_METHODS: RpcMethod[] = [
export const BROWSER_CORE_METHODS = [
defineMethod({
name: 'browser.snapshot',
params: BrowserTarget,
src/main/runtime/rpc/methods/browser-extras.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { assertRpcClipboardTextWriteWithinLimit } from '../rpc-clipboard-text-validation'
import { BrowserTarget } from '../schemas'
import {
@@ -23,7 +23,7 @@ import {
} from './browser-schemas'
import { MouseClick } from '../../../../shared/rpc-contract/browser-extras-params'
export const BROWSER_EXTRA_METHODS: RpcMethod[] = [
export const BROWSER_EXTRA_METHODS = [
defineMethod({
name: 'browser.cookie.get',
params: CookieGet,
src/main/runtime/rpc/methods/browser-network-tunnel.ts
+2 -2
View File
@@ -12,14 +12,14 @@ import {
BROWSER_NETWORK_TUNNEL_RUNTIME_CAPABILITY
} from '../../../../shared/protocol-version'
import { getBrowserHostLeaseRegistry } from '../../browser-host-lease-registry-instance'
import { defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineStreamingMethod } from '../core'
const outboundMemoryBudgets = new BrowserNetworkTunnelOutboundMemoryBudgetRegistry()
export function createBrowserNetworkTunnelMethods(
memoryBudgets: BrowserNetworkTunnelOutboundMemoryBudgetRegistry = outboundMemoryBudgets,
resolveExecutionRoute: BrowserNetworkExecutionRouteResolver = resolveBrowserNetworkExecutionRoute
): RpcAnyMethod[] {
) {
return [
defineStreamingMethod({
name: 'network.browserTunnel',
src/main/runtime/rpc/methods/browser-screencast.ts
+2 -2
View File
@@ -1,11 +1,11 @@
import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineMethod, defineStreamingMethod } from '../core'
import { Screencast } from './browser-schemas'
import { BrowserError } from '../../../browser/browser-error'
import { BROWSER_UNAVAILABLE_ERROR_CODE } from '../../../../shared/runtime-types'
import { runtimeBrowserCommandsFactoryIsAvailable } from '../../runtime-browser-commands-factory'
import { ScreencastUnsubscribe } from '../../../../shared/rpc-contract/browser-screencast-params'
export const BROWSER_SCREENCAST_METHODS: RpcAnyMethod[] = [
export const BROWSER_SCREENCAST_METHODS = [
defineStreamingMethod({
name: 'browser.screencast',
params: Screencast,
src/main/runtime/rpc/methods/browser-text-rpc-methods.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { assertRpcClipboardTextWriteWithinLimit } from '../rpc-clipboard-text-validation'
import { Fill, KeyboardInsert, Type } from './browser-schemas'
export const BROWSER_TEXT_METHODS: RpcMethod[] = [
export const BROWSER_TEXT_METHODS = [
defineMethod({
name: 'browser.fill',
params: Fill,
src/main/runtime/rpc/methods/client-events.test.ts
+7 -2
View File
@@ -1,11 +1,16 @@
import { describe, expect, it, vi } from 'vitest'
import type { RuntimeClientEvent } from '../../../../shared/runtime-client-events'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { isStreamingMethod, type RpcContext, type RpcStreamingMethod } from '../core'
import {
eraseRpcMethods,
isStreamingMethod,
type RpcContext,
type RpcStreamingMethod
} from '../core'
// Why: importing client-events directly trips its module-init cycle through ipc/ssh; the index resolves it.
import { ALL_RPC_METHODS } from './index'
const subscribeMethod = ALL_RPC_METHODS.find(
const subscribeMethod = eraseRpcMethods(ALL_RPC_METHODS).find(
(method) => method.name === 'runtime.clientEvents.subscribe' && isStreamingMethod(method)
) as RpcStreamingMethod
src/main/runtime/rpc/methods/client-events.ts
+2 -2
View File
@@ -1,11 +1,11 @@
import { getRegisteredSshState, listRegisteredSshTargets } from '../../../ssh/ssh-target-registry'
import { getPublicSshState } from '../../public-ssh-state'
import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineMethod, defineStreamingMethod } from '../core'
import { ClientEventsUnsubscribeParams } from '../../../../shared/rpc-contract/client-events-params'
let clientEventSubscriptionSeq = 0
export const CLIENT_EVENT_METHODS: readonly RpcAnyMethod[] = [
export const CLIENT_EVENT_METHODS = [
defineStreamingMethod({
name: 'runtime.clientEvents.subscribe',
params: null,
src/main/runtime/rpc/methods/client-ui.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { omitPairingLocalUiFields } from '../../../../shared/pairing-local-ui-fields'
import type { PersistedUIState } from '../../../../shared/persisted-ui-state-types'
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import {
NativeChatSessionOptionsMutation,
PRBotAuthorOverrideUpdate,
@@ -12,7 +12,7 @@ import { FeatureInteractionIdParam, UiUpdate } from './client-ui-schemas'
import { TerminalQuickCommandsUpdate } from './terminal-quick-command-rpc-schema'
export const CLIENT_UI_METHODS: RpcMethod[] = [
export const CLIENT_UI_METHODS = [
defineMethod({
name: 'settings.get',
params: null,
src/main/runtime/rpc/methods/clipboard.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcContext, type RpcMethod } from '../core'
import { defineMethod, type RpcContext } from '../core'
import { saveClipboardImageBufferAsTempFile } from '../../../window/clipboard-image-temp-file'
import { randomUUID } from 'node:crypto'
import { recordMobileClipboardImagePath } from '../mobile-clipboard-image-provenance'
@@ -95,7 +95,7 @@ function assertValidBase64Content(value: string): void {
}
}
export const CLIPBOARD_METHODS: RpcMethod[] = [
export const CLIPBOARD_METHODS = [
defineMethod({
name: 'clipboard.saveImageAsTempFile',
params: SaveImageAsTempFile,
src/main/runtime/rpc/methods/computer-actions.test.ts
+2 -1
View File
@@ -27,6 +27,7 @@ vi.mock('../../../computer/macos-computer-use-permissions', () => ({
}))
import { COMPUTER_METHODS, resetComputerSessionsForTest } from './computer'
import { eraseRpcMethods } from '../core'
describe('computer action RPC methods', () => {
beforeEach(() => {
@@ -269,7 +270,7 @@ describe('computer action RPC methods', () => {
})
function findMethod(name: string) {
const method = COMPUTER_METHODS.find((candidate) => candidate.name === name)
const method = eraseRpcMethods(COMPUTER_METHODS).find((candidate) => candidate.name === name)
if (!method) {
throw new Error(`missing method ${name}`)
}
src/main/runtime/rpc/methods/computer.test.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { buildRegistry } from '../core'
import { eraseRpcMethods, buildRegistry } from '../core'
import { CLIPBOARD_TEXT_WRITE_MAX_BYTES } from '../../../../shared/clipboard-text'
const computerMocks = vi.hoisted(() => ({
@@ -249,7 +249,7 @@ describe('computer RPC methods', () => {
})
function findMethod(name: string) {
const method = COMPUTER_METHODS.find((candidate) => candidate.name === name)
const method = eraseRpcMethods(COMPUTER_METHODS).find((candidate) => candidate.name === name)
if (!method) {
throw new Error(`missing method ${name}`)
}
src/main/runtime/rpc/methods/computer.ts
+2 -2
View File
@@ -6,7 +6,7 @@ import {
callComputerSidecarSnapshot,
resetComputerSidecarForTest
} from '../../../computer/sidecar-client'
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import {
Click,
ComputerObserveTarget,
@@ -31,7 +31,7 @@ export function resetComputerSessionsForTest(): void {
resetComputerSidecarForTest()
}
export const COMPUTER_METHODS: RpcMethod[] = [
export const COMPUTER_METHODS = [
defineMethod({
name: 'computer.capabilities',
params: ComputerCapabilitiesParams,
src/main/runtime/rpc/methods/diagnostics.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
export const DIAGNOSTICS_METHODS: RpcMethod[] = [
export const DIAGNOSTICS_METHODS = [
defineMethod({
name: 'diagnostics.memory',
params: null,
src/main/runtime/rpc/methods/emulator.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import path from 'node:path'
import { z } from 'zod'
import {
@@ -32,7 +32,7 @@ const InstallParams = z.object({
worktree: z.string().optional()
})
export const EMULATOR_METHODS: RpcMethod[] = [
export const EMULATOR_METHODS = [
defineMethod({
name: 'emulator.list',
params: ListParams,
src/main/runtime/rpc/methods/files-mutation-methods.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcAnyMethod } from '../core'
import { defineMethod } from '../core'
import {
FileCommitUpload,
FileCopy,
@@ -33,7 +33,7 @@ function sshMutationArguments(
]
}
export const FILE_MUTATION_METHODS: RpcAnyMethod[] = [
export const FILE_MUTATION_METHODS = [
defineMethod({
name: 'files.write',
params: FileWrite,
src/main/runtime/rpc/methods/files-terminal-artifact-methods.ts
+2 -2
View File
@@ -1,11 +1,11 @@
import { defineMethod, type RpcAnyMethod } from '../core'
import { defineMethod } from '../core'
import { remoteFileContentBudget } from './files-remote-content-budget'
import {
TerminalArtifactFile,
TerminalArtifactFileWrite
} from '../../../../shared/rpc-contract/files-terminal-artifact-params'
export const FILE_TERMINAL_ARTIFACT_METHODS: RpcAnyMethod[] = [
export const FILE_TERMINAL_ARTIFACT_METHODS = [
defineMethod({
name: 'files.readTerminalArtifact',
params: TerminalArtifactFile,
src/main/runtime/rpc/methods/files.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core'
import { defineMethod, defineStreamingMethod } from '../core'
import { runFileWatchStream } from './file-watch-stream-lifecycle'
import { FILE_MUTATION_METHODS } from './files-mutation-methods'
import { remoteFileContentBudget } from './files-remote-content-budget'
@@ -21,7 +21,7 @@ import {
let filesWatchSubscriptionSeq = 0
export const FILE_METHODS: RpcAnyMethod[] = [
export const FILE_METHODS = [
defineMethod({
name: 'files.list',
params: WorktreeSelector,
src/main/runtime/rpc/methods/folder-workspace.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { defineMethod, type RpcMethod } from '../core'
import { defineMethod } from '../core'
import { resolveRpcWorkspaceCreatorProvenance } from '../workspace-creator-context'
import {
FolderWorkspaceCreate,
@@ -7,7 +7,7 @@ import {
FolderWorkspaceUpdate
} from '../../../../shared/rpc-contract/folder-workspace-params'
export const FOLDER_WORKSPACE_METHODS: RpcMethod[] = [
export const FOLDER_WORKSPACE_METHODS = [
defineMethod({
name: 'folderWorkspace.list',
params: null,

Some files were not shown because too many files have changed in this diff Show More