From fa9bd2d5225b4377294bd2b6d654d8acd132346d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:32:32 -0800 Subject: [PATCH] Resume indexed session reads in fair bounded batches Use indexed ordered cursors and retain only the next read page, with live admission and a per-pass retry cutoff. Alternate changed and older work before byte admission so a large page cannot starve history. Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com> --- .../session-search-candidate-inventory.ts | 12 +- .../session-search-catalog-batching.test.ts | 251 ++++++++++++++---- .../session-search-catalog-candidate-types.ts | 2 +- .../session-search-catalog-client.ts | 18 +- .../session-search-catalog-query.ts | 56 ++++ .../session-search-catalog-types.ts | 3 + .../ai-vault-search/session-search-catalog.ts | 39 +-- .../session-search-index-pass.ts | 5 +- .../ai-vault-search/session-search-pass.ts | 10 +- .../session-search-store-transitions.ts | 20 +- .../session-search-work-catalog.ts | 85 +++++- .../session-search-work-queue.ts | 36 ++- .../session-search-worker-entry.ts | 1 + .../session-search-worker-store-base.ts | 4 + 14 files changed, 433 insertions(+), 109 deletions(-) create mode 100644 src/main/ai-vault-search/session-search-catalog-query.ts diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index e1d40054e61..cea5d1df7c3 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -45,10 +45,14 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli super.reset() this.sourceOrders.clear() } - async rootFor(candidate: SessionFileCandidate): Promise { - return ( - await this.request!({ kind: 'path', path: candidate.file.path, agent: candidate.agent }) - ).rows[0]?.root + async readAdmission(candidate: SessionFileCandidate) { + const result = await this.request!({ + kind: 'path', + path: candidate.file.path, + agent: candidate.agent, + read: true + }) + return { root: result.rows.find((row) => row.selected)?.root, work: result.work! } } async has(path: string, agent?: string | null): Promise { return (await this.request!({ kind: 'path', path, agent })).rows.length > 0 diff --git a/src/main/ai-vault-search/session-search-catalog-batching.test.ts b/src/main/ai-vault-search/session-search-catalog-batching.test.ts index ece31de0afd..31382aef33c 100644 --- a/src/main/ai-vault-search/session-search-catalog-batching.test.ts +++ b/src/main/ai-vault-search/session-search-catalog-batching.test.ts @@ -15,6 +15,10 @@ import { import { SessionSearchWorkerClient } from './session-search-worker-client' import * as filesystem from '../native-chat/wsl-transcript-fs-access' import { SessionSearchIndexer } from './session-search-indexer' +import { SessionSearchWorkerStore } from './session-search-worker-store' +import { SessionSearchWorkQueue } from './session-search-work-queue' +import { configureSessionSearchWorkerFixture } from './session-search-worker.test-fixture' +import type { SessionFileCandidate } from '../ai-vault/session-scanner-types' it.each([40, 400])( 'batches discovery and replacement metadata for %s retained files', @@ -146,53 +150,56 @@ it.each([40, 400])( } ) -it('searches the newest known transcript before an older candidate page', async () => { - const harness = await openSessionSearchIndexerHarness('ss-newest-page') - const clock = new FakeSessionSearchClock() - clock.costPerNowMs = 1000 - const subject = new SessionSearchIndexer({ - databasePath: harness.databasePath, - roots: harness.roots, - historyDays: null, - clock, - passDeadlineMs: 1000, - discoveryDeadlineMs: Infinity - }) - try { - for (let index = 0; index < 300; index++) { - const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`) - await writeClaudeTranscript( - path, - [index === 299 ? 'newestknownneedle' : 'olderknownneedle'], - `session-${index}` - ) - const time = new Date(index === 299 ? '2025-01-02' : '2025-01-01') - await utimes(path, time, time) +it.each([40, 400])( + 'searches the newest known transcript before an older candidate page at %s files', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-newest-page') + const clock = new FakeSessionSearchClock() + clock.costPerNowMs = 1000 + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock, + passDeadlineMs: 1000, + discoveryDeadlineMs: Infinity + }) + try { + for (let index = 0; index < population; index++) { + const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`) + await writeClaudeTranscript( + path, + [index === population - 1 ? 'newestknownneedle' : 'olderknownneedle'], + `session-${index}` + ) + const time = new Date(index === population - 1 ? '2025-01-02' : '2025-01-01') + await utimes(path, time, time) + } + await subject.start() + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'" + ) + .get() + ) + ).toEqual({ n: 2 }) + expect( + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'" + ) + .get() + ) + ).toEqual({ n: 0 }) + } finally { + await subject.close() + await harness.cleanup() } - await subject.start() - expect( - harness.read((db) => - db - .prepare( - "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'" - ) - .get() - ) - ).toEqual({ n: 2 }) - expect( - harness.read((db) => - db - .prepare( - "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'" - ) - .get() - ) - ).toEqual({ n: 0 }) - } finally { - await subject.close() - await harness.cleanup() } -}) +) it('redelivers an unaccepted observation and retains a newer rewrite after an older acknowledgement', async () => { const harness = await openSessionSearchIndexerHarness('ss-observation-ack') @@ -373,3 +380,157 @@ it.each([40, 400])( } } ) + +it.each([40, 400])( + 'searches a new change and owed history in the same limited pass with %s changes', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-read-fairness') + const clock = new FakeSessionSearchClock() + clock.costPerNowMs = 1000 + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock, + passDeadlineMs: 2000, + discoveryDeadlineMs: Infinity, + fullSweepEveryCycles: 1000 + }) + const directory = join( + harness.claudeProjectDir, + ...Array.from({ length: 8 }, () => 'p'.repeat(90)) + ) + const hits = () => + harness.read((db) => + db + .prepare(`SELECT DISTINCT s.session_id AS id FROM messages_fts + JOIN messages m ON m.id=messages_fts.rowid + JOIN sessions s ON s.id=m.session_row_id + WHERE messages_fts MATCH 'fairnessneedle' ORDER BY s.session_id`) + .all() + ) + try { + for (let index = 0; index < 3; index++) { + const path = join(directory, `backlog-${index}.jsonl`) + await writeClaudeTranscript(path, ['fairnessneedle'], `backlog-${index}`) + const time = new Date(Date.UTC(2024, 0, 1) - index * 1000) + await utimes(path, time, time) + } + await subject.start() + expect(hits()).toEqual([{ id: 'backlog-0' }, { id: 'backlog-1' }]) + for (let index = 0; index < population; index++) { + const path = join(directory, `changed-${String(index).padStart(3, '0')}.jsonl`) + await writeClaudeTranscript(path, ['fairnessneedle'], `changed-${index}`) + const time = new Date(Date.UTC(2025, 0, 1) + index * 1000) + await utimes(path, time, time) + } + await subject.reconcile() + expect(hits()).toEqual([ + { id: 'backlog-0' }, + { id: 'backlog-1' }, + { id: 'backlog-2' }, + { id: `changed-${population - 1}` } + ]) + } finally { + await subject.close() + await harness.cleanup() + } + } +) + +it.each([40, 400])( + 'preserves ranked read obligations across bounded pages for %s candidates', + async (population) => { + const harness = await openSessionSearchIndexerHarness('ss-read-pages') + const store = new SessionSearchWorkerStore(harness.databasePath, (error) => { + throw error + }) + const queue = new SessionSearchWorkQueue() + queue.attach(store) + const traffic = vi.spyOn(SessionSearchWorkerClient.prototype, 'request') + const candidates: SessionFileCandidate[] = [] + let largestResponse = 0 + configureSessionSearchWorkerFixture({ + prelude: '', + data: undefined, + onResponse: (response) => { + largestResponse = Math.max(largestResponse, JSON.stringify(response).length * 2) + } + }) + try { + await store.prepare() + const prefix = join(harness.root, ...Array.from({ length: 10 }, () => 'p'.repeat(100))) + for (let index = 0; index < population; index++) { + const candidate: SessionFileCandidate = { + agent: 'claude', + codexHome: null, + file: { + path: join(prefix, `${String(population - index).padStart(4, '0')}.jsonl`), + mtimeMs: 10000 - Math.floor(index / 4), + modifiedAt: '1970-01-01T00:00:10.000Z', + sizeBytes: 20 + } + } + if (index === 1 || index === 3) { + const segments = index === 1 ? 12000 : 6000 + const windowsRoot = `C:\\${Array.from({ length: segments }, () => 'p').join('\\')}` + candidate.agent = 'antigravity' + candidate.file.path = `${windowsRoot}\\conversation-${index}.pb` + candidate.antigravityHistoryPath = `${windowsRoot}\\history-${index}.pb` + } + candidates.push(candidate) + await queue.enqueue(candidate, store, false, index % 2 === 0) + } + traffic.mockClear() + queue.beginPass() + const first = await queue.next() + expect(first?.candidate.file.path).toBe(candidates[0].file.path) + queue.beginPass() + expect(await queue.next()).toEqual(first) + await queue.acknowledge(first!, true) + const observed: string[] = [] + for (let index = 1; index < population; index++) { + let work = await queue.next() + if (index === 1) { + await queue.enqueue(candidates[1], store, false, false, true) + await queue.enqueue(candidates[2], store, true, true) + await queue.acknowledge(work!, false) + work = await queue.next() + expect(work).toMatchObject({ candidate: candidates[1], metadata: true }) + } + if (index === 2) { + expect(work).toMatchObject({ candidate: candidates[2], force: true }) + } + observed.push(work!.candidate.file.path) + await queue.acknowledge(work!, false) + } + expect(observed).toEqual(candidates.slice(1).map((candidate) => candidate.file.path)) + expect(await queue.next()).toBeUndefined() + // Each lane needs a bounded projection per row/byte page; per-file accepted acknowledgements are separate. + const pages = + Math.ceil(population / 256) + + Math.ceil((JSON.stringify(candidates).length * 2) / (128 * 1024)) + expect( + traffic.mock.calls.filter( + ([command]) => command.op === 'work' && command.command.kind === 'next' + ).length + ).toBeLessThanOrEqual(40 + 12 * pages) + queue.beginPass() + const retry = await queue.next() + expect(retry?.candidate.file.path).toBe(candidates[0].file.path) + await queue.enqueue(candidates[0], store, false, true, true) + await queue.acknowledge(retry!, false) + const latest = await queue.next() + expect(latest).toMatchObject({ candidate: candidates[0], metadata: true }) + await queue.acknowledge(latest!, false) + expect(await queue.next()).toBeUndefined() + expect(largestResponse).toBeLessThanOrEqual(256 * 1024) + } finally { + configureSessionSearchWorkerFixture() + traffic.mockRestore() + await store.flush() + store.close() + await harness.cleanup() + } + } +) diff --git a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts index 67a705a7278..09ce39e7799 100644 --- a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts @@ -38,7 +38,7 @@ export type SessionSearchCandidateCommand = rows: { path: string; agents?: string[]; observation?: SessionSearchRemovalReceipt }[] } | { kind: 'unowned-page'; rows: { path: string; agent?: string | null }[] } - | { kind: 'path'; path: string; agent?: string | null } + | { kind: 'path'; path: string; agent?: string | null; read?: boolean } | { kind: 'group'; group: string; after?: string } | { kind: 'changes'; after?: string; root?: string; agent?: string } | { kind: 'rerank'; root: string; agent: string; order: [string, number][]; after?: string } diff --git a/src/main/ai-vault-search/session-search-catalog-client.ts b/src/main/ai-vault-search/session-search-catalog-client.ts index 5071c538cb7..44e448dd352 100644 --- a/src/main/ai-vault-search/session-search-catalog-client.ts +++ b/src/main/ai-vault-search/session-search-catalog-client.ts @@ -3,6 +3,7 @@ import type { SessionSearchCatalogResult, SessionSearchCatalogRow } from './session-search-catalog' +import type { SessionSearchCatalogPosition } from './session-search-catalog-types' import { sessionSearchRecordPages } from './session-search-catalog-pages' import { SESSION_SEARCH_CATALOG_PAGE_BYTES, @@ -161,6 +162,7 @@ export class SessionSearchCatalogClient { async page( options: { after?: string + afterOrder?: SessionSearchCatalogPosition parent?: string group?: string ready?: number @@ -169,12 +171,24 @@ export class SessionSearchCatalogClient { limit?: number prefix?: string throughRevision?: number + eligibleThrough?: number } = {} - ): Promise<{ rows: { key: string; value: T }[]; after?: string }> { + ): Promise<{ + rows: { key: string; value: T }[] + after?: string + position?: SessionSearchCatalogPosition + }> { const result = await this.request({ kind: 'page', scope: this.scope, ...options }) return { rows: (result.rows ?? []).map((row) => ({ key: row.key, value: this.decode(row) })), - after: result.after + after: result.after, + position: result.rows?.length + ? { + key: result.rows.at(-1)!.key, + rank: result.rows.at(-1)!.rank, + tie: result.rows.at(-1)!.tie ?? 0 + } + : undefined } } diff --git a/src/main/ai-vault-search/session-search-catalog-query.ts b/src/main/ai-vault-search/session-search-catalog-query.ts new file mode 100644 index 00000000000..308057c234d --- /dev/null +++ b/src/main/ai-vault-search/session-search-catalog-query.ts @@ -0,0 +1,56 @@ +import type SyncDatabase from '../sqlite/sync-database' +import type { SessionSearchCatalogBaseCommand } from './session-search-catalog-types' + +export function sessionSearchCatalogPageRows( + db: SyncDatabase, + command: Extract, + lane?: 'changed' | 'backlog' +) { + const conditions = ['scope = ?'] + const values: (string | number)[] = [command.scope] + if (lane) { + // Simultaneous lane cursors must not share one cached native statement. + conditions.push(lane === 'changed' ? "group_key='changed'" : "group_key='backlog'") + } + if (command.afterOrder) { + const { rank, tie, key } = command.afterOrder + conditions.push(command.orderRevision ? '(tie,key) > (?,?)' : '(rank,tie,key) > (?,?,?)') + values.push(...(command.orderRevision ? [tie, key] : [rank, tie, key])) + } else if (command.after !== undefined) { + if (command.orderRank || command.orderRevision) { + throw new Error('Ordered session catalog continuation requires its full position') + } + conditions.push('key > ?') + values.push(command.after) + } + if (command.prefix !== undefined) { + conditions.push('key >= ?', 'key < ?') + values.push(command.prefix, `${command.prefix}\uffff`) + } + for (const [column, value] of [ + ['parent', command.parent], + ['group_key', command.group], + ['ready', command.ready] + ] as const) { + if (value !== undefined) { + conditions.push(`${column} = ?`) + values.push(value) + } + } + if (command.throughRevision !== undefined) { + conditions.push('tie <= ?') + values.push(command.throughRevision) + } + if (command.beforeRank !== undefined) { + conditions.push('rank <= ?') + values.push(command.beforeRank) + } + if (command.eligibleThrough !== undefined) { + conditions.push('(ready=1 OR tie<=?)') + values.push(command.eligibleThrough) + } + return db + .prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog + WHERE ${conditions.join(' AND ')} ORDER BY ${command.orderRevision ? 'tie,key' : command.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`) + .iterate(...values, Math.max(1, Math.min(command.limit ?? 256, 256))) +} diff --git a/src/main/ai-vault-search/session-search-catalog-types.ts b/src/main/ai-vault-search/session-search-catalog-types.ts index 67e5b3f3440..e493e0b27f1 100644 --- a/src/main/ai-vault-search/session-search-catalog-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-types.ts @@ -28,6 +28,7 @@ export type SessionSearchCatalogRow = { tie?: number } export type SessionSearchInputReceipt = { consumer: string; revision: number } +export type SessionSearchCatalogPosition = { key: string; rank: number; tie: number } export type SessionSearchCatalogBaseCommand = ( | { kind: 'get'; scope: string; key: string } | { kind: 'get-page'; scope: string; keys: string[] } @@ -44,6 +45,7 @@ export type SessionSearchCatalogBaseCommand = ( kind: 'page' scope: string after?: string + afterOrder?: SessionSearchCatalogPosition parent?: string group?: string ready?: number @@ -51,6 +53,7 @@ export type SessionSearchCatalogBaseCommand = ( orderRevision?: boolean throughRevision?: number beforeRank?: number + eligibleThrough?: number limit?: number prefix?: string } diff --git a/src/main/ai-vault-search/session-search-catalog.ts b/src/main/ai-vault-search/session-search-catalog.ts index ea72a85e168..f331a166f41 100644 --- a/src/main/ai-vault-search/session-search-catalog.ts +++ b/src/main/ai-vault-search/session-search-catalog.ts @@ -1,3 +1,4 @@ +import { sessionSearchCatalogPageRows } from './session-search-catalog-query' import type SyncDatabase from '../sqlite/sync-database' import { transitionSessionSearchCatalog } from './session-search-catalog-transition' import { SessionSearchCatalogStorage } from './session-search-catalog-storage' @@ -248,45 +249,9 @@ export class SessionSearchCatalog extends SessionSearchCatalogStorage { } } case 'page': { - const conditions = ['scope = ?', 'key > ?'] - const values: (string | number)[] = [base.scope, base.after ?? ''] - if (base.prefix !== undefined) { - conditions.push('key >= ?', 'key < ?') - values.push(base.prefix, `${base.prefix}\uffff`) - } - for (const [column, value] of [ - ['parent', base.parent], - ['group_key', base.group], - ['ready', base.ready] - ] as const) { - if (value !== undefined) { - conditions.push(`${column} = ?`) - values.push(value) - } - } - if (base.throughRevision !== undefined) { - conditions.push('tie <= ?') - values.push(base.throughRevision) - } - if (base.beforeRank !== undefined) { - conditions.push('rank <= ?') - values.push(base.beforeRank) - } const rows: SessionSearchCatalogRow[] = [] let bytes = 1024 - const iterator = this.db - .prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog - WHERE ${conditions.join(' AND ')} ORDER BY ${base.orderRevision ? 'tie,key' : base.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`) - .iterate( - ...values, - Math.max( - 1, - Math.min( - base.limit ?? SESSION_SEARCH_CATALOG_PAGE_ROWS, - SESSION_SEARCH_CATALOG_PAGE_ROWS - ) - ) - ) + const iterator = sessionSearchCatalogPageRows(this.db, base) for (const value of iterator) { const row = this.decode(value) const next = this.bytes(row) diff --git a/src/main/ai-vault-search/session-search-index-pass.ts b/src/main/ai-vault-search/session-search-index-pass.ts index e947f323d85..e59a433aac1 100644 --- a/src/main/ai-vault-search/session-search-index-pass.ts +++ b/src/main/ai-vault-search/session-search-index-pass.ts @@ -30,6 +30,7 @@ export async function runSessionSearchIndexPass( } ): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> { const stats = createSessionParseStats() + queue.beginPass() // Retry once per cycle; rotating the failed path cannot monopolize a pass. const available = queue.size let consumed = 0 @@ -53,7 +54,9 @@ export async function runSessionSearchIndexPass( } const { candidate } = work const path = candidate.file.path - await store.prepareCoverage?.([path]) + if (!options.admit || !store.coverageKnown?.(path)) { + await store.prepareCoverage?.([path]) + } let row = store.fileRow(path) const cursor = row ? store.indexedFile(path, fileIdentity(candidate.file)) : null const entry = work.metadata diff --git a/src/main/ai-vault-search/session-search-pass.ts b/src/main/ai-vault-search/session-search-pass.ts index 3e22d138605..516dbabf640 100644 --- a/src/main/ai-vault-search/session-search-pass.ts +++ b/src/main/ai-vault-search/session-search-pass.ts @@ -179,9 +179,17 @@ export async function runSessionSearchPass( overdue: readOverdue, publications: state.candidates, admit: async (work) => { + const admission = await state.candidates.readAdmission(work.candidate) + if (!state.queue.selectionCurrent(admission.work)) { + return undefined + } + if (admission.work.work?.version !== work.version) { + return undefined + } + work = admission.work.work const { candidate } = work const path = candidate.file.path - const rootDir = await state.candidates.rootFor(candidate) + const rootDir = admission.root if (rootDir === undefined) { await state.queue.acknowledge(work, false) return undefined diff --git a/src/main/ai-vault-search/session-search-store-transitions.ts b/src/main/ai-vault-search/session-search-store-transitions.ts index bf0990ee9fc..e5dfaa2c9fb 100644 --- a/src/main/ai-vault-search/session-search-store-transitions.ts +++ b/src/main/ai-vault-search/session-search-store-transitions.ts @@ -94,7 +94,11 @@ export async function transitionSessionSearchCandidates( } } if (command.kind !== 'admit') { - return store.catalogCandidates.execute(command) + const result = store.catalogCandidates.execute(command) + if (command.kind === 'path' && command.read) { + return { ...result, work: await store.work({ kind: 'get', path: command.path }) } + } + return result } for (const receipt of command.rows) { let row = store.catalogCandidates.entry(receipt.key) @@ -149,6 +153,8 @@ export async function transitionSessionSearchWork( ): Promise { queue.attach(store) let work + let works + let eligibleThrough switch (command.kind) { case 'enqueue': if (store.fileRow(command.candidate.file.path)?.state === 'failed') { @@ -172,7 +178,8 @@ export async function transitionSessionSearchWork( work = await queue.get(command.path) break case 'next': - work = await queue.next() + eligibleThrough = command.eligibleThrough ?? queue.eligibleThrough + works = queue.next(store.connection, eligibleThrough) break case 'stats': break @@ -222,5 +229,12 @@ export async function transitionSessionSearchWork( await queue.rowChanged(command.path, command.row) break } - return { size: queue.size, left: queue.left, work } + return { + size: queue.size, + left: queue.left, + work, + works, + eligibleThrough, + revision: queue.revision + } } diff --git a/src/main/ai-vault-search/session-search-work-catalog.ts b/src/main/ai-vault-search/session-search-work-catalog.ts index 94a41c170fd..ab9bdee58f1 100644 --- a/src/main/ai-vault-search/session-search-work-catalog.ts +++ b/src/main/ai-vault-search/session-search-work-catalog.ts @@ -2,6 +2,9 @@ import { SessionSearchCatalogClient } from './session-search-catalog-client' import type { SessionFileCandidate } from '../ai-vault/session-scanner-types' import { fileIdentity } from './session-search-file-cursor' import { sessionSearchReadDecision } from './session-search-read-decision' +import type SyncDatabase from '../sqlite/sync-database' +import { sessionSearchCatalogPageRows } from './session-search-catalog-query' +import { SESSION_SEARCH_CATALOG_PAGE_BYTES } from './session-search-catalog' import type { SessionSearchFileRow, SessionSearchIndexStore } from './session-search-store' export type SessionSearchWork = { @@ -10,7 +13,14 @@ export type SessionSearchWork = { version: number metadata: boolean } -type Queued = { work: SessionSearchWork; lane: string; order: number; due: boolean; rank: number } +type Queued = { + work: SessionSearchWork + lane: string + order: number + due: boolean + rank: number + deferred: boolean +} /** A unique path's obligation lives in the writer even after it leaves the recent list. */ export class SessionSearchWorkCatalog { @@ -20,6 +30,10 @@ export class SessionSearchWorkCatalog { private chooseChanged = true private count = 0 private due = 0 + revision = 0 + get eligibleThrough(): number { + return this.order + } attach(store: SessionSearchIndexStore): void { this.catalog ??= new SessionSearchCatalogClient('read-work', (command) => @@ -63,9 +77,11 @@ export class SessionSearchWorkCatalog { lane: existing?.lane ?? (recent ? 'changed' : 'backlog'), order: existing?.order ?? ++this.order, due: row?.state === 'due', + deferred: false, rank: -(candidate.file.aliasMtimeMs ?? candidate.file.mtimeMs) } await this.put(path, value) + this.revision++ this.count += existing ? 0 : 1 this.due += Number(value.due) - Number(existing?.due ?? false) } @@ -75,16 +91,53 @@ export class SessionSearchWorkCatalog { return !!work && (work.force || work.metadata) } - async next(): Promise { - const preferred = this.chooseChanged ? 'changed' : 'backlog' - this.chooseChanged = !this.chooseChanged - for (const lane of [preferred, preferred === 'changed' ? 'backlog' : 'changed']) { - const page = await this.catalog!.page({ group: lane, orderRank: true, limit: 1 }) - if (page.rows.length) { - return page.rows[0].value.work + next(db: SyncDatabase, eligibleThrough = this.order): SessionSearchWork[] { + const lanes: ReturnType[] = [] + const works: SessionSearchWork[] = [] + let bytes = 1024 + let selected = this.chooseChanged ? 0 : 1 + const exhausted = [false, false] + try { + for (const lane of ['changed', 'backlog'] as const) { + lanes.push( + sessionSearchCatalogPageRows( + db, + { kind: 'page', scope: 'read-work', orderRank: true, eligibleThrough, limit: 256 }, + lane + ) + ) + } + while (works.length < 256) { + if (exhausted[selected]) { + selected = 1 - selected + if (exhausted[selected]) { + break + } + } + const row = lanes[selected].next() + if (row.done) { + exhausted[selected] = true + continue + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this queue encoded its own catalog record. + const work = (JSON.parse(String(row.value.value)) as Queued).work + const size = JSON.stringify(work).length * 2 + 64 + if (works.length && bytes + size > SESSION_SEARCH_CATALOG_PAGE_BYTES) { + break + } + works.push(work) + bytes += size + if (bytes >= SESSION_SEARCH_CATALOG_PAGE_BYTES) { + break + } + selected = 1 - selected + } + } finally { + for (const lane of lanes) { + lane.return?.() } } - return undefined + return works } async get(path: string): Promise { @@ -97,6 +150,7 @@ export class SessionSearchWorkCatalog { if (entry?.work.version !== work.version) { return } + this.chooseChanged = !this.chooseChanged if (retry) { await this.put(path, { ...entry, @@ -104,20 +158,22 @@ export class SessionSearchWorkCatalog { lane: 'backlog', order: ++this.order, due: false, + deferred: true, rank: this.order }) this.due -= Number(entry.due) } else { - await this.remove(path) + await this.remove(path, false) } } - async remove(path: string): Promise { + async remove(path: string, invalidate = true): Promise { const entry = await this.catalog?.get(path) if (!entry) { return } await this.catalog!.delete(path) + this.revision += Number(invalidate) this.count-- this.due -= Number(entry.due) } @@ -133,6 +189,11 @@ export class SessionSearchWorkCatalog { } private put(path: string, value: Queued): Promise { - return this.catalog!.put(path, value, { group: value.lane, rank: value.rank, tie: value.order }) + return this.catalog!.put(path, value, { + group: value.lane, + rank: value.rank, + tie: value.order, + ready: Number(!value.deferred) + }) } } diff --git a/src/main/ai-vault-search/session-search-work-queue.ts b/src/main/ai-vault-search/session-search-work-queue.ts index 9ce93cae26f..e2b01c33e5f 100644 --- a/src/main/ai-vault-search/session-search-work-queue.ts +++ b/src/main/ai-vault-search/session-search-work-queue.ts @@ -11,7 +11,7 @@ export type SessionSearchWorkCommand = recent: boolean metadata: boolean } - | { kind: 'next' } + | { kind: 'next'; eligibleThrough?: number } | { kind: 'stats' } | { kind: 'get'; path: string } | { @@ -22,13 +22,33 @@ export type SessionSearchWorkCommand = } | { kind: 'remove'; path: string } | { kind: 'row'; path: string; row?: SessionSearchFileRow } -export type SessionSearchWorkResult = { size: number; left: number; work?: SessionSearchWork } +export type SessionSearchWorkResult = { + size: number + left: number + work?: SessionSearchWork + works?: SessionSearchWork[] + revision?: number + eligibleThrough?: number +} /** Only active reads cross the boundary; the writer owns ready work and its transitions. */ export class SessionSearchWorkQueue { private store: SessionSearchIndexStore | undefined private count = 0 private remaining = 0 + private page: SessionSearchWork[] = [] + private revision: number | undefined + private selectedRevision: number | undefined + private eligibleThrough: number | undefined + beginPass(): void { + this.page = [] + this.eligibleThrough = undefined + } + selectionCurrent(result: SessionSearchWorkResult): boolean { + const current = this.selectedRevision === result.revision + this.update(result) + return current + } attach(store: SessionSearchIndexStore): void { this.store ??= store } @@ -39,6 +59,10 @@ export class SessionSearchWorkQueue { return this.remaining } update(result: SessionSearchWorkResult): void { + if (this.revision !== result.revision) { + this.page = [] + this.revision = result.revision + } this.count = result.size this.remaining = result.left } @@ -62,7 +86,13 @@ export class SessionSearchWorkQueue { return !!work && (work.force || work.metadata) } async next(): Promise { - return (await this.request({ kind: 'next' })).work + if (!this.page.length) { + const result = await this.request({ kind: 'next', eligibleThrough: this.eligibleThrough }) + this.eligibleThrough ??= result.eligibleThrough + this.page = result.works ?? (result.work ? [result.work] : []) + } + this.selectedRevision = this.revision + return this.page.shift() } async get(path: string): Promise { return (await this.request({ kind: 'get', path })).work diff --git a/src/main/ai-vault-search/session-search-worker-entry.ts b/src/main/ai-vault-search/session-search-worker-entry.ts index 9cfc0022ec4..b6fc40c9b65 100644 --- a/src/main/ai-vault-search/session-search-worker-entry.ts +++ b/src/main/ai-vault-search/session-search-worker-entry.ts @@ -251,6 +251,7 @@ async function execute({ id, command: c }: SearchWorkerRequest): Promise delta(row.path, row)), diff --git a/src/main/ai-vault-search/session-search-worker-store-base.ts b/src/main/ai-vault-search/session-search-worker-store-base.ts index e55bcabb04a..e42debc0e0f 100644 --- a/src/main/ai-vault-search/session-search-worker-store-base.ts +++ b/src/main/ai-vault-search/session-search-worker-store-base.ts @@ -222,6 +222,10 @@ export class SessionSearchWorkerStoreBase extends SessionSearchCursorSnapshot { if (!response.candidates) { throw new Error('Session search candidate catalog unavailable') } + if (command.kind === 'path' && command.read) { + this.projectCoverage(command.path, response.coverage?.[0]) + this.trimCoverage() + } return response.candidates } async work(command: SessionSearchWorkCommand): Promise {