Resume indexed session reads in fair bounded batches

Use indexed ordered cursors and retain only the next read page, with live
admission and a per-pass retry cutoff. Alternate changed and older work
before byte admission so a large page cannot starve history.

Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
Brennan Benson
2026-11-11 01:32:32 -08:00
co-authored by Jinjing
parent 585f3cd69f
commit fa9bd2d522
14 changed files with 433 additions and 109 deletions
@@ -45,10 +45,14 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli
super.reset()
this.sourceOrders.clear()
}
async rootFor(candidate: SessionFileCandidate): Promise<string | undefined> {
return (
await this.request!({ kind: 'path', path: candidate.file.path, agent: candidate.agent })
).rows[0]?.root
async readAdmission(candidate: SessionFileCandidate) {
const result = await this.request!({
kind: 'path',
path: candidate.file.path,
agent: candidate.agent,
read: true
})
return { root: result.rows.find((row) => row.selected)?.root, work: result.work! }
}
async has(path: string, agent?: string | null): Promise<boolean> {
return (await this.request!({ kind: 'path', path, agent })).rows.length > 0
@@ -15,6 +15,10 @@ import {
import { SessionSearchWorkerClient } from './session-search-worker-client'
import * as filesystem from '../native-chat/wsl-transcript-fs-access'
import { SessionSearchIndexer } from './session-search-indexer'
import { SessionSearchWorkerStore } from './session-search-worker-store'
import { SessionSearchWorkQueue } from './session-search-work-queue'
import { configureSessionSearchWorkerFixture } from './session-search-worker.test-fixture'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
it.each([40, 400])(
'batches discovery and replacement metadata for %s retained files',
@@ -146,53 +150,56 @@ it.each([40, 400])(
}
)
it('searches the newest known transcript before an older candidate page', async () => {
const harness = await openSessionSearchIndexerHarness('ss-newest-page')
const clock = new FakeSessionSearchClock()
clock.costPerNowMs = 1000
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
passDeadlineMs: 1000,
discoveryDeadlineMs: Infinity
})
try {
for (let index = 0; index < 300; index++) {
const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`)
await writeClaudeTranscript(
path,
[index === 299 ? 'newestknownneedle' : 'olderknownneedle'],
`session-${index}`
)
const time = new Date(index === 299 ? '2025-01-02' : '2025-01-01')
await utimes(path, time, time)
it.each([40, 400])(
'searches the newest known transcript before an older candidate page at %s files',
async (population) => {
const harness = await openSessionSearchIndexerHarness('ss-newest-page')
const clock = new FakeSessionSearchClock()
clock.costPerNowMs = 1000
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
passDeadlineMs: 1000,
discoveryDeadlineMs: Infinity
})
try {
for (let index = 0; index < population; index++) {
const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`)
await writeClaudeTranscript(
path,
[index === population - 1 ? 'newestknownneedle' : 'olderknownneedle'],
`session-${index}`
)
const time = new Date(index === population - 1 ? '2025-01-02' : '2025-01-01')
await utimes(path, time, time)
}
await subject.start()
expect(
harness.read((db) =>
db
.prepare(
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'"
)
.get()
)
).toEqual({ n: 2 })
expect(
harness.read((db) =>
db
.prepare(
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'"
)
.get()
)
).toEqual({ n: 0 })
} finally {
await subject.close()
await harness.cleanup()
}
await subject.start()
expect(
harness.read((db) =>
db
.prepare(
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'"
)
.get()
)
).toEqual({ n: 2 })
expect(
harness.read((db) =>
db
.prepare(
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'"
)
.get()
)
).toEqual({ n: 0 })
} finally {
await subject.close()
await harness.cleanup()
}
})
)
it('redelivers an unaccepted observation and retains a newer rewrite after an older acknowledgement', async () => {
const harness = await openSessionSearchIndexerHarness('ss-observation-ack')
@@ -373,3 +380,157 @@ it.each([40, 400])(
}
}
)
it.each([40, 400])(
'searches a new change and owed history in the same limited pass with %s changes',
async (population) => {
const harness = await openSessionSearchIndexerHarness('ss-read-fairness')
const clock = new FakeSessionSearchClock()
clock.costPerNowMs = 1000
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: harness.roots,
historyDays: null,
clock,
passDeadlineMs: 2000,
discoveryDeadlineMs: Infinity,
fullSweepEveryCycles: 1000
})
const directory = join(
harness.claudeProjectDir,
...Array.from({ length: 8 }, () => 'p'.repeat(90))
)
const hits = () =>
harness.read((db) =>
db
.prepare(`SELECT DISTINCT s.session_id AS id FROM messages_fts
JOIN messages m ON m.id=messages_fts.rowid
JOIN sessions s ON s.id=m.session_row_id
WHERE messages_fts MATCH 'fairnessneedle' ORDER BY s.session_id`)
.all()
)
try {
for (let index = 0; index < 3; index++) {
const path = join(directory, `backlog-${index}.jsonl`)
await writeClaudeTranscript(path, ['fairnessneedle'], `backlog-${index}`)
const time = new Date(Date.UTC(2024, 0, 1) - index * 1000)
await utimes(path, time, time)
}
await subject.start()
expect(hits()).toEqual([{ id: 'backlog-0' }, { id: 'backlog-1' }])
for (let index = 0; index < population; index++) {
const path = join(directory, `changed-${String(index).padStart(3, '0')}.jsonl`)
await writeClaudeTranscript(path, ['fairnessneedle'], `changed-${index}`)
const time = new Date(Date.UTC(2025, 0, 1) + index * 1000)
await utimes(path, time, time)
}
await subject.reconcile()
expect(hits()).toEqual([
{ id: 'backlog-0' },
{ id: 'backlog-1' },
{ id: 'backlog-2' },
{ id: `changed-${population - 1}` }
])
} finally {
await subject.close()
await harness.cleanup()
}
}
)
it.each([40, 400])(
'preserves ranked read obligations across bounded pages for %s candidates',
async (population) => {
const harness = await openSessionSearchIndexerHarness('ss-read-pages')
const store = new SessionSearchWorkerStore(harness.databasePath, (error) => {
throw error
})
const queue = new SessionSearchWorkQueue()
queue.attach(store)
const traffic = vi.spyOn(SessionSearchWorkerClient.prototype, 'request')
const candidates: SessionFileCandidate[] = []
let largestResponse = 0
configureSessionSearchWorkerFixture({
prelude: '',
data: undefined,
onResponse: (response) => {
largestResponse = Math.max(largestResponse, JSON.stringify(response).length * 2)
}
})
try {
await store.prepare()
const prefix = join(harness.root, ...Array.from({ length: 10 }, () => 'p'.repeat(100)))
for (let index = 0; index < population; index++) {
const candidate: SessionFileCandidate = {
agent: 'claude',
codexHome: null,
file: {
path: join(prefix, `${String(population - index).padStart(4, '0')}.jsonl`),
mtimeMs: 10000 - Math.floor(index / 4),
modifiedAt: '1970-01-01T00:00:10.000Z',
sizeBytes: 20
}
}
if (index === 1 || index === 3) {
const segments = index === 1 ? 12000 : 6000
const windowsRoot = `C:\\${Array.from({ length: segments }, () => 'p').join('\\')}`
candidate.agent = 'antigravity'
candidate.file.path = `${windowsRoot}\\conversation-${index}.pb`
candidate.antigravityHistoryPath = `${windowsRoot}\\history-${index}.pb`
}
candidates.push(candidate)
await queue.enqueue(candidate, store, false, index % 2 === 0)
}
traffic.mockClear()
queue.beginPass()
const first = await queue.next()
expect(first?.candidate.file.path).toBe(candidates[0].file.path)
queue.beginPass()
expect(await queue.next()).toEqual(first)
await queue.acknowledge(first!, true)
const observed: string[] = []
for (let index = 1; index < population; index++) {
let work = await queue.next()
if (index === 1) {
await queue.enqueue(candidates[1], store, false, false, true)
await queue.enqueue(candidates[2], store, true, true)
await queue.acknowledge(work!, false)
work = await queue.next()
expect(work).toMatchObject({ candidate: candidates[1], metadata: true })
}
if (index === 2) {
expect(work).toMatchObject({ candidate: candidates[2], force: true })
}
observed.push(work!.candidate.file.path)
await queue.acknowledge(work!, false)
}
expect(observed).toEqual(candidates.slice(1).map((candidate) => candidate.file.path))
expect(await queue.next()).toBeUndefined()
// Each lane needs a bounded projection per row/byte page; per-file accepted acknowledgements are separate.
const pages =
Math.ceil(population / 256) +
Math.ceil((JSON.stringify(candidates).length * 2) / (128 * 1024))
expect(
traffic.mock.calls.filter(
([command]) => command.op === 'work' && command.command.kind === 'next'
).length
).toBeLessThanOrEqual(40 + 12 * pages)
queue.beginPass()
const retry = await queue.next()
expect(retry?.candidate.file.path).toBe(candidates[0].file.path)
await queue.enqueue(candidates[0], store, false, true, true)
await queue.acknowledge(retry!, false)
const latest = await queue.next()
expect(latest).toMatchObject({ candidate: candidates[0], metadata: true })
await queue.acknowledge(latest!, false)
expect(await queue.next()).toBeUndefined()
expect(largestResponse).toBeLessThanOrEqual(256 * 1024)
} finally {
configureSessionSearchWorkerFixture()
traffic.mockRestore()
await store.flush()
store.close()
await harness.cleanup()
}
}
)
@@ -38,7 +38,7 @@ export type SessionSearchCandidateCommand =
rows: { path: string; agents?: string[]; observation?: SessionSearchRemovalReceipt }[]
}
| { kind: 'unowned-page'; rows: { path: string; agent?: string | null }[] }
| { kind: 'path'; path: string; agent?: string | null }
| { kind: 'path'; path: string; agent?: string | null; read?: boolean }
| { kind: 'group'; group: string; after?: string }
| { kind: 'changes'; after?: string; root?: string; agent?: string }
| { kind: 'rerank'; root: string; agent: string; order: [string, number][]; after?: string }
@@ -3,6 +3,7 @@ import type {
SessionSearchCatalogResult,
SessionSearchCatalogRow
} from './session-search-catalog'
import type { SessionSearchCatalogPosition } from './session-search-catalog-types'
import { sessionSearchRecordPages } from './session-search-catalog-pages'
import {
SESSION_SEARCH_CATALOG_PAGE_BYTES,
@@ -161,6 +162,7 @@ export class SessionSearchCatalogClient<T> {
async page(
options: {
after?: string
afterOrder?: SessionSearchCatalogPosition
parent?: string
group?: string
ready?: number
@@ -169,12 +171,24 @@ export class SessionSearchCatalogClient<T> {
limit?: number
prefix?: string
throughRevision?: number
eligibleThrough?: number
} = {}
): Promise<{ rows: { key: string; value: T }[]; after?: string }> {
): Promise<{
rows: { key: string; value: T }[]
after?: string
position?: SessionSearchCatalogPosition
}> {
const result = await this.request({ kind: 'page', scope: this.scope, ...options })
return {
rows: (result.rows ?? []).map((row) => ({ key: row.key, value: this.decode(row) })),
after: result.after
after: result.after,
position: result.rows?.length
? {
key: result.rows.at(-1)!.key,
rank: result.rows.at(-1)!.rank,
tie: result.rows.at(-1)!.tie ?? 0
}
: undefined
}
}
@@ -0,0 +1,56 @@
import type SyncDatabase from '../sqlite/sync-database'
import type { SessionSearchCatalogBaseCommand } from './session-search-catalog-types'
export function sessionSearchCatalogPageRows(
db: SyncDatabase,
command: Extract<SessionSearchCatalogBaseCommand, { kind: 'page' }>,
lane?: 'changed' | 'backlog'
) {
const conditions = ['scope = ?']
const values: (string | number)[] = [command.scope]
if (lane) {
// Simultaneous lane cursors must not share one cached native statement.
conditions.push(lane === 'changed' ? "group_key='changed'" : "group_key='backlog'")
}
if (command.afterOrder) {
const { rank, tie, key } = command.afterOrder
conditions.push(command.orderRevision ? '(tie,key) > (?,?)' : '(rank,tie,key) > (?,?,?)')
values.push(...(command.orderRevision ? [tie, key] : [rank, tie, key]))
} else if (command.after !== undefined) {
if (command.orderRank || command.orderRevision) {
throw new Error('Ordered session catalog continuation requires its full position')
}
conditions.push('key > ?')
values.push(command.after)
}
if (command.prefix !== undefined) {
conditions.push('key >= ?', 'key < ?')
values.push(command.prefix, `${command.prefix}\uffff`)
}
for (const [column, value] of [
['parent', command.parent],
['group_key', command.group],
['ready', command.ready]
] as const) {
if (value !== undefined) {
conditions.push(`${column} = ?`)
values.push(value)
}
}
if (command.throughRevision !== undefined) {
conditions.push('tie <= ?')
values.push(command.throughRevision)
}
if (command.beforeRank !== undefined) {
conditions.push('rank <= ?')
values.push(command.beforeRank)
}
if (command.eligibleThrough !== undefined) {
conditions.push('(ready=1 OR tie<=?)')
values.push(command.eligibleThrough)
}
return db
.prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog
WHERE ${conditions.join(' AND ')} ORDER BY ${command.orderRevision ? 'tie,key' : command.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`)
.iterate(...values, Math.max(1, Math.min(command.limit ?? 256, 256)))
}
@@ -28,6 +28,7 @@ export type SessionSearchCatalogRow = {
tie?: number
}
export type SessionSearchInputReceipt = { consumer: string; revision: number }
export type SessionSearchCatalogPosition = { key: string; rank: number; tie: number }
export type SessionSearchCatalogBaseCommand = (
| { kind: 'get'; scope: string; key: string }
| { kind: 'get-page'; scope: string; keys: string[] }
@@ -44,6 +45,7 @@ export type SessionSearchCatalogBaseCommand = (
kind: 'page'
scope: string
after?: string
afterOrder?: SessionSearchCatalogPosition
parent?: string
group?: string
ready?: number
@@ -51,6 +53,7 @@ export type SessionSearchCatalogBaseCommand = (
orderRevision?: boolean
throughRevision?: number
beforeRank?: number
eligibleThrough?: number
limit?: number
prefix?: string
}
@@ -1,3 +1,4 @@
import { sessionSearchCatalogPageRows } from './session-search-catalog-query'
import type SyncDatabase from '../sqlite/sync-database'
import { transitionSessionSearchCatalog } from './session-search-catalog-transition'
import { SessionSearchCatalogStorage } from './session-search-catalog-storage'
@@ -248,45 +249,9 @@ export class SessionSearchCatalog extends SessionSearchCatalogStorage {
}
}
case 'page': {
const conditions = ['scope = ?', 'key > ?']
const values: (string | number)[] = [base.scope, base.after ?? '']
if (base.prefix !== undefined) {
conditions.push('key >= ?', 'key < ?')
values.push(base.prefix, `${base.prefix}\uffff`)
}
for (const [column, value] of [
['parent', base.parent],
['group_key', base.group],
['ready', base.ready]
] as const) {
if (value !== undefined) {
conditions.push(`${column} = ?`)
values.push(value)
}
}
if (base.throughRevision !== undefined) {
conditions.push('tie <= ?')
values.push(base.throughRevision)
}
if (base.beforeRank !== undefined) {
conditions.push('rank <= ?')
values.push(base.beforeRank)
}
const rows: SessionSearchCatalogRow[] = []
let bytes = 1024
const iterator = this.db
.prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog
WHERE ${conditions.join(' AND ')} ORDER BY ${base.orderRevision ? 'tie,key' : base.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`)
.iterate(
...values,
Math.max(
1,
Math.min(
base.limit ?? SESSION_SEARCH_CATALOG_PAGE_ROWS,
SESSION_SEARCH_CATALOG_PAGE_ROWS
)
)
)
const iterator = sessionSearchCatalogPageRows(this.db, base)
for (const value of iterator) {
const row = this.decode(value)
const next = this.bytes(row)
@@ -30,6 +30,7 @@ export async function runSessionSearchIndexPass(
}
): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> {
const stats = createSessionParseStats()
queue.beginPass()
// Retry once per cycle; rotating the failed path cannot monopolize a pass.
const available = queue.size
let consumed = 0
@@ -53,7 +54,9 @@ export async function runSessionSearchIndexPass(
}
const { candidate } = work
const path = candidate.file.path
await store.prepareCoverage?.([path])
if (!options.admit || !store.coverageKnown?.(path)) {
await store.prepareCoverage?.([path])
}
let row = store.fileRow(path)
const cursor = row ? store.indexedFile(path, fileIdentity(candidate.file)) : null
const entry = work.metadata
@@ -179,9 +179,17 @@ export async function runSessionSearchPass(
overdue: readOverdue,
publications: state.candidates,
admit: async (work) => {
const admission = await state.candidates.readAdmission(work.candidate)
if (!state.queue.selectionCurrent(admission.work)) {
return undefined
}
if (admission.work.work?.version !== work.version) {
return undefined
}
work = admission.work.work
const { candidate } = work
const path = candidate.file.path
const rootDir = await state.candidates.rootFor(candidate)
const rootDir = admission.root
if (rootDir === undefined) {
await state.queue.acknowledge(work, false)
return undefined
@@ -94,7 +94,11 @@ export async function transitionSessionSearchCandidates(
}
}
if (command.kind !== 'admit') {
return store.catalogCandidates.execute(command)
const result = store.catalogCandidates.execute(command)
if (command.kind === 'path' && command.read) {
return { ...result, work: await store.work({ kind: 'get', path: command.path }) }
}
return result
}
for (const receipt of command.rows) {
let row = store.catalogCandidates.entry(receipt.key)
@@ -149,6 +153,8 @@ export async function transitionSessionSearchWork(
): Promise<SessionSearchWorkResult> {
queue.attach(store)
let work
let works
let eligibleThrough
switch (command.kind) {
case 'enqueue':
if (store.fileRow(command.candidate.file.path)?.state === 'failed') {
@@ -172,7 +178,8 @@ export async function transitionSessionSearchWork(
work = await queue.get(command.path)
break
case 'next':
work = await queue.next()
eligibleThrough = command.eligibleThrough ?? queue.eligibleThrough
works = queue.next(store.connection, eligibleThrough)
break
case 'stats':
break
@@ -222,5 +229,12 @@ export async function transitionSessionSearchWork(
await queue.rowChanged(command.path, command.row)
break
}
return { size: queue.size, left: queue.left, work }
return {
size: queue.size,
left: queue.left,
work,
works,
eligibleThrough,
revision: queue.revision
}
}
@@ -2,6 +2,9 @@ import { SessionSearchCatalogClient } from './session-search-catalog-client'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import { fileIdentity } from './session-search-file-cursor'
import { sessionSearchReadDecision } from './session-search-read-decision'
import type SyncDatabase from '../sqlite/sync-database'
import { sessionSearchCatalogPageRows } from './session-search-catalog-query'
import { SESSION_SEARCH_CATALOG_PAGE_BYTES } from './session-search-catalog'
import type { SessionSearchFileRow, SessionSearchIndexStore } from './session-search-store'
export type SessionSearchWork = {
@@ -10,7 +13,14 @@ export type SessionSearchWork = {
version: number
metadata: boolean
}
type Queued = { work: SessionSearchWork; lane: string; order: number; due: boolean; rank: number }
type Queued = {
work: SessionSearchWork
lane: string
order: number
due: boolean
rank: number
deferred: boolean
}
/** A unique path's obligation lives in the writer even after it leaves the recent list. */
export class SessionSearchWorkCatalog {
@@ -20,6 +30,10 @@ export class SessionSearchWorkCatalog {
private chooseChanged = true
private count = 0
private due = 0
revision = 0
get eligibleThrough(): number {
return this.order
}
attach(store: SessionSearchIndexStore): void {
this.catalog ??= new SessionSearchCatalogClient('read-work', (command) =>
@@ -63,9 +77,11 @@ export class SessionSearchWorkCatalog {
lane: existing?.lane ?? (recent ? 'changed' : 'backlog'),
order: existing?.order ?? ++this.order,
due: row?.state === 'due',
deferred: false,
rank: -(candidate.file.aliasMtimeMs ?? candidate.file.mtimeMs)
}
await this.put(path, value)
this.revision++
this.count += existing ? 0 : 1
this.due += Number(value.due) - Number(existing?.due ?? false)
}
@@ -75,16 +91,53 @@ export class SessionSearchWorkCatalog {
return !!work && (work.force || work.metadata)
}
async next(): Promise<SessionSearchWork | undefined> {
const preferred = this.chooseChanged ? 'changed' : 'backlog'
this.chooseChanged = !this.chooseChanged
for (const lane of [preferred, preferred === 'changed' ? 'backlog' : 'changed']) {
const page = await this.catalog!.page({ group: lane, orderRank: true, limit: 1 })
if (page.rows.length) {
return page.rows[0].value.work
next(db: SyncDatabase, eligibleThrough = this.order): SessionSearchWork[] {
const lanes: ReturnType<typeof sessionSearchCatalogPageRows>[] = []
const works: SessionSearchWork[] = []
let bytes = 1024
let selected = this.chooseChanged ? 0 : 1
const exhausted = [false, false]
try {
for (const lane of ['changed', 'backlog'] as const) {
lanes.push(
sessionSearchCatalogPageRows(
db,
{ kind: 'page', scope: 'read-work', orderRank: true, eligibleThrough, limit: 256 },
lane
)
)
}
while (works.length < 256) {
if (exhausted[selected]) {
selected = 1 - selected
if (exhausted[selected]) {
break
}
}
const row = lanes[selected].next()
if (row.done) {
exhausted[selected] = true
continue
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this queue encoded its own catalog record.
const work = (JSON.parse(String(row.value.value)) as Queued).work
const size = JSON.stringify(work).length * 2 + 64
if (works.length && bytes + size > SESSION_SEARCH_CATALOG_PAGE_BYTES) {
break
}
works.push(work)
bytes += size
if (bytes >= SESSION_SEARCH_CATALOG_PAGE_BYTES) {
break
}
selected = 1 - selected
}
} finally {
for (const lane of lanes) {
lane.return?.()
}
}
return undefined
return works
}
async get(path: string): Promise<SessionSearchWork | undefined> {
@@ -97,6 +150,7 @@ export class SessionSearchWorkCatalog {
if (entry?.work.version !== work.version) {
return
}
this.chooseChanged = !this.chooseChanged
if (retry) {
await this.put(path, {
...entry,
@@ -104,20 +158,22 @@ export class SessionSearchWorkCatalog {
lane: 'backlog',
order: ++this.order,
due: false,
deferred: true,
rank: this.order
})
this.due -= Number(entry.due)
} else {
await this.remove(path)
await this.remove(path, false)
}
}
async remove(path: string): Promise<void> {
async remove(path: string, invalidate = true): Promise<void> {
const entry = await this.catalog?.get(path)
if (!entry) {
return
}
await this.catalog!.delete(path)
this.revision += Number(invalidate)
this.count--
this.due -= Number(entry.due)
}
@@ -133,6 +189,11 @@ export class SessionSearchWorkCatalog {
}
private put(path: string, value: Queued): Promise<void> {
return this.catalog!.put(path, value, { group: value.lane, rank: value.rank, tie: value.order })
return this.catalog!.put(path, value, {
group: value.lane,
rank: value.rank,
tie: value.order,
ready: Number(!value.deferred)
})
}
}
@@ -11,7 +11,7 @@ export type SessionSearchWorkCommand =
recent: boolean
metadata: boolean
}
| { kind: 'next' }
| { kind: 'next'; eligibleThrough?: number }
| { kind: 'stats' }
| { kind: 'get'; path: string }
| {
@@ -22,13 +22,33 @@ export type SessionSearchWorkCommand =
}
| { kind: 'remove'; path: string }
| { kind: 'row'; path: string; row?: SessionSearchFileRow }
export type SessionSearchWorkResult = { size: number; left: number; work?: SessionSearchWork }
export type SessionSearchWorkResult = {
size: number
left: number
work?: SessionSearchWork
works?: SessionSearchWork[]
revision?: number
eligibleThrough?: number
}
/** Only active reads cross the boundary; the writer owns ready work and its transitions. */
export class SessionSearchWorkQueue {
private store: SessionSearchIndexStore | undefined
private count = 0
private remaining = 0
private page: SessionSearchWork[] = []
private revision: number | undefined
private selectedRevision: number | undefined
private eligibleThrough: number | undefined
beginPass(): void {
this.page = []
this.eligibleThrough = undefined
}
selectionCurrent(result: SessionSearchWorkResult): boolean {
const current = this.selectedRevision === result.revision
this.update(result)
return current
}
attach(store: SessionSearchIndexStore): void {
this.store ??= store
}
@@ -39,6 +59,10 @@ export class SessionSearchWorkQueue {
return this.remaining
}
update(result: SessionSearchWorkResult): void {
if (this.revision !== result.revision) {
this.page = []
this.revision = result.revision
}
this.count = result.size
this.remaining = result.left
}
@@ -62,7 +86,13 @@ export class SessionSearchWorkQueue {
return !!work && (work.force || work.metadata)
}
async next(): Promise<SessionSearchWork | undefined> {
return (await this.request({ kind: 'next' })).work
if (!this.page.length) {
const result = await this.request({ kind: 'next', eligibleThrough: this.eligibleThrough })
this.eligibleThrough ??= result.eligibleThrough
this.page = result.works ?? (result.work ? [result.work] : [])
}
this.selectedRevision = this.revision
return this.page.shift()
}
async get(path: string): Promise<SessionSearchWork | undefined> {
return (await this.request({ kind: 'get', path })).work
@@ -251,6 +251,7 @@ async function execute({ id, command: c }: SearchWorkerRequest): Promise<SearchW
id,
reset,
candidates: result,
coverage: c.command.kind === 'path' && c.command.read ? rows(c.command.path) : undefined,
deltas: path
? [delta(path, previous)]
: result.withdrawn?.map((row) => delta(row.path, row)),
@@ -222,6 +222,10 @@ export class SessionSearchWorkerStoreBase extends SessionSearchCursorSnapshot {
if (!response.candidates) {
throw new Error('Session search candidate catalog unavailable')
}
if (command.kind === 'path' && command.read) {
this.projectCoverage(command.path, response.coverage?.[0])
this.trimCoverage()
}
return response.candidates
}
async work(command: SessionSearchWorkCommand): Promise<SessionSearchWorkResult> {