mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
Resume indexed session reads in fair bounded batches
Use indexed ordered cursors and retain only the next read page, with live admission and a per-pass retry cutoff. Alternate changed and older work before byte admission so a large page cannot starve history. Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
co-authored by
Jinjing
parent
585f3cd69f
commit
fa9bd2d522
@@ -45,10 +45,14 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli
|
||||
super.reset()
|
||||
this.sourceOrders.clear()
|
||||
}
|
||||
async rootFor(candidate: SessionFileCandidate): Promise<string | undefined> {
|
||||
return (
|
||||
await this.request!({ kind: 'path', path: candidate.file.path, agent: candidate.agent })
|
||||
).rows[0]?.root
|
||||
async readAdmission(candidate: SessionFileCandidate) {
|
||||
const result = await this.request!({
|
||||
kind: 'path',
|
||||
path: candidate.file.path,
|
||||
agent: candidate.agent,
|
||||
read: true
|
||||
})
|
||||
return { root: result.rows.find((row) => row.selected)?.root, work: result.work! }
|
||||
}
|
||||
async has(path: string, agent?: string | null): Promise<boolean> {
|
||||
return (await this.request!({ kind: 'path', path, agent })).rows.length > 0
|
||||
|
||||
@@ -15,6 +15,10 @@ import {
|
||||
import { SessionSearchWorkerClient } from './session-search-worker-client'
|
||||
import * as filesystem from '../native-chat/wsl-transcript-fs-access'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import { SessionSearchWorkerStore } from './session-search-worker-store'
|
||||
import { SessionSearchWorkQueue } from './session-search-work-queue'
|
||||
import { configureSessionSearchWorkerFixture } from './session-search-worker.test-fixture'
|
||||
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
|
||||
|
||||
it.each([40, 400])(
|
||||
'batches discovery and replacement metadata for %s retained files',
|
||||
@@ -146,53 +150,56 @@ it.each([40, 400])(
|
||||
}
|
||||
)
|
||||
|
||||
it('searches the newest known transcript before an older candidate page', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-newest-page')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
clock.costPerNowMs = 1000
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock,
|
||||
passDeadlineMs: 1000,
|
||||
discoveryDeadlineMs: Infinity
|
||||
})
|
||||
try {
|
||||
for (let index = 0; index < 300; index++) {
|
||||
const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`)
|
||||
await writeClaudeTranscript(
|
||||
path,
|
||||
[index === 299 ? 'newestknownneedle' : 'olderknownneedle'],
|
||||
`session-${index}`
|
||||
)
|
||||
const time = new Date(index === 299 ? '2025-01-02' : '2025-01-01')
|
||||
await utimes(path, time, time)
|
||||
it.each([40, 400])(
|
||||
'searches the newest known transcript before an older candidate page at %s files',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-newest-page')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
clock.costPerNowMs = 1000
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock,
|
||||
passDeadlineMs: 1000,
|
||||
discoveryDeadlineMs: Infinity
|
||||
})
|
||||
try {
|
||||
for (let index = 0; index < population; index++) {
|
||||
const path = join(harness.claudeProjectDir, `${String(index).padStart(3, '0')}.jsonl`)
|
||||
await writeClaudeTranscript(
|
||||
path,
|
||||
[index === population - 1 ? 'newestknownneedle' : 'olderknownneedle'],
|
||||
`session-${index}`
|
||||
)
|
||||
const time = new Date(index === population - 1 ? '2025-01-02' : '2025-01-01')
|
||||
await utimes(path, time, time)
|
||||
}
|
||||
await subject.start()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 2 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 0 })
|
||||
} finally {
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
await subject.start()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'newestknownneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 2 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'olderknownneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 0 })
|
||||
} finally {
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
it('redelivers an unaccepted observation and retains a newer rewrite after an older acknowledgement', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-observation-ack')
|
||||
@@ -373,3 +380,157 @@ it.each([40, 400])(
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.each([40, 400])(
|
||||
'searches a new change and owed history in the same limited pass with %s changes',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-read-fairness')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
clock.costPerNowMs = 1000
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock,
|
||||
passDeadlineMs: 2000,
|
||||
discoveryDeadlineMs: Infinity,
|
||||
fullSweepEveryCycles: 1000
|
||||
})
|
||||
const directory = join(
|
||||
harness.claudeProjectDir,
|
||||
...Array.from({ length: 8 }, () => 'p'.repeat(90))
|
||||
)
|
||||
const hits = () =>
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(`SELECT DISTINCT s.session_id AS id FROM messages_fts
|
||||
JOIN messages m ON m.id=messages_fts.rowid
|
||||
JOIN sessions s ON s.id=m.session_row_id
|
||||
WHERE messages_fts MATCH 'fairnessneedle' ORDER BY s.session_id`)
|
||||
.all()
|
||||
)
|
||||
try {
|
||||
for (let index = 0; index < 3; index++) {
|
||||
const path = join(directory, `backlog-${index}.jsonl`)
|
||||
await writeClaudeTranscript(path, ['fairnessneedle'], `backlog-${index}`)
|
||||
const time = new Date(Date.UTC(2024, 0, 1) - index * 1000)
|
||||
await utimes(path, time, time)
|
||||
}
|
||||
await subject.start()
|
||||
expect(hits()).toEqual([{ id: 'backlog-0' }, { id: 'backlog-1' }])
|
||||
for (let index = 0; index < population; index++) {
|
||||
const path = join(directory, `changed-${String(index).padStart(3, '0')}.jsonl`)
|
||||
await writeClaudeTranscript(path, ['fairnessneedle'], `changed-${index}`)
|
||||
const time = new Date(Date.UTC(2025, 0, 1) + index * 1000)
|
||||
await utimes(path, time, time)
|
||||
}
|
||||
await subject.reconcile()
|
||||
expect(hits()).toEqual([
|
||||
{ id: 'backlog-0' },
|
||||
{ id: 'backlog-1' },
|
||||
{ id: 'backlog-2' },
|
||||
{ id: `changed-${population - 1}` }
|
||||
])
|
||||
} finally {
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.each([40, 400])(
|
||||
'preserves ranked read obligations across bounded pages for %s candidates',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-read-pages')
|
||||
const store = new SessionSearchWorkerStore(harness.databasePath, (error) => {
|
||||
throw error
|
||||
})
|
||||
const queue = new SessionSearchWorkQueue()
|
||||
queue.attach(store)
|
||||
const traffic = vi.spyOn(SessionSearchWorkerClient.prototype, 'request')
|
||||
const candidates: SessionFileCandidate[] = []
|
||||
let largestResponse = 0
|
||||
configureSessionSearchWorkerFixture({
|
||||
prelude: '',
|
||||
data: undefined,
|
||||
onResponse: (response) => {
|
||||
largestResponse = Math.max(largestResponse, JSON.stringify(response).length * 2)
|
||||
}
|
||||
})
|
||||
try {
|
||||
await store.prepare()
|
||||
const prefix = join(harness.root, ...Array.from({ length: 10 }, () => 'p'.repeat(100)))
|
||||
for (let index = 0; index < population; index++) {
|
||||
const candidate: SessionFileCandidate = {
|
||||
agent: 'claude',
|
||||
codexHome: null,
|
||||
file: {
|
||||
path: join(prefix, `${String(population - index).padStart(4, '0')}.jsonl`),
|
||||
mtimeMs: 10000 - Math.floor(index / 4),
|
||||
modifiedAt: '1970-01-01T00:00:10.000Z',
|
||||
sizeBytes: 20
|
||||
}
|
||||
}
|
||||
if (index === 1 || index === 3) {
|
||||
const segments = index === 1 ? 12000 : 6000
|
||||
const windowsRoot = `C:\\${Array.from({ length: segments }, () => 'p').join('\\')}`
|
||||
candidate.agent = 'antigravity'
|
||||
candidate.file.path = `${windowsRoot}\\conversation-${index}.pb`
|
||||
candidate.antigravityHistoryPath = `${windowsRoot}\\history-${index}.pb`
|
||||
}
|
||||
candidates.push(candidate)
|
||||
await queue.enqueue(candidate, store, false, index % 2 === 0)
|
||||
}
|
||||
traffic.mockClear()
|
||||
queue.beginPass()
|
||||
const first = await queue.next()
|
||||
expect(first?.candidate.file.path).toBe(candidates[0].file.path)
|
||||
queue.beginPass()
|
||||
expect(await queue.next()).toEqual(first)
|
||||
await queue.acknowledge(first!, true)
|
||||
const observed: string[] = []
|
||||
for (let index = 1; index < population; index++) {
|
||||
let work = await queue.next()
|
||||
if (index === 1) {
|
||||
await queue.enqueue(candidates[1], store, false, false, true)
|
||||
await queue.enqueue(candidates[2], store, true, true)
|
||||
await queue.acknowledge(work!, false)
|
||||
work = await queue.next()
|
||||
expect(work).toMatchObject({ candidate: candidates[1], metadata: true })
|
||||
}
|
||||
if (index === 2) {
|
||||
expect(work).toMatchObject({ candidate: candidates[2], force: true })
|
||||
}
|
||||
observed.push(work!.candidate.file.path)
|
||||
await queue.acknowledge(work!, false)
|
||||
}
|
||||
expect(observed).toEqual(candidates.slice(1).map((candidate) => candidate.file.path))
|
||||
expect(await queue.next()).toBeUndefined()
|
||||
// Each lane needs a bounded projection per row/byte page; per-file accepted acknowledgements are separate.
|
||||
const pages =
|
||||
Math.ceil(population / 256) +
|
||||
Math.ceil((JSON.stringify(candidates).length * 2) / (128 * 1024))
|
||||
expect(
|
||||
traffic.mock.calls.filter(
|
||||
([command]) => command.op === 'work' && command.command.kind === 'next'
|
||||
).length
|
||||
).toBeLessThanOrEqual(40 + 12 * pages)
|
||||
queue.beginPass()
|
||||
const retry = await queue.next()
|
||||
expect(retry?.candidate.file.path).toBe(candidates[0].file.path)
|
||||
await queue.enqueue(candidates[0], store, false, true, true)
|
||||
await queue.acknowledge(retry!, false)
|
||||
const latest = await queue.next()
|
||||
expect(latest).toMatchObject({ candidate: candidates[0], metadata: true })
|
||||
await queue.acknowledge(latest!, false)
|
||||
expect(await queue.next()).toBeUndefined()
|
||||
expect(largestResponse).toBeLessThanOrEqual(256 * 1024)
|
||||
} finally {
|
||||
configureSessionSearchWorkerFixture()
|
||||
traffic.mockRestore()
|
||||
await store.flush()
|
||||
store.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -38,7 +38,7 @@ export type SessionSearchCandidateCommand =
|
||||
rows: { path: string; agents?: string[]; observation?: SessionSearchRemovalReceipt }[]
|
||||
}
|
||||
| { kind: 'unowned-page'; rows: { path: string; agent?: string | null }[] }
|
||||
| { kind: 'path'; path: string; agent?: string | null }
|
||||
| { kind: 'path'; path: string; agent?: string | null; read?: boolean }
|
||||
| { kind: 'group'; group: string; after?: string }
|
||||
| { kind: 'changes'; after?: string; root?: string; agent?: string }
|
||||
| { kind: 'rerank'; root: string; agent: string; order: [string, number][]; after?: string }
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
SessionSearchCatalogResult,
|
||||
SessionSearchCatalogRow
|
||||
} from './session-search-catalog'
|
||||
import type { SessionSearchCatalogPosition } from './session-search-catalog-types'
|
||||
import { sessionSearchRecordPages } from './session-search-catalog-pages'
|
||||
import {
|
||||
SESSION_SEARCH_CATALOG_PAGE_BYTES,
|
||||
@@ -161,6 +162,7 @@ export class SessionSearchCatalogClient<T> {
|
||||
async page(
|
||||
options: {
|
||||
after?: string
|
||||
afterOrder?: SessionSearchCatalogPosition
|
||||
parent?: string
|
||||
group?: string
|
||||
ready?: number
|
||||
@@ -169,12 +171,24 @@ export class SessionSearchCatalogClient<T> {
|
||||
limit?: number
|
||||
prefix?: string
|
||||
throughRevision?: number
|
||||
eligibleThrough?: number
|
||||
} = {}
|
||||
): Promise<{ rows: { key: string; value: T }[]; after?: string }> {
|
||||
): Promise<{
|
||||
rows: { key: string; value: T }[]
|
||||
after?: string
|
||||
position?: SessionSearchCatalogPosition
|
||||
}> {
|
||||
const result = await this.request({ kind: 'page', scope: this.scope, ...options })
|
||||
return {
|
||||
rows: (result.rows ?? []).map((row) => ({ key: row.key, value: this.decode(row) })),
|
||||
after: result.after
|
||||
after: result.after,
|
||||
position: result.rows?.length
|
||||
? {
|
||||
key: result.rows.at(-1)!.key,
|
||||
rank: result.rows.at(-1)!.rank,
|
||||
tie: result.rows.at(-1)!.tie ?? 0
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import type SyncDatabase from '../sqlite/sync-database'
|
||||
import type { SessionSearchCatalogBaseCommand } from './session-search-catalog-types'
|
||||
|
||||
export function sessionSearchCatalogPageRows(
|
||||
db: SyncDatabase,
|
||||
command: Extract<SessionSearchCatalogBaseCommand, { kind: 'page' }>,
|
||||
lane?: 'changed' | 'backlog'
|
||||
) {
|
||||
const conditions = ['scope = ?']
|
||||
const values: (string | number)[] = [command.scope]
|
||||
if (lane) {
|
||||
// Simultaneous lane cursors must not share one cached native statement.
|
||||
conditions.push(lane === 'changed' ? "group_key='changed'" : "group_key='backlog'")
|
||||
}
|
||||
if (command.afterOrder) {
|
||||
const { rank, tie, key } = command.afterOrder
|
||||
conditions.push(command.orderRevision ? '(tie,key) > (?,?)' : '(rank,tie,key) > (?,?,?)')
|
||||
values.push(...(command.orderRevision ? [tie, key] : [rank, tie, key]))
|
||||
} else if (command.after !== undefined) {
|
||||
if (command.orderRank || command.orderRevision) {
|
||||
throw new Error('Ordered session catalog continuation requires its full position')
|
||||
}
|
||||
conditions.push('key > ?')
|
||||
values.push(command.after)
|
||||
}
|
||||
if (command.prefix !== undefined) {
|
||||
conditions.push('key >= ?', 'key < ?')
|
||||
values.push(command.prefix, `${command.prefix}\uffff`)
|
||||
}
|
||||
for (const [column, value] of [
|
||||
['parent', command.parent],
|
||||
['group_key', command.group],
|
||||
['ready', command.ready]
|
||||
] as const) {
|
||||
if (value !== undefined) {
|
||||
conditions.push(`${column} = ?`)
|
||||
values.push(value)
|
||||
}
|
||||
}
|
||||
if (command.throughRevision !== undefined) {
|
||||
conditions.push('tie <= ?')
|
||||
values.push(command.throughRevision)
|
||||
}
|
||||
if (command.beforeRank !== undefined) {
|
||||
conditions.push('rank <= ?')
|
||||
values.push(command.beforeRank)
|
||||
}
|
||||
if (command.eligibleThrough !== undefined) {
|
||||
conditions.push('(ready=1 OR tie<=?)')
|
||||
values.push(command.eligibleThrough)
|
||||
}
|
||||
return db
|
||||
.prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog
|
||||
WHERE ${conditions.join(' AND ')} ORDER BY ${command.orderRevision ? 'tie,key' : command.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`)
|
||||
.iterate(...values, Math.max(1, Math.min(command.limit ?? 256, 256)))
|
||||
}
|
||||
@@ -28,6 +28,7 @@ export type SessionSearchCatalogRow = {
|
||||
tie?: number
|
||||
}
|
||||
export type SessionSearchInputReceipt = { consumer: string; revision: number }
|
||||
export type SessionSearchCatalogPosition = { key: string; rank: number; tie: number }
|
||||
export type SessionSearchCatalogBaseCommand = (
|
||||
| { kind: 'get'; scope: string; key: string }
|
||||
| { kind: 'get-page'; scope: string; keys: string[] }
|
||||
@@ -44,6 +45,7 @@ export type SessionSearchCatalogBaseCommand = (
|
||||
kind: 'page'
|
||||
scope: string
|
||||
after?: string
|
||||
afterOrder?: SessionSearchCatalogPosition
|
||||
parent?: string
|
||||
group?: string
|
||||
ready?: number
|
||||
@@ -51,6 +53,7 @@ export type SessionSearchCatalogBaseCommand = (
|
||||
orderRevision?: boolean
|
||||
throughRevision?: number
|
||||
beforeRank?: number
|
||||
eligibleThrough?: number
|
||||
limit?: number
|
||||
prefix?: string
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { sessionSearchCatalogPageRows } from './session-search-catalog-query'
|
||||
import type SyncDatabase from '../sqlite/sync-database'
|
||||
import { transitionSessionSearchCatalog } from './session-search-catalog-transition'
|
||||
import { SessionSearchCatalogStorage } from './session-search-catalog-storage'
|
||||
@@ -248,45 +249,9 @@ export class SessionSearchCatalog extends SessionSearchCatalogStorage {
|
||||
}
|
||||
}
|
||||
case 'page': {
|
||||
const conditions = ['scope = ?', 'key > ?']
|
||||
const values: (string | number)[] = [base.scope, base.after ?? '']
|
||||
if (base.prefix !== undefined) {
|
||||
conditions.push('key >= ?', 'key < ?')
|
||||
values.push(base.prefix, `${base.prefix}\uffff`)
|
||||
}
|
||||
for (const [column, value] of [
|
||||
['parent', base.parent],
|
||||
['group_key', base.group],
|
||||
['ready', base.ready]
|
||||
] as const) {
|
||||
if (value !== undefined) {
|
||||
conditions.push(`${column} = ?`)
|
||||
values.push(value)
|
||||
}
|
||||
}
|
||||
if (base.throughRevision !== undefined) {
|
||||
conditions.push('tie <= ?')
|
||||
values.push(base.throughRevision)
|
||||
}
|
||||
if (base.beforeRank !== undefined) {
|
||||
conditions.push('rank <= ?')
|
||||
values.push(base.beforeRank)
|
||||
}
|
||||
const rows: SessionSearchCatalogRow[] = []
|
||||
let bytes = 1024
|
||||
const iterator = this.db
|
||||
.prepare(`SELECT key,value,parent,group_key,rank,ready,tie FROM temp.search_catalog
|
||||
WHERE ${conditions.join(' AND ')} ORDER BY ${base.orderRevision ? 'tie,key' : base.orderRank ? 'rank,tie,key' : 'key'} LIMIT ?`)
|
||||
.iterate(
|
||||
...values,
|
||||
Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
base.limit ?? SESSION_SEARCH_CATALOG_PAGE_ROWS,
|
||||
SESSION_SEARCH_CATALOG_PAGE_ROWS
|
||||
)
|
||||
)
|
||||
)
|
||||
const iterator = sessionSearchCatalogPageRows(this.db, base)
|
||||
for (const value of iterator) {
|
||||
const row = this.decode(value)
|
||||
const next = this.bytes(row)
|
||||
|
||||
@@ -30,6 +30,7 @@ export async function runSessionSearchIndexPass(
|
||||
}
|
||||
): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> {
|
||||
const stats = createSessionParseStats()
|
||||
queue.beginPass()
|
||||
// Retry once per cycle; rotating the failed path cannot monopolize a pass.
|
||||
const available = queue.size
|
||||
let consumed = 0
|
||||
@@ -53,7 +54,9 @@ export async function runSessionSearchIndexPass(
|
||||
}
|
||||
const { candidate } = work
|
||||
const path = candidate.file.path
|
||||
await store.prepareCoverage?.([path])
|
||||
if (!options.admit || !store.coverageKnown?.(path)) {
|
||||
await store.prepareCoverage?.([path])
|
||||
}
|
||||
let row = store.fileRow(path)
|
||||
const cursor = row ? store.indexedFile(path, fileIdentity(candidate.file)) : null
|
||||
const entry = work.metadata
|
||||
|
||||
@@ -179,9 +179,17 @@ export async function runSessionSearchPass(
|
||||
overdue: readOverdue,
|
||||
publications: state.candidates,
|
||||
admit: async (work) => {
|
||||
const admission = await state.candidates.readAdmission(work.candidate)
|
||||
if (!state.queue.selectionCurrent(admission.work)) {
|
||||
return undefined
|
||||
}
|
||||
if (admission.work.work?.version !== work.version) {
|
||||
return undefined
|
||||
}
|
||||
work = admission.work.work
|
||||
const { candidate } = work
|
||||
const path = candidate.file.path
|
||||
const rootDir = await state.candidates.rootFor(candidate)
|
||||
const rootDir = admission.root
|
||||
if (rootDir === undefined) {
|
||||
await state.queue.acknowledge(work, false)
|
||||
return undefined
|
||||
|
||||
@@ -94,7 +94,11 @@ export async function transitionSessionSearchCandidates(
|
||||
}
|
||||
}
|
||||
if (command.kind !== 'admit') {
|
||||
return store.catalogCandidates.execute(command)
|
||||
const result = store.catalogCandidates.execute(command)
|
||||
if (command.kind === 'path' && command.read) {
|
||||
return { ...result, work: await store.work({ kind: 'get', path: command.path }) }
|
||||
}
|
||||
return result
|
||||
}
|
||||
for (const receipt of command.rows) {
|
||||
let row = store.catalogCandidates.entry(receipt.key)
|
||||
@@ -149,6 +153,8 @@ export async function transitionSessionSearchWork(
|
||||
): Promise<SessionSearchWorkResult> {
|
||||
queue.attach(store)
|
||||
let work
|
||||
let works
|
||||
let eligibleThrough
|
||||
switch (command.kind) {
|
||||
case 'enqueue':
|
||||
if (store.fileRow(command.candidate.file.path)?.state === 'failed') {
|
||||
@@ -172,7 +178,8 @@ export async function transitionSessionSearchWork(
|
||||
work = await queue.get(command.path)
|
||||
break
|
||||
case 'next':
|
||||
work = await queue.next()
|
||||
eligibleThrough = command.eligibleThrough ?? queue.eligibleThrough
|
||||
works = queue.next(store.connection, eligibleThrough)
|
||||
break
|
||||
case 'stats':
|
||||
break
|
||||
@@ -222,5 +229,12 @@ export async function transitionSessionSearchWork(
|
||||
await queue.rowChanged(command.path, command.row)
|
||||
break
|
||||
}
|
||||
return { size: queue.size, left: queue.left, work }
|
||||
return {
|
||||
size: queue.size,
|
||||
left: queue.left,
|
||||
work,
|
||||
works,
|
||||
eligibleThrough,
|
||||
revision: queue.revision
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ import { SessionSearchCatalogClient } from './session-search-catalog-client'
|
||||
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
|
||||
import { fileIdentity } from './session-search-file-cursor'
|
||||
import { sessionSearchReadDecision } from './session-search-read-decision'
|
||||
import type SyncDatabase from '../sqlite/sync-database'
|
||||
import { sessionSearchCatalogPageRows } from './session-search-catalog-query'
|
||||
import { SESSION_SEARCH_CATALOG_PAGE_BYTES } from './session-search-catalog'
|
||||
import type { SessionSearchFileRow, SessionSearchIndexStore } from './session-search-store'
|
||||
|
||||
export type SessionSearchWork = {
|
||||
@@ -10,7 +13,14 @@ export type SessionSearchWork = {
|
||||
version: number
|
||||
metadata: boolean
|
||||
}
|
||||
type Queued = { work: SessionSearchWork; lane: string; order: number; due: boolean; rank: number }
|
||||
type Queued = {
|
||||
work: SessionSearchWork
|
||||
lane: string
|
||||
order: number
|
||||
due: boolean
|
||||
rank: number
|
||||
deferred: boolean
|
||||
}
|
||||
|
||||
/** A unique path's obligation lives in the writer even after it leaves the recent list. */
|
||||
export class SessionSearchWorkCatalog {
|
||||
@@ -20,6 +30,10 @@ export class SessionSearchWorkCatalog {
|
||||
private chooseChanged = true
|
||||
private count = 0
|
||||
private due = 0
|
||||
revision = 0
|
||||
get eligibleThrough(): number {
|
||||
return this.order
|
||||
}
|
||||
|
||||
attach(store: SessionSearchIndexStore): void {
|
||||
this.catalog ??= new SessionSearchCatalogClient('read-work', (command) =>
|
||||
@@ -63,9 +77,11 @@ export class SessionSearchWorkCatalog {
|
||||
lane: existing?.lane ?? (recent ? 'changed' : 'backlog'),
|
||||
order: existing?.order ?? ++this.order,
|
||||
due: row?.state === 'due',
|
||||
deferred: false,
|
||||
rank: -(candidate.file.aliasMtimeMs ?? candidate.file.mtimeMs)
|
||||
}
|
||||
await this.put(path, value)
|
||||
this.revision++
|
||||
this.count += existing ? 0 : 1
|
||||
this.due += Number(value.due) - Number(existing?.due ?? false)
|
||||
}
|
||||
@@ -75,16 +91,53 @@ export class SessionSearchWorkCatalog {
|
||||
return !!work && (work.force || work.metadata)
|
||||
}
|
||||
|
||||
async next(): Promise<SessionSearchWork | undefined> {
|
||||
const preferred = this.chooseChanged ? 'changed' : 'backlog'
|
||||
this.chooseChanged = !this.chooseChanged
|
||||
for (const lane of [preferred, preferred === 'changed' ? 'backlog' : 'changed']) {
|
||||
const page = await this.catalog!.page({ group: lane, orderRank: true, limit: 1 })
|
||||
if (page.rows.length) {
|
||||
return page.rows[0].value.work
|
||||
next(db: SyncDatabase, eligibleThrough = this.order): SessionSearchWork[] {
|
||||
const lanes: ReturnType<typeof sessionSearchCatalogPageRows>[] = []
|
||||
const works: SessionSearchWork[] = []
|
||||
let bytes = 1024
|
||||
let selected = this.chooseChanged ? 0 : 1
|
||||
const exhausted = [false, false]
|
||||
try {
|
||||
for (const lane of ['changed', 'backlog'] as const) {
|
||||
lanes.push(
|
||||
sessionSearchCatalogPageRows(
|
||||
db,
|
||||
{ kind: 'page', scope: 'read-work', orderRank: true, eligibleThrough, limit: 256 },
|
||||
lane
|
||||
)
|
||||
)
|
||||
}
|
||||
while (works.length < 256) {
|
||||
if (exhausted[selected]) {
|
||||
selected = 1 - selected
|
||||
if (exhausted[selected]) {
|
||||
break
|
||||
}
|
||||
}
|
||||
const row = lanes[selected].next()
|
||||
if (row.done) {
|
||||
exhausted[selected] = true
|
||||
continue
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this queue encoded its own catalog record.
|
||||
const work = (JSON.parse(String(row.value.value)) as Queued).work
|
||||
const size = JSON.stringify(work).length * 2 + 64
|
||||
if (works.length && bytes + size > SESSION_SEARCH_CATALOG_PAGE_BYTES) {
|
||||
break
|
||||
}
|
||||
works.push(work)
|
||||
bytes += size
|
||||
if (bytes >= SESSION_SEARCH_CATALOG_PAGE_BYTES) {
|
||||
break
|
||||
}
|
||||
selected = 1 - selected
|
||||
}
|
||||
} finally {
|
||||
for (const lane of lanes) {
|
||||
lane.return?.()
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
return works
|
||||
}
|
||||
|
||||
async get(path: string): Promise<SessionSearchWork | undefined> {
|
||||
@@ -97,6 +150,7 @@ export class SessionSearchWorkCatalog {
|
||||
if (entry?.work.version !== work.version) {
|
||||
return
|
||||
}
|
||||
this.chooseChanged = !this.chooseChanged
|
||||
if (retry) {
|
||||
await this.put(path, {
|
||||
...entry,
|
||||
@@ -104,20 +158,22 @@ export class SessionSearchWorkCatalog {
|
||||
lane: 'backlog',
|
||||
order: ++this.order,
|
||||
due: false,
|
||||
deferred: true,
|
||||
rank: this.order
|
||||
})
|
||||
this.due -= Number(entry.due)
|
||||
} else {
|
||||
await this.remove(path)
|
||||
await this.remove(path, false)
|
||||
}
|
||||
}
|
||||
|
||||
async remove(path: string): Promise<void> {
|
||||
async remove(path: string, invalidate = true): Promise<void> {
|
||||
const entry = await this.catalog?.get(path)
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
await this.catalog!.delete(path)
|
||||
this.revision += Number(invalidate)
|
||||
this.count--
|
||||
this.due -= Number(entry.due)
|
||||
}
|
||||
@@ -133,6 +189,11 @@ export class SessionSearchWorkCatalog {
|
||||
}
|
||||
|
||||
private put(path: string, value: Queued): Promise<void> {
|
||||
return this.catalog!.put(path, value, { group: value.lane, rank: value.rank, tie: value.order })
|
||||
return this.catalog!.put(path, value, {
|
||||
group: value.lane,
|
||||
rank: value.rank,
|
||||
tie: value.order,
|
||||
ready: Number(!value.deferred)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ export type SessionSearchWorkCommand =
|
||||
recent: boolean
|
||||
metadata: boolean
|
||||
}
|
||||
| { kind: 'next' }
|
||||
| { kind: 'next'; eligibleThrough?: number }
|
||||
| { kind: 'stats' }
|
||||
| { kind: 'get'; path: string }
|
||||
| {
|
||||
@@ -22,13 +22,33 @@ export type SessionSearchWorkCommand =
|
||||
}
|
||||
| { kind: 'remove'; path: string }
|
||||
| { kind: 'row'; path: string; row?: SessionSearchFileRow }
|
||||
export type SessionSearchWorkResult = { size: number; left: number; work?: SessionSearchWork }
|
||||
export type SessionSearchWorkResult = {
|
||||
size: number
|
||||
left: number
|
||||
work?: SessionSearchWork
|
||||
works?: SessionSearchWork[]
|
||||
revision?: number
|
||||
eligibleThrough?: number
|
||||
}
|
||||
|
||||
/** Only active reads cross the boundary; the writer owns ready work and its transitions. */
|
||||
export class SessionSearchWorkQueue {
|
||||
private store: SessionSearchIndexStore | undefined
|
||||
private count = 0
|
||||
private remaining = 0
|
||||
private page: SessionSearchWork[] = []
|
||||
private revision: number | undefined
|
||||
private selectedRevision: number | undefined
|
||||
private eligibleThrough: number | undefined
|
||||
beginPass(): void {
|
||||
this.page = []
|
||||
this.eligibleThrough = undefined
|
||||
}
|
||||
selectionCurrent(result: SessionSearchWorkResult): boolean {
|
||||
const current = this.selectedRevision === result.revision
|
||||
this.update(result)
|
||||
return current
|
||||
}
|
||||
attach(store: SessionSearchIndexStore): void {
|
||||
this.store ??= store
|
||||
}
|
||||
@@ -39,6 +59,10 @@ export class SessionSearchWorkQueue {
|
||||
return this.remaining
|
||||
}
|
||||
update(result: SessionSearchWorkResult): void {
|
||||
if (this.revision !== result.revision) {
|
||||
this.page = []
|
||||
this.revision = result.revision
|
||||
}
|
||||
this.count = result.size
|
||||
this.remaining = result.left
|
||||
}
|
||||
@@ -62,7 +86,13 @@ export class SessionSearchWorkQueue {
|
||||
return !!work && (work.force || work.metadata)
|
||||
}
|
||||
async next(): Promise<SessionSearchWork | undefined> {
|
||||
return (await this.request({ kind: 'next' })).work
|
||||
if (!this.page.length) {
|
||||
const result = await this.request({ kind: 'next', eligibleThrough: this.eligibleThrough })
|
||||
this.eligibleThrough ??= result.eligibleThrough
|
||||
this.page = result.works ?? (result.work ? [result.work] : [])
|
||||
}
|
||||
this.selectedRevision = this.revision
|
||||
return this.page.shift()
|
||||
}
|
||||
async get(path: string): Promise<SessionSearchWork | undefined> {
|
||||
return (await this.request({ kind: 'get', path })).work
|
||||
|
||||
@@ -251,6 +251,7 @@ async function execute({ id, command: c }: SearchWorkerRequest): Promise<SearchW
|
||||
id,
|
||||
reset,
|
||||
candidates: result,
|
||||
coverage: c.command.kind === 'path' && c.command.read ? rows(c.command.path) : undefined,
|
||||
deltas: path
|
||||
? [delta(path, previous)]
|
||||
: result.withdrawn?.map((row) => delta(row.path, row)),
|
||||
|
||||
@@ -222,6 +222,10 @@ export class SessionSearchWorkerStoreBase extends SessionSearchCursorSnapshot {
|
||||
if (!response.candidates) {
|
||||
throw new Error('Session search candidate catalog unavailable')
|
||||
}
|
||||
if (command.kind === 'path' && command.read) {
|
||||
this.projectCoverage(command.path, response.coverage?.[0])
|
||||
this.trimCoverage()
|
||||
}
|
||||
return response.candidates
|
||||
}
|
||||
async work(command: SessionSearchWorkCommand): Promise<SessionSearchWorkResult> {
|
||||
|
||||
Reference in New Issue
Block a user