mirror of
https://github.com/stablyai/orca.git
synced 2026-09-27 16:02:35 +00:00
feat(ai-vault-search): give the index a generation a page cursor can be fenced by
A search page is a slice of one ranked list, so a cursor only means anything against the snapshot that produced it. The store now keeps a monotone generation in `meta`, bumped by every mutation that can change which rows a read returns, and starts a new one on open so a write whose bump never landed cannot leave a cursor pointing at content that is already gone. Schema version 2 adds the two tables the query layer needs: `messages_vocab` (fts5vocab over messages_fts, the typo repair's whole dictionary) and `search_log`. Both are pure additions and the index is a cache, so a version-1 file is dropped and rebuilt exactly as any other mismatch is.
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
import { mkdtemp, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { removeTree } from '../../shared/windows-transient-lock-removal'
|
||||
import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
|
||||
import { resetSessionParseCacheForTests } from '../ai-vault/session-scanner-parse-cache'
|
||||
import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
|
||||
import { SessionSearchStore } from './session-search-store'
|
||||
import { parseTranscript, userRecord } from './session-search-transcript-fixtures'
|
||||
|
||||
let roots: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
resetTranscriptConsumersForTests()
|
||||
resetSessionParseCacheForTests()
|
||||
await Promise.all(roots.map((root) => removeTree(root)))
|
||||
roots = []
|
||||
})
|
||||
|
||||
async function tempRoot(): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-search-generation-'))
|
||||
roots.push(root)
|
||||
return root
|
||||
}
|
||||
|
||||
async function indexOneTranscript(root: string, store: SessionSearchStore): Promise<void> {
|
||||
resetSessionParseCacheForTests()
|
||||
const sessionId = `aaaaaaaa-0000-4000-8000-${String(roots.length).padStart(12, '0')}`
|
||||
const path = join(root, `${Math.random().toString(36).slice(2)}.jsonl`)
|
||||
await writeFile(path, `${userRecord(0, 'generation fixture', sessionId)}\n`)
|
||||
const unregister = registerSessionSearchIndexConsumer(store)
|
||||
try {
|
||||
await parseTranscript(path)
|
||||
} finally {
|
||||
unregister()
|
||||
}
|
||||
}
|
||||
|
||||
it('moves the generation forward when a published read changes what a read returns', async () => {
|
||||
const root = await tempRoot()
|
||||
const store = new SessionSearchStore(join(root, 'index.sqlite'), (error) => {
|
||||
throw error
|
||||
})
|
||||
try {
|
||||
const before = store.generation
|
||||
await indexOneTranscript(root, store)
|
||||
expect(store.generation).toBeGreaterThan(before)
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('starts a new generation on every open, so a cursor cannot outlive a crash', async () => {
|
||||
const root = await tempRoot()
|
||||
const path = join(root, 'index.sqlite')
|
||||
const first = new SessionSearchStore(path)
|
||||
const firstGeneration = first.generation
|
||||
first.close()
|
||||
|
||||
const second = new SessionSearchStore(path)
|
||||
try {
|
||||
// Why not merely "different": the value is persisted and monotone, so a
|
||||
// generation can never be reused for content that has moved on.
|
||||
expect(second.generation).toBeGreaterThan(firstGeneration)
|
||||
} finally {
|
||||
second.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('moves the generation forward when a proven deletion retires a session', async () => {
|
||||
const root = await tempRoot()
|
||||
const store = new SessionSearchStore(join(root, 'index.sqlite'), (error) => {
|
||||
throw error
|
||||
})
|
||||
try {
|
||||
store.removeFile('/synthetic/absent.jsonl')
|
||||
const afterRemove = store.generation
|
||||
store.removeFile('/synthetic/absent-two.jsonl')
|
||||
expect(store.generation).toBeGreaterThan(afterRemove)
|
||||
} finally {
|
||||
store.close()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,28 @@
|
||||
import type SyncDatabase from '../sqlite/sync-database'
|
||||
|
||||
const GENERATION_KEY = 'index_generation'
|
||||
|
||||
/**
|
||||
* A monotone id for what the index currently publishes.
|
||||
*
|
||||
* A search page is a slice of one ranked list, so a cursor only means anything
|
||||
* against the snapshot that produced it. Every mutation that can change which
|
||||
* rows a read returns bumps this, and a cursor minted under an older value is
|
||||
* refused rather than silently re-run against a list it no longer indexes into.
|
||||
*/
|
||||
export function readIndexGeneration(db: SyncDatabase): number {
|
||||
const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(GENERATION_KEY) as
|
||||
| { value: string }
|
||||
| undefined
|
||||
const parsed = row ? Number(row.value) : Number.NaN
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0
|
||||
}
|
||||
|
||||
export function bumpIndexGeneration(db: SyncDatabase): number {
|
||||
const next = readIndexGeneration(db) + 1
|
||||
db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)').run(
|
||||
GENERATION_KEY,
|
||||
String(next)
|
||||
)
|
||||
return next
|
||||
}
|
||||
Reference in New Issue
Block a user