Withdraw superseded search copies after acknowledged publication

Retain accepted alias output through replacement failures, fence suppressed search reads, and acknowledge derived-output cleanup. Own refused family enumeration and bound discovery work with linked frontiers.

Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
Brennan Benson
2026-11-09 18:28:03 -08:00
co-authored by Jinjing
parent 685ff776dc
commit f3f6db8f0e
22 changed files with 833 additions and 66 deletions
@@ -0,0 +1,145 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { SessionSearchCandidateInventory } from './session-search-candidate-inventory'
import { SessionSearchIndexConsumer } from './session-search-index-consumer'
import { SessionSearchEngine } from './session-search-engine'
import { SessionSearchStore } from './session-search-store'
import {
openSessionSearchIndexFile,
syntheticCandidate,
syntheticSession,
type SessionSearchIndexFile
} from './session-search-index-test-fixture'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
let index: SessionSearchIndexFile
let store: SessionSearchStore
let inventory: SessionSearchCandidateInventory
let engine: SessionSearchEngine
const first = candidate('opencode-first.db', 200)
const previous = candidate('opencode-previous.db', 100)
const latest = candidate('opencode-latest.db', 300)
function candidate(database: string, mtimeMs: number): SessionFileCandidate {
return {
...syntheticCandidate({ path: `/archive/${database}#shared`, mtimeMs }),
agent: 'opencode',
file: { path: `/archive/${database}#shared`, mtimeMs, modifiedAt: '2025-01-01' }
}
}
function discover(candidates: SessionFileCandidate[], rootDir = '/archive/storage') {
return { agent: 'opencode' as const, rootDir, files: candidates.map((entry) => entry.file) }
}
function reader(
input: SessionFileCandidate,
owner: SessionSearchCandidateInventory | undefined = inventory
) {
return new SessionSearchIndexConsumer(store, undefined, owner).beginRead({
candidate: input,
mode: 'replace',
previousByteOffset: 0
})
}
function finish(input: SessionFileCandidate, text: string, empty = false, owner = inventory) {
const consumer = reader(input, owner)!
consumer.message({ role: 'user', text, timestamp: null })
consumer.finish({
session: empty ? null : syntheticSession({ agent: 'opencode', filePath: input.file.path }),
byteOffset: 100,
incomplete: false
})
}
async function settle(wake = false) {
inventory.beginPublications(wake)
await inventory.reconcilePublications(store, () => false)
}
function hits(text: string) {
return engine.search({ query: text }).hits.length
}
beforeEach(async () => {
index = await openSessionSearchIndexFile('ss-alias-publication')
store = new SessionSearchStore(index.path)
inventory = new SessionSearchCandidateInventory()
engine = new SessionSearchEngine(index.db)
})
afterEach(async () => {
inventory.closePublications()
vi.restoreAllMocks()
store.close()
await index.close()
})
it('keeps the published predecessor through deferred, incomplete and held-out replacement reads', async () => {
await inventory.update([discover([previous])], new Set(), {})
finish(previous, 'predecessorneedle')
await inventory.update([discover([first])], new Set(), {})
await settle()
expect(hits('predecessorneedle')).toBe(1)
reader(first)!.finish({ session: null, byteOffset: 0, incomplete: true })
await settle()
expect(hits('predecessorneedle')).toBe(1)
store.setFileState(first.file.path, 'failed', first.file.mtimeMs)
store.setFileState(first.file.path, 'failed', first.file.mtimeMs)
await settle()
expect(hits('predecessorneedle')).toBe(1)
finish(first, 'replacementneedle')
await settle()
expect(hits('replacementneedle')).toBe(1)
expect(hits('predecessorneedle')).toBe(0)
expect(reader(previous)).toBeNull()
// The suppressed raw member remains available for promotion after positive removal of its winner.
await inventory.update([], new Set([first.file.path]), {})
finish(previous, 'promotedneedle')
expect(hits('promotedneedle')).toBe(1)
})
it('reconstructs cleanup from committed rows after restart and retries acknowledged withdrawal', async () => {
// These committed rows model a crash after replacement publication but before predecessor cleanup.
const detached = new SessionSearchCandidateInventory()
finish(previous, 'predecessorneedle', false, detached)
finish(first, 'replacementneedle', false, detached)
await inventory.update([discover([first, previous])], new Set(), {})
const remove = vi.spyOn(store, 'removeFile').mockReturnValue(false)
for (let cycle = 0; cycle < 4; cycle++) {
await settle()
}
expect(hits('predecessorneedle')).toBe(1)
expect(inventory.pendingPublications).toBe(false)
remove.mockRestore()
await settle(true)
expect(hits('replacementneedle')).toBe(1)
expect(hits('predecessorneedle')).toBe(0)
expect(reader(previous)).toBeNull()
})
it('fences stale search publication without blocking the foreground parser or removing the last good copy', async () => {
await inventory.update([discover([previous])], new Set(), {})
finish(previous, 'predecessorneedle')
await inventory.update([discover([first])], new Set(), {})
const stale = reader(first)!
stale.message({ role: 'user', text: 'staleackneedle', timestamp: null })
await inventory.update([discover([latest])], new Set(), {})
stale.finish({ session: syntheticSession(), byteOffset: 100, incomplete: false })
await settle()
expect(hits('staleackneedle')).toBe(0)
expect(hits('predecessorneedle')).toBe(1)
finish(latest, 'currentwinnerneedle')
await settle()
expect(hits('predecessorneedle')).toBe(0)
expect(hits('currentwinnerneedle')).toBe(1)
})
it('treats successful empty publication as authoritative while protecting a path selected by another group', async () => {
await inventory.update([discover([previous])], new Set(), {})
finish(previous, 'predecessorneedle')
await inventory.update([discover([first])], new Set(), {})
finish(first, 'excludedneedle', true)
await settle()
expect(hits('predecessorneedle')).toBe(0)
expect(hits('excludedneedle')).toBe(0)
await inventory.update([discover([previous], '/another/storage')], new Set(), {})
finish(previous, 'sharedpathneedle')
await settle()
expect(hits('sharedpathneedle')).toBe(1)
})
@@ -1,3 +1,8 @@
import {
SessionSearchCandidatePublications,
rows,
type Row
} from './session-search-candidate-publication'
import { splitOpenCodeSqliteCandidate } from '../../shared/opencode-sqlite-session-path'
import { basename, dirname, extname } from 'node:path'
import { sessionFileExecutionNamespace } from '../../shared/ai-vault/scanning/ai-vault-scope-cwd-match'
@@ -11,11 +16,6 @@ import type {
SessionFileDiscovery
} from '../ai-vault/session-scanner-types'
type Row = { candidate: SessionFileCandidate; root: string }
type Group = Row | Map<string, Row>
function rows(group: Group | undefined): Row[] {
return group ? (group instanceof Map ? [...group.values()] : [group]) : []
}
function rowId(row: Row): string {
return `${row.candidate.agent}:${row.candidate.file.path}`
}
@@ -33,11 +33,8 @@ function groupKey(candidate: SessionFileCandidate, root: string): string {
}
/** Ordinary paths own one record; only competing aliases allocate a group and selected set. */
export class SessionSearchCandidateInventory {
export class SessionSearchCandidateInventory extends SessionSearchCandidatePublications {
private readonly sourceOrders = new Map<string, ReadonlyMap<string, number>>()
private readonly groups = new Map<string, Group>()
private readonly byPath = new Map<string, string | Set<string>>()
private readonly selected = new Map<string, Set<string>>()
has(path: string, agent?: string | null): boolean {
const owner = this.byPath.get(path)
if (!agent) {
@@ -65,12 +62,16 @@ export class SessionSearchCandidateInventory {
}
}
const previous = new Map<string, Set<string>>()
const affectedPaths = new Set<string>()
const forced = new Set<string>()
const touch = (key: string) => {
if (previous.has(key)) {
return
}
const group = this.groups.get(key)
for (const { candidate } of rows(group)) {
affectedPaths.add(candidate.file.path)
}
previous.set(
key,
this.selected.get(key) ??
@@ -224,6 +225,7 @@ export class SessionSearchCandidateInventory {
}
candidates.push(...selected)
}
this.updatePublications(previous, affectedPaths)
for (const path of suppressed) {
const owner = this.byPath.get(path)
if (
@@ -0,0 +1,265 @@
import { SessionInputFrontier } from '../ai-vault/session-input-frontier'
import { SessionObservationRetries } from '../ai-vault/session-observation-retries'
import { fileIdentity } from './session-search-file-cursor'
import type { SessionSearchFileRow, SessionSearchIndexStore } from './session-search-store'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
export type Row = { candidate: SessionFileCandidate; root: string }
export type Group = Row | Map<string, Row>
type Publication = {
revision: number
required: Set<string>
withdrawals?: SessionInputFrontier<string>
}
export type SessionSearchPublicationLease = {
signal: AbortSignal
valid(): boolean
committed(): void
release(): void
}
export function rows(group: Group | undefined): Row[] {
return group ? (group instanceof Map ? [...group.values()] : [group]) : []
}
export abstract class SessionSearchCandidatePublications {
protected readonly groups = new Map<string, Group>()
protected readonly byPath = new Map<string, string | Set<string>>()
protected readonly selected = new Map<string, Set<string>>()
private readonly publications = new Map<string, Publication>()
private readonly dirtyPublications = new SessionInputFrontier<Publication>()
private readonly withdrawalRetries = new SessionObservationRetries()
private readonly readers = new Map<string, Set<AbortController>>()
private revision = 0
private owners(path: string): readonly string[] {
const owner = this.byPath.get(path)
return typeof owner === 'string' ? [owner] : [...(owner ?? [])]
}
private isSelected(path: string): boolean {
return this.owners(path).some((key) => {
const group = this.groups.get(key)
return group instanceof Map
? this.selected.get(key)?.has(path)
: group?.candidate.file.path === path
})
}
beginPublications(wake: boolean): void {
this.withdrawalRetries.begin(wake)
if (wake) {
for (const key of this.withdrawalRetries.keys()) {
const publication = this.publications.get(key)
if (publication) {
this.dirtyPublications.set(key, publication)
}
}
}
}
get pendingPublications(): boolean {
return this.dirtyPublications.size > 0
}
requirePublication(path: string): void {
for (const key of this.owners(path)) {
const publication = this.publications.get(key)
if (publication && this.selected.get(key)?.has(path)) {
publication.required.add(path)
publication.withdrawals = undefined
this.dirtyPublications.set(key, publication)
}
}
}
rowChanged(path: string, row: SessionSearchFileRow | undefined): void {
for (const key of this.owners(path)) {
const publication = this.publications.get(key)
if (publication) {
if (this.selected.get(key)?.has(path)) {
publication.withdrawals = undefined
} else if (row) {
publication.withdrawals?.set(path, path)
}
if (!this.withdrawalRetries.heldOut(key)) {
this.dirtyPublications.set(key, publication)
}
}
}
}
beginPublication(candidate: SessionFileCandidate): SessionSearchPublicationLease | null {
const path = candidate.file.path
if (this.byPath.has(path) && !this.isSelected(path)) {
return null
}
const owner = new AbortController()
const readers = this.readers.get(path) ?? new Set<AbortController>()
readers.add(owner)
this.readers.set(path, readers)
const revisions = new Map(
this.owners(path).map((key) => [key, this.publications.get(key)?.revision])
)
const valid = () =>
!owner.signal.aborted &&
(!this.byPath.has(path) || this.isSelected(path)) &&
this.owners(path).every(
(key) => revisions.has(key) && revisions.get(key) === this.publications.get(key)?.revision
)
return {
signal: owner.signal,
valid,
committed: () => {
if (!valid()) {
return
}
for (const key of this.owners(path)) {
const publication = this.publications.get(key)
if (publication && this.selected.get(key)?.has(path)) {
publication.required.delete(path)
publication.withdrawals = undefined
this.dirtyPublications.set(key, publication)
}
}
},
release: () => {
readers.delete(owner)
if (!readers.size) {
this.readers.delete(path)
}
}
}
}
async reconcilePublications(
store: SessionSearchIndexStore,
overdue: () => boolean,
signal?: AbortSignal
): Promise<void> {
const available = this.dirtyPublications.size
let consumed = 0
while (this.dirtyPublications.size && consumed < available && (consumed === 0 || !overdue())) {
if (signal?.aborted) {
return
}
const [key, publication] = this.dirtyPublications.firstEntry()!
this.dirtyPublications.delete(key)
consumed++
if (!this.withdrawalRetries.mayAttempt(key)) {
if (!this.withdrawalRetries.heldOut(key)) {
this.dirtyPublications.set(key, publication)
}
continue
}
const members = rows(this.groups.get(key))
const selected = this.selected.get(key)
if (
!selected?.size ||
!members
.filter((row) => selected.has(row.candidate.file.path))
.every(
({ candidate }) =>
!publication.required.has(candidate.file.path) && this.covered(store, candidate)
)
) {
continue
}
if (!publication.withdrawals) {
publication.withdrawals = new SessionInputFrontier<string>()
for (const { candidate } of members) {
const path = candidate.file.path
if (!selected.has(path) && store.fileRow(path)) {
publication.withdrawals.set(path, path)
}
}
}
let attempted = 0
while (publication.withdrawals.size && (attempted === 0 || !overdue())) {
if (signal?.aborted || this.publications.get(key) !== publication) {
return
}
const path = publication.withdrawals.first!
// Another source can still select this physical path.
if (this.isSelected(path)) {
publication.withdrawals.delete(path)
continue
}
attempted++
const accepted = await store.removeFile(path)
if (
signal?.aborted ||
this.publications.get(key) !== publication ||
this.isSelected(path)
) {
return
}
if (!accepted) {
this.withdrawalRetries.fail(key)
break
}
publication.withdrawals.delete(path)
this.withdrawalRetries.delete(key)
}
this.dirtyPublications.delete(key)
if (publication.withdrawals.size && !this.withdrawalRetries.heldOut(key)) {
this.dirtyPublications.set(key, publication)
}
}
}
private covered(store: SessionSearchIndexStore, candidate: SessionFileCandidate): boolean {
const row = store.fileRow(candidate.file.path)
const identity = fileIdentity(candidate.file)
return (
store.acceptingReads !== false &&
!!row &&
row.state === 'current' &&
row.mtimeMs === candidate.file.mtimeMs &&
row.byteOffset !== null &&
row.byteOffset !== undefined &&
row.byteOffset >= 0 &&
(candidate.file.sizeBytes === undefined ||
(row.sizeBytes === candidate.file.sizeBytes &&
row.byteOffset === candidate.file.sizeBytes)) &&
(!identity ||
(!!row.identity && identity.dev === row.identity.dev && identity.ino === row.identity.ino))
)
}
closePublications(): void {
for (const readers of this.readers.values()) {
for (const reader of readers) {
reader.abort()
}
}
this.readers.clear()
this.publications.clear()
}
protected updatePublications(
previous: ReadonlyMap<string, ReadonlySet<string>>,
affectedPaths: Set<string>
): void {
for (const key of previous.keys()) {
for (const { candidate } of rows(this.groups.get(key))) {
affectedPaths.add(candidate.file.path)
}
if (!(this.groups.get(key) instanceof Map)) {
this.publications.delete(key)
this.dirtyPublications.delete(key)
this.withdrawalRetries.delete(key)
}
}
const changedGroups = new Set(previous.keys())
for (const path of affectedPaths) {
for (const reader of this.readers.get(path) ?? []) {
reader.abort()
}
for (const key of this.owners(path)) {
changedGroups.add(key)
}
}
for (const key of changedGroups) {
if (this.groups.get(key) instanceof Map) {
const old = this.publications.get(key)
const publication = {
revision: ++this.revision,
required: new Set(
[...(old?.required ?? [])].filter((path) => this.selected.get(key)?.has(path))
)
}
this.publications.set(key, publication)
this.withdrawalRetries.delete(key)
this.dirtyPublications.set(key, publication)
}
}
}
}
@@ -1,3 +1,4 @@
import { recordSessionScanIssue } from '../ai-vault/session-scan-issues'
import { SessionObservationRetries } from '../ai-vault/session-observation-retries'
import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types'
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
@@ -20,6 +21,11 @@ type Family = {
/** Recent observations add facts; only a successful exhaustive observation can withdraw them. */
export class SessionSearchContainerInventory {
private readonly pathLists = new Map<
string,
{ paths: readonly string[]; complete: boolean; issues: AiVaultScanIssue[] }
>()
private seenPathLists = new Set<string>()
private readonly families = new Map<string, Family>()
private seen = new Set<string>()
private recentLimit = 12
@@ -29,6 +35,7 @@ export class SessionSearchContainerInventory {
begin(full: boolean, recentLimit: number): void {
this.seen = new Set()
this.seenPathLists = new Set()
this.retries.begin(full)
this.recentLimit = recentLimit
this.removed.clear()
@@ -41,7 +48,15 @@ export class SessionSearchContainerInventory {
}
}
get pending(): boolean {
for (const [root, list] of this.pathLists) {
if (!list.complete && !this.retries.heldOut(`paths:${root}`)) {
return true
}
}
for (const [key, family] of this.families) {
if (this.retries.heldOut(`paths:${key.slice(key.indexOf(':') + 1)}`)) {
continue
}
for (const [path, physical] of family.physical) {
if (physical.fullDue && !this.retries.heldOut(`${key}:${path}`)) {
return true
@@ -50,6 +65,37 @@ export class SessionSearchContainerInventory {
}
return false
}
async containerPaths(
root: string,
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
issues: AiVaultScanIssue[]
): Promise<{ paths: readonly string[]; complete: boolean }> {
this.seenPathLists.add(root)
const key = `paths:${root}`
const previous = this.pathLists.get(root)
if (!this.retries.mayAttempt(key)) {
for (const issue of previous?.issues ?? []) {
recordSessionScanIssue(issues, issue)
}
return previous ?? { paths: [], complete: false }
}
const ownIssues: AiVaultScanIssue[] = []
const paths = await read(ownIssues)
const complete = !ownIssues.some((issue) => issue.kind !== 'notice')
const result = {
paths: complete ? paths : [...new Set([...(previous?.paths ?? []), ...paths])],
complete,
issues: ownIssues
}
this.pathLists.set(root, result)
issues.push(...ownIssues)
if (complete) {
this.retries.delete(key)
} else {
this.retries.fail(key)
}
return result
}
async discover(
key: string,
paths: readonly string[],
@@ -95,7 +141,10 @@ export class SessionSearchContainerInventory {
this.release(key, path, family, physical)
continue
}
const full = physical.fullDue && this.retries.mayAttempt(retryKey)
const full =
physical.fullDue &&
!this.retries.heldOut(`paths:${key.slice(key.indexOf(':') + 1)}`) &&
this.retries.mayAttempt(retryKey)
const discovery = await read([path], full ? Infinity : this.recentLimit)
family.discovery = discovery
throwIfAiVaultScanCancelled(signal)
@@ -162,6 +211,12 @@ export class SessionSearchContainerInventory {
return { ...family.discovery, files: changes, sourceOrder: family.sourceOrder }
}
end(): void {
for (const root of this.pathLists.keys()) {
if (!this.seenPathLists.has(root)) {
this.pathLists.delete(root)
this.retries.delete(`paths:${root}`)
}
}
for (const [key, family] of this.families) {
if (!this.seen.has(key)) {
for (const [path, physical] of family.physical) {
@@ -172,6 +227,7 @@ export class SessionSearchContainerInventory {
}
}
clear(): void {
this.pathLists.clear()
this.families.clear()
this.retries.clear()
this.proofs.clear()
@@ -213,6 +213,14 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
}
}
containerPaths(
root: string,
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
issues: AiVaultScanIssue[]
) {
return this.containers.containerPaths(root, read, issues)
}
container(
key: string,
paths: readonly string[],
@@ -318,7 +318,7 @@ it.each(['first-file', 'accepted-file', 'directory'] as const)(
async (kind) => {
const harness = await openSessionSearchIndexerHarness('ss-refused-observation')
const clock = new FakeSessionSearchClock()
const directory = join(harness.claudeProjectDir, 'refused')
const directory = join(harness.roots.claudeProjectsDir, 'refused')
const path = join(directory, 'history.jsonl')
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
@@ -532,3 +532,57 @@ it.each([40, 400])(
}
}
)
it.each([40, 400])(
'charges refused directory attempts to the expired %s-directory discovery budget',
async (population) => {
const harness = await openSessionSearchIndexerHarness('ss-refused-budget')
const root = harness.claudeProjectDir
for (let index = 0; index < population; index++) {
await mkdir(join(root, `child-${index}`), { recursive: true })
}
const inventory = new SessionFileInventory(root)
const args = {
rootDir: root,
agent: 'claude' as const,
limit: Infinity,
extensions: ['.jsonl'],
issues: []
}
let quanta = 0
await inventory.drain(args, () => ++quanta >= Math.ceil(population / 16), 12)
const actual = filesystem.wslGatedReaddir
let refused = true
const reads = vi.spyOn(filesystem, 'wslGatedReaddir').mockImplementation((path, ...rest) => {
if (refused && path.startsWith(`${root}/`)) {
return Promise.reject(
Object.assign(new Error('fixture refused directory'), { code: 'EACCES' })
)
}
return actual(path, ...rest)
})
try {
inventory.begin(false, args)
await inventory.drain(args, () => true, 12)
// An expired slice admits one attempted input, including a failed observation.
expect(reads.mock.calls).toHaveLength(1)
expect(inventory.pending).toBe(true)
for (let cycle = 0; cycle < population * 3 && inventory.pending; cycle++) {
inventory.begin(false, args)
inventory.retryFailures()
await inventory.drain(args, () => true, 12)
}
expect(inventory.pending).toBe(false)
refused = false
const recovered = join(root, 'child-0', 'recovered.jsonl')
await writeFile(recovered, '{}\n')
inventory.invalidate(join(root, 'child-0'))
inventory.begin(false, args)
const result = await inventory.drain(args, () => false, 12)
expect(result.changes.map((change) => change.file.path)).toContain(recovered)
} finally {
reads.mockRestore()
await harness.cleanup()
}
}
)
@@ -1,3 +1,7 @@
import type {
SessionSearchCandidatePublications,
SessionSearchPublicationLease
} from './session-search-candidate-publication'
import { inSessionParseFileLane } from '../ai-vault/parse-cache/session-parse-file-lane'
import { requestWholeTranscriptRead } from '../ai-vault/transcripts/session-transcript-reader'
import { traceSessionSearch } from './session-search-trace'
@@ -47,7 +51,8 @@ export const SELF_CONTAINED_TRANSCRIPT_AGENTS = {
export class SessionSearchIndexConsumer implements TranscriptConsumer {
constructor(
private readonly store: SessionSearchIndexStore,
private readonly onCommitted?: (write: SessionSearchFileWrite) => void
private readonly onCommitted?: (write: SessionSearchFileWrite) => void,
private readonly publications?: SessionSearchCandidatePublications
) {}
resumableOffset(candidate: SessionFileCandidate): number | null {
@@ -87,6 +92,15 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
}
// A restart dropped the parser's resume point; append past our cursor once the reread proves the prefix.
const resume = start.mode === 'replace' ? this.provableCursor(start) : null
const publication = this.publications?.beginPublication(candidate)
if (this.publications && !publication) {
return null
}
const signal = publication
? start.signal
? AbortSignal.any([start.signal, publication.signal])
: publication.signal
: start.signal
const write = resume
? this.store.beginWrite(
candidate,
@@ -94,7 +108,7 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
resume.byteOffset,
start.identity,
start.indexing,
start.signal
signal
)
: this.store.beginWrite(
candidate,
@@ -102,12 +116,23 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
start.previousByteOffset,
start.identity,
start.indexing,
start.signal
signal
)
if (!write) {
publication?.release()
return null
}
return new SessionSearchReadConsumer(this.store, start, write, resume, this.onCommitted)
if (publication) {
publication.signal.addEventListener('abort', () => write.discard(), { once: true })
}
return new SessionSearchReadConsumer(
this.store,
start,
write,
resume,
this.onCommitted,
publication ?? undefined
)
}
private provableCursor(start: TranscriptReadStart): SessionSearchResumableCursor | null {
@@ -128,7 +153,8 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
private readonly start: TranscriptReadStart,
private readonly write: SessionSearchFileWrite,
private readonly resume: SessionSearchResumableCursor | null = null,
private readonly onCommitted?: (write: SessionSearchFileWrite) => void
private readonly onCommitted?: (write: SessionSearchFileWrite) => void,
private readonly publication?: SessionSearchPublicationLease
) {}
async drain(): Promise<void> {
@@ -157,9 +183,15 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
}
finish(outcome: TranscriptReadOutcome): void {
if (outcome.cancelled || this.write.cancelled?.() || this.write.failed?.()) {
if (
outcome.cancelled ||
this.publication?.valid() === false ||
this.write.cancelled?.() ||
this.write.failed?.()
) {
// Cancellation must not count toward the failure limit for a healthy file.
this.write.discard()
this.publication?.release()
return
}
const { candidate } = this.start
@@ -172,6 +204,7 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
rescueRejections: 1,
rescueRejection: 'prefix-mismatch-or-unreached'
})
this.publication?.release()
return
}
let committed = false
@@ -185,14 +218,29 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
}
if (committed) {
this.store.writeCommitted(candidate)
this.onCommitted?.(this.write)
const acknowledge = (accepted: boolean) => {
if (accepted && this.publication?.valid() !== false) {
this.publication?.committed()
this.onCommitted?.(this.write)
}
this.publication?.release()
}
if (this.write.settled) {
void this.write.settled().then(acknowledge, () => acknowledge(false))
} else {
acknowledge(true)
}
return
}
// Stamp and count repeated read failures so unchanged unreadable files can be held out.
this.store.setFileState(candidate.file.path, 'failed', candidate.file.mtimeMs)
this.publication?.release()
}
}
export function registerSessionSearchIndexConsumer(store: SessionSearchIndexStore): () => void {
return registerTranscriptConsumer(new SessionSearchIndexConsumer(store))
export function registerSessionSearchIndexConsumer(
store: SessionSearchIndexStore,
publications?: SessionSearchCandidatePublications
): () => void {
return registerTranscriptConsumer(new SessionSearchIndexConsumer(store, undefined, publications))
}
@@ -1,3 +1,4 @@
import type { SessionSearchCandidateInventory } from './session-search-candidate-inventory'
import {
getSessionParseCacheEntry,
sessionParseCacheKey
@@ -18,7 +19,11 @@ import type { SessionSearchWorkQueue } from './session-search-work-queue'
export async function runSessionSearchIndexPass(
store: SessionSearchIndexStore,
queue: SessionSearchWorkQueue,
options: { signal?: AbortSignal; overdue?: () => boolean }
options: {
signal?: AbortSignal
overdue?: () => boolean
publications?: SessionSearchCandidateInventory
}
): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> {
const stats = createSessionParseStats()
// Retry once per cycle; rotating the failed path cannot monopolize a pass.
@@ -65,9 +70,13 @@ export async function runSessionSearchIndexPass(
true
)
let published: SessionSearchFileWrite | undefined
const consumer = new SessionSearchIndexConsumer(store, (write) => {
published = write
}).beginRead({
const consumer = new SessionSearchIndexConsumer(
store,
(write) => {
published = write
},
options.publications
).beginRead({
candidate,
mode: 'append',
previousByteOffset: row.byteOffset,
@@ -43,6 +43,8 @@ type FileCursor = Pick<FileRow, 'session_row_id' | 'byte_offset'>
export type SessionSearchFileWrite = {
/** Commit named chunks at the ceiling; spill anonymous reads privately until atomic publication. */
add(message: TranscriptMessage): void
/** Resolves only after the writer acknowledges final publication. */
settled?(): Promise<boolean>
failed?(): boolean
cancelled?(): boolean
drain?(): Promise<void>
@@ -101,11 +101,15 @@ export class SessionSearchIndexer {
() => {
this.sweepNext = true
},
(path) => this.work.queue.rowChanged(path, this.store.fileRow(path))
(path) => {
const row = this.store.fileRow(path)
this.work.queue.rowChanged(path, row)
this.work.candidates.rowChanged(path, row)
}
)
liveIndexerPaths.add(this.ownershipPath)
this.store.setRetentionCutoffMs(this.cutoffMs())
this.unregister = registerSessionSearchIndexConsumer(this.store)
this.unregister = registerSessionSearchIndexConsumer(this.store, this.work.candidates)
}
/** Begin inventory capture, then consume changes and retained debt at the normal cadence. */
@@ -67,6 +67,8 @@ vi.mock(
}
}
)
import { readWholeTranscript } from '../ai-vault/transcripts/session-transcript-reader'
import { buildOpenCodeSqliteCandidatePath } from '../../shared/opencode-sqlite-session-path'
import { resetSessionParseCacheForTests } from '../ai-vault/parse-cache/session-parse-cache-store'
import { resetTranscriptConsumersForTests } from '../ai-vault/transcripts/session-transcript-consumers'
import {
@@ -421,6 +423,22 @@ it('chooses the original earlier database when an equal-time alias is imported l
'ses_shared'
)
expect(openEngine().search({ query: 'laterdatabasecopy' }).hits).toHaveLength(0)
// Opening the retained raw alias still parses it for History, without republishing search output.
const foreground = await readWholeTranscript({
candidate: {
agent: 'opencode',
codexHome: null,
file: {
path: buildOpenCodeSqliteCandidatePath(alternative, 'ses_shared'),
mtimeMs: 1740000060000,
modifiedAt: '2025-02-19T21:21:00.000Z'
}
},
platform: process.platform
})
await subject.settled()
expect(foreground?.sessionId).toBe('ses_shared')
expect(openEngine().search({ query: 'laterdatabasecopy' }).hits).toHaveLength(0)
})
it('settles a healthy physical full list when another family reports a concurrent refusal', async () => {
@@ -505,7 +523,11 @@ it('retains a restored old row when a family listing loses the earlier absence p
source.close()
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: { ...harness.roots, opencodeStorageDir: join(dirname(dbPath()), 'storage') },
roots: {
...harness.roots,
opencodeDbPaths: undefined,
opencodeStorageDir: join(dirname(dbPath()), 'storage')
},
historyDays: null,
clock,
discoveryDeadlineMs: 0,
@@ -551,3 +573,52 @@ it('retains a restored old row when a family listing loses the earlier absence p
listing.mockRestore()
}
})
it('settles a full attempt blocked by persistent family enumeration while retaining recovery', async () => {
writeOpenCodeSqliteDatabase(dbPath(), [
{ id: 'ses_family', turns: [{ role: 'user', parts: ['retainedfamilyneedle'] }] }
])
const subject = new SessionSearchIndexer({
databasePath: harness.databasePath,
roots: {
...harness.roots,
opencodeDbPaths: undefined,
opencodeStorageDir: join(dirname(dbPath()), 'storage')
},
historyDays: null,
clock,
fullSweepEveryCycles: 1000
})
indexer = subject
await subject.start()
const completed = subject.status().lastSweepCompletedAt
expect(openEngine().search({ query: 'retainedfamilyneedle' }).hits).toHaveLength(1)
const actual = filesystem.wslGatedReaddir
let refused = true
const reads = vi.spyOn(filesystem, 'wslGatedReaddir').mockImplementation((path, ...rest) => {
if (refused && path === dirname(dbPath())) {
return Promise.reject(Object.assign(new Error('fixture family refusal'), { code: 'EACCES' }))
}
return actual(path, ...rest)
})
try {
clock.costPerNowMs = 1
await subject.reconcile({ full: true })
for (let cycle = 0; cycle < 6; cycle++) {
clock.advance(20000)
await subject.settled()
}
expect(subject.status().lastSweepCompletedAt).not.toBe(completed)
expect(openEngine().search({ query: 'retainedfamilyneedle' }).hits).toHaveLength(1)
refused = false
appendOpenCodeSqliteTurn(dbPath(), 'ses_family', {
role: 'assistant',
parts: ['recoveredfamilyneedle']
})
await subject.reconcile({ full: true })
expect(openEngine().search({ query: 'recoveredfamilyneedle' }).hits).toHaveLength(1)
expect(subject.status().phase).toBe('current')
} finally {
reads.mockRestore()
}
})
@@ -40,11 +40,13 @@ export class SessionSearchPassState {
return (
this.discovery.pending ||
this.queue.size > 0 ||
this.candidates.pendingPublications ||
this.retirement !== null ||
this.retire.size > 0
)
}
close(): Promise<void> {
this.candidates.closePublications()
return this.discovery.close()
}
}
@@ -82,6 +84,7 @@ export async function runSessionSearchPass(
await store.purgeOlderThan(store.retentionCutoff, signal)
args.onPurged?.()
}
state.candidates.beginPublications(args.full || !!args.purge)
state.discovery.begin({
full: args.full,
maintenance: args.purge ?? args.full,
@@ -125,6 +128,13 @@ export async function runSessionSearchPass(
}
for (const candidate of update.candidates) {
state.parkedRetire.delete(candidate.file.path)
if (
state.discovery.forced.has(candidate.file.path) ||
update.forced.has(candidate.file.path) ||
state.discovery.metadata.has(candidate.file.path)
) {
state.candidates.requirePublication(candidate.file.path)
}
state.queue.enqueue(
candidate,
store,
@@ -133,10 +143,13 @@ export async function runSessionSearchPass(
state.discovery.metadata.has(candidate.file.path)
)
}
const readOverdue = args.startReadDeadline?.() ?? (() => false)
const read = await runSessionSearchIndexPass(store, state.queue, {
signal,
overdue: args.startReadDeadline?.()
overdue: readOverdue,
publications: state.candidates
})
await state.candidates.reconcilePublications(store, readOverdue, signal)
const listings = state.discovery.listings()
const roots = listings.map((listing) => listing.root)
const rootsWithFiles = new Set(
@@ -41,12 +41,12 @@ export type SessionSearchIndexStore = Pick<
| 'fileRows'
| 'setFileState'
| 'stateCounts'
| 'removeFile'
| 'purgeOlderThan'
| 'close'
| 'retentionCutoff'
| 'setRetentionCutoffMs'
> & {
removeFile(path: string): boolean | Promise<boolean>
readonly acceptingReads?: boolean
prepare?(): Promise<void>
flush?(): Promise<void>
@@ -276,11 +276,13 @@ export class SessionSearchStore {
}
/** Only proven deletion permits removal; unreadable sources retain their indexed rows. */
removeFile(path: string): void {
removeFile(path: string): boolean {
try {
this.writer.removeFile(path)
return true
} catch (error) {
this.onError(error)
return false
}
}
@@ -1,3 +1,4 @@
import { SessionInputFrontier } from '../ai-vault/session-input-frontier'
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
import { fileIdentity } from './session-search-file-cursor'
import { sessionSearchReadDecision } from './session-search-read-decision'
@@ -12,8 +13,8 @@ export type SessionSearchWork = {
/** A unique path owns its debt even when it leaves the next recent list. */
export class SessionSearchWorkQueue {
private readonly backlog = new Map<string, SessionSearchWork>()
private readonly changed = new Map<string, SessionSearchWork>()
private readonly backlog = new SessionInputFrontier<SessionSearchWork>()
private readonly changed = new SessionInputFrontier<SessionSearchWork>()
private version = 0
private chooseChanged = true
private due = new Set<string>()
@@ -64,10 +65,7 @@ export class SessionSearchWorkQueue {
next(): SessionSearchWork | undefined {
const preferred = this.chooseChanged ? this.changed : this.backlog
this.chooseChanged = !this.chooseChanged
return (
preferred.values().next().value ??
(preferred === this.changed ? this.backlog : this.changed).values().next().value
)
return preferred.first ?? (preferred === this.changed ? this.backlog : this.changed).first
}
acknowledge(work: SessionSearchWork, retry: boolean): void {
@@ -230,7 +230,7 @@ async function execute({ id, command: c }: SearchWorkerRequest): Promise<SearchW
path = c.path
break
case 'remove':
store.removeFile(c.path)
accepted = store.removeFile(c.path)
path = c.path
break
case 'purge': {
@@ -146,9 +146,24 @@ export class SessionSearchWorkerStoreBase extends SessionSearchCursorSnapshot {
void this.send({ op: 'state', path, state, mtime })
}
}
removeFile(path: string): void {
if (this.acceptingReads) {
void this.send({ op: 'remove', path })
async removeFile(path: string): Promise<boolean> {
if (!this.acceptingReads) {
return false
}
const epoch = this.client.epoch,
snapshotEpoch = this.snapshotEpoch
try {
const response = await this.send({ op: 'remove', path })
return (
this.acceptingReads &&
epoch === this.client.epoch &&
snapshotEpoch === this.snapshotEpoch &&
response.accepted === true &&
!response.failure &&
!this.committed.has(path)
)
} catch {
return false
}
}
async purgeOlderThan(cutoff: number | null, signal?: AbortSignal): Promise<void> {
@@ -43,6 +43,7 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
const owner = new AbortController()
this.invalidations.set(owner, candidate.file.path)
const readSignal = signal ? AbortSignal.any([owner.signal, signal]) : owner.signal
let settlement: Promise<boolean> = Promise.resolve(false)
let ended = false
let cancelled = false
let failed = false
@@ -124,6 +125,7 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
.catch(() => undefined)
}
return {
settled: () => settlement,
failed: () => failed,
cancelled: () => !active(),
drain: async () => {
@@ -186,14 +188,19 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
contentHash: hash
}
this.speculative.set(candidate.file.path, speculative)
void post({ op: 'finish', handle, outcome, hash })
.then(() => {
settlement = post({ op: 'finish', handle, outcome, hash })
.then((response) => {
const accepted =
response.accepted === true &&
valid() &&
this.committed.get(candidate.file.path)?.state === 'current'
if (this.speculative.get(candidate.file.path) === speculative) {
this.speculative.delete(candidate.file.path)
}
this.invalidations.delete(owner)
return accepted
})
.catch(() => undefined)
.catch(() => false)
return true
},
discard: () => {
@@ -1,4 +1,8 @@
import { currentSessionDiscovery, discoverSessionContainer } from '../../session-discovery-context'
import {
currentSessionDiscovery,
discoverSessionContainer,
discoverSessionContainerPaths
} from '../../session-discovery-context'
import { basename, dirname, join } from 'node:path'
import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types'
import type { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error'
@@ -28,16 +32,12 @@ export function opencodeDiscoveries(
): Promise<SessionFileDiscovery>[] {
return opencodeSources(options, wslHomeDirs).flatMap((source) => {
const { storageDir } = source
const enumerationIssues: AiVaultScanIssue[] = []
const paths = source.dbPaths(enumerationIssues).then((dbPaths) => {
issues.push(...enumerationIssues)
return dbPaths
})
const paths = discoverSessionContainerPaths(storageDir, source.dbPaths, issues)
return [
paths.then((dbPaths) =>
paths.then(({ paths: dbPaths, complete }) =>
discoverOpenCodeSessions({
storageDir,
enumerationComplete: !enumerationIssues.some((issue) => issue.kind !== 'notice'),
enumerationComplete: complete,
dbPaths: dbPaths.filter((path) => !isOpenCodeV2DatabaseName(basename(path))),
limitPerAgent: limit,
issues,
@@ -46,15 +46,8 @@ export function opencodeDiscoveries(
})
),
// Current releases share opencode.db with v1; the worker checks for v2 tables.
paths.then((dbPaths) =>
discoverOpenCode2Sessions(
storageDir,
dbPaths,
limit,
issues,
options,
!enumerationIssues.some((issue) => issue.kind !== 'notice')
)
paths.then(({ paths: dbPaths, complete }) =>
discoverOpenCode2Sessions(storageDir, dbPaths, limit, issues, options, complete)
)
]
})
@@ -14,6 +14,11 @@ export type SessionDiscoveryDependency = {
}
export type SessionDiscoveryContext = {
containerPaths(
root: string,
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
issues: AiVaultScanIssue[]
): Promise<{ paths: readonly string[]; complete: boolean }>
files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery>
container(
key: string,
@@ -49,3 +54,15 @@ export function discoverSessionContainer(
read(paths, limit)
)
}
export async function discoverSessionContainerPaths(
root: string,
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
issues: AiVaultScanIssue[]
): Promise<{ paths: readonly string[]; complete: boolean }> {
const context = currentSessionDiscovery()
if (context) {
return context.containerPaths(root, read, issues)
}
return { paths: await read(issues), complete: true }
}
+6 -6
View File
@@ -3,6 +3,7 @@ import {
type SessionInventoryDirectory as Directory
} from './session-file-inventory-directories'
import type { Dirent } from 'node:fs'
import { SessionInputFrontier } from './session-input-frontier'
import { SessionObservationRetries } from './session-observation-retries'
import { dirname, join, relative } from 'node:path'
import { wslGatedReaddir, wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
@@ -35,8 +36,8 @@ export class SessionFileInventory {
return this.observations.dependencies
}
private readonly directories: SessionInventoryDirectories
private readonly dirty = new Map<string, boolean>()
private readonly walks = new Map<string, Walk>()
private readonly dirty = new SessionInputFrontier<boolean>()
private readonly walks = new SessionInputFrontier<Walk>()
private readonly trailing = new Set<string>()
private readonly failures = new SessionObservationRetries()
private reconcileAgain = false
@@ -133,15 +134,15 @@ export class SessionFileInventory {
): Promise<{ changes: SessionFileChange[]; removed: string[]; removedDirectories: string[] }> {
let consumed = 0
while ((this.dirty.size || this.walks.size) && (consumed === 0 || !overdue())) {
consumed++
if (this.dirty.size && (!this.walks.size || this.chooseDirty)) {
this.chooseDirty = false
await this.drainDirty(args, recentLimit)
consumed++
continue
}
this.chooseDirty = true
throwIfAiVaultScanCancelled(args.signal)
const walk = this.walks.values().next().value!
const walk = this.walks.first!
if (!this.failures.mayAttempt(walk.path)) {
this.walks.delete(walk.path)
continue
@@ -201,7 +202,6 @@ export class SessionFileInventory {
}
walk.offset++
}
consumed++
if (walk.offset === walk.entries.length) {
this.walks.delete(walk.path)
if (this.trailing.delete(walk.path)) {
@@ -226,7 +226,7 @@ export class SessionFileInventory {
}
private async drainDirty(args: DiscoverSessionFilesArgs, recentLimit: number): Promise<void> {
const first = this.dirty.entries().next().value
const first = this.dirty.firstEntry()
if (!first) {
return
}
@@ -0,0 +1,58 @@
type Link<T> = { key: string; value: T; previous?: Link<T>; next?: Link<T> }
/** Keyed obligations retain their FIFO position until acknowledged or cancelled. */
export class SessionInputFrontier<T> {
private readonly members = new Map<string, Link<T>>()
private head: Link<T> | undefined
private tail: Link<T> | undefined
get size(): number {
return this.members.size
}
get first(): T | undefined {
return this.head?.value
}
firstEntry(): readonly [string, T] | undefined {
return this.head ? [this.head.key, this.head.value] : undefined
}
has(key: string): boolean {
return this.members.has(key)
}
get(key: string): T | undefined {
return this.members.get(key)?.value
}
set(key: string, value: T): void {
const current = this.members.get(key)
if (current) {
current.value = value
return
}
const link: Link<T> = { key, value, previous: this.tail }
if (this.tail) {
this.tail.next = link
} else {
this.head = link
}
this.tail = link
this.members.set(key, link)
}
delete(key: string): void {
const link = this.members.get(key)
if (!link) {
return
}
if (link.previous) {
link.previous.next = link.next
} else {
this.head = link.next
}
if (link.next) {
link.next.previous = link.previous
} else {
this.tail = link.previous
}
this.members.delete(key)
link.previous = undefined
link.next = undefined
}
}