mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Withdraw superseded search copies after acknowledged publication
Retain accepted alias output through replacement failures, fence suppressed search reads, and acknowledge derived-output cleanup. Own refused family enumeration and bound discovery work with linked frontiers. Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
co-authored by
Jinjing
parent
685ff776dc
commit
f3f6db8f0e
@@ -0,0 +1,145 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import { SessionSearchCandidateInventory } from './session-search-candidate-inventory'
|
||||
import { SessionSearchIndexConsumer } from './session-search-index-consumer'
|
||||
import { SessionSearchEngine } from './session-search-engine'
|
||||
import { SessionSearchStore } from './session-search-store'
|
||||
import {
|
||||
openSessionSearchIndexFile,
|
||||
syntheticCandidate,
|
||||
syntheticSession,
|
||||
type SessionSearchIndexFile
|
||||
} from './session-search-index-test-fixture'
|
||||
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
|
||||
|
||||
let index: SessionSearchIndexFile
|
||||
let store: SessionSearchStore
|
||||
let inventory: SessionSearchCandidateInventory
|
||||
let engine: SessionSearchEngine
|
||||
const first = candidate('opencode-first.db', 200)
|
||||
const previous = candidate('opencode-previous.db', 100)
|
||||
const latest = candidate('opencode-latest.db', 300)
|
||||
|
||||
function candidate(database: string, mtimeMs: number): SessionFileCandidate {
|
||||
return {
|
||||
...syntheticCandidate({ path: `/archive/${database}#shared`, mtimeMs }),
|
||||
agent: 'opencode',
|
||||
file: { path: `/archive/${database}#shared`, mtimeMs, modifiedAt: '2025-01-01' }
|
||||
}
|
||||
}
|
||||
function discover(candidates: SessionFileCandidate[], rootDir = '/archive/storage') {
|
||||
return { agent: 'opencode' as const, rootDir, files: candidates.map((entry) => entry.file) }
|
||||
}
|
||||
function reader(
|
||||
input: SessionFileCandidate,
|
||||
owner: SessionSearchCandidateInventory | undefined = inventory
|
||||
) {
|
||||
return new SessionSearchIndexConsumer(store, undefined, owner).beginRead({
|
||||
candidate: input,
|
||||
mode: 'replace',
|
||||
previousByteOffset: 0
|
||||
})
|
||||
}
|
||||
function finish(input: SessionFileCandidate, text: string, empty = false, owner = inventory) {
|
||||
const consumer = reader(input, owner)!
|
||||
consumer.message({ role: 'user', text, timestamp: null })
|
||||
consumer.finish({
|
||||
session: empty ? null : syntheticSession({ agent: 'opencode', filePath: input.file.path }),
|
||||
byteOffset: 100,
|
||||
incomplete: false
|
||||
})
|
||||
}
|
||||
async function settle(wake = false) {
|
||||
inventory.beginPublications(wake)
|
||||
await inventory.reconcilePublications(store, () => false)
|
||||
}
|
||||
function hits(text: string) {
|
||||
return engine.search({ query: text }).hits.length
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
index = await openSessionSearchIndexFile('ss-alias-publication')
|
||||
store = new SessionSearchStore(index.path)
|
||||
inventory = new SessionSearchCandidateInventory()
|
||||
engine = new SessionSearchEngine(index.db)
|
||||
})
|
||||
afterEach(async () => {
|
||||
inventory.closePublications()
|
||||
vi.restoreAllMocks()
|
||||
store.close()
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it('keeps the published predecessor through deferred, incomplete and held-out replacement reads', async () => {
|
||||
await inventory.update([discover([previous])], new Set(), {})
|
||||
finish(previous, 'predecessorneedle')
|
||||
await inventory.update([discover([first])], new Set(), {})
|
||||
await settle()
|
||||
expect(hits('predecessorneedle')).toBe(1)
|
||||
reader(first)!.finish({ session: null, byteOffset: 0, incomplete: true })
|
||||
await settle()
|
||||
expect(hits('predecessorneedle')).toBe(1)
|
||||
store.setFileState(first.file.path, 'failed', first.file.mtimeMs)
|
||||
store.setFileState(first.file.path, 'failed', first.file.mtimeMs)
|
||||
await settle()
|
||||
expect(hits('predecessorneedle')).toBe(1)
|
||||
finish(first, 'replacementneedle')
|
||||
await settle()
|
||||
expect(hits('replacementneedle')).toBe(1)
|
||||
expect(hits('predecessorneedle')).toBe(0)
|
||||
expect(reader(previous)).toBeNull()
|
||||
// The suppressed raw member remains available for promotion after positive removal of its winner.
|
||||
await inventory.update([], new Set([first.file.path]), {})
|
||||
finish(previous, 'promotedneedle')
|
||||
expect(hits('promotedneedle')).toBe(1)
|
||||
})
|
||||
|
||||
it('reconstructs cleanup from committed rows after restart and retries acknowledged withdrawal', async () => {
|
||||
// These committed rows model a crash after replacement publication but before predecessor cleanup.
|
||||
const detached = new SessionSearchCandidateInventory()
|
||||
finish(previous, 'predecessorneedle', false, detached)
|
||||
finish(first, 'replacementneedle', false, detached)
|
||||
await inventory.update([discover([first, previous])], new Set(), {})
|
||||
const remove = vi.spyOn(store, 'removeFile').mockReturnValue(false)
|
||||
for (let cycle = 0; cycle < 4; cycle++) {
|
||||
await settle()
|
||||
}
|
||||
expect(hits('predecessorneedle')).toBe(1)
|
||||
expect(inventory.pendingPublications).toBe(false)
|
||||
remove.mockRestore()
|
||||
await settle(true)
|
||||
expect(hits('replacementneedle')).toBe(1)
|
||||
expect(hits('predecessorneedle')).toBe(0)
|
||||
expect(reader(previous)).toBeNull()
|
||||
})
|
||||
|
||||
it('fences stale search publication without blocking the foreground parser or removing the last good copy', async () => {
|
||||
await inventory.update([discover([previous])], new Set(), {})
|
||||
finish(previous, 'predecessorneedle')
|
||||
await inventory.update([discover([first])], new Set(), {})
|
||||
const stale = reader(first)!
|
||||
stale.message({ role: 'user', text: 'staleackneedle', timestamp: null })
|
||||
await inventory.update([discover([latest])], new Set(), {})
|
||||
stale.finish({ session: syntheticSession(), byteOffset: 100, incomplete: false })
|
||||
await settle()
|
||||
expect(hits('staleackneedle')).toBe(0)
|
||||
expect(hits('predecessorneedle')).toBe(1)
|
||||
finish(latest, 'currentwinnerneedle')
|
||||
await settle()
|
||||
expect(hits('predecessorneedle')).toBe(0)
|
||||
expect(hits('currentwinnerneedle')).toBe(1)
|
||||
})
|
||||
|
||||
it('treats successful empty publication as authoritative while protecting a path selected by another group', async () => {
|
||||
await inventory.update([discover([previous])], new Set(), {})
|
||||
finish(previous, 'predecessorneedle')
|
||||
await inventory.update([discover([first])], new Set(), {})
|
||||
finish(first, 'excludedneedle', true)
|
||||
await settle()
|
||||
expect(hits('predecessorneedle')).toBe(0)
|
||||
expect(hits('excludedneedle')).toBe(0)
|
||||
|
||||
await inventory.update([discover([previous], '/another/storage')], new Set(), {})
|
||||
finish(previous, 'sharedpathneedle')
|
||||
await settle()
|
||||
expect(hits('sharedpathneedle')).toBe(1)
|
||||
})
|
||||
@@ -1,3 +1,8 @@
|
||||
import {
|
||||
SessionSearchCandidatePublications,
|
||||
rows,
|
||||
type Row
|
||||
} from './session-search-candidate-publication'
|
||||
import { splitOpenCodeSqliteCandidate } from '../../shared/opencode-sqlite-session-path'
|
||||
import { basename, dirname, extname } from 'node:path'
|
||||
import { sessionFileExecutionNamespace } from '../../shared/ai-vault/scanning/ai-vault-scope-cwd-match'
|
||||
@@ -11,11 +16,6 @@ import type {
|
||||
SessionFileDiscovery
|
||||
} from '../ai-vault/session-scanner-types'
|
||||
|
||||
type Row = { candidate: SessionFileCandidate; root: string }
|
||||
type Group = Row | Map<string, Row>
|
||||
function rows(group: Group | undefined): Row[] {
|
||||
return group ? (group instanceof Map ? [...group.values()] : [group]) : []
|
||||
}
|
||||
function rowId(row: Row): string {
|
||||
return `${row.candidate.agent}:${row.candidate.file.path}`
|
||||
}
|
||||
@@ -33,11 +33,8 @@ function groupKey(candidate: SessionFileCandidate, root: string): string {
|
||||
}
|
||||
|
||||
/** Ordinary paths own one record; only competing aliases allocate a group and selected set. */
|
||||
export class SessionSearchCandidateInventory {
|
||||
export class SessionSearchCandidateInventory extends SessionSearchCandidatePublications {
|
||||
private readonly sourceOrders = new Map<string, ReadonlyMap<string, number>>()
|
||||
private readonly groups = new Map<string, Group>()
|
||||
private readonly byPath = new Map<string, string | Set<string>>()
|
||||
private readonly selected = new Map<string, Set<string>>()
|
||||
has(path: string, agent?: string | null): boolean {
|
||||
const owner = this.byPath.get(path)
|
||||
if (!agent) {
|
||||
@@ -65,12 +62,16 @@ export class SessionSearchCandidateInventory {
|
||||
}
|
||||
}
|
||||
const previous = new Map<string, Set<string>>()
|
||||
const affectedPaths = new Set<string>()
|
||||
const forced = new Set<string>()
|
||||
const touch = (key: string) => {
|
||||
if (previous.has(key)) {
|
||||
return
|
||||
}
|
||||
const group = this.groups.get(key)
|
||||
for (const { candidate } of rows(group)) {
|
||||
affectedPaths.add(candidate.file.path)
|
||||
}
|
||||
previous.set(
|
||||
key,
|
||||
this.selected.get(key) ??
|
||||
@@ -224,6 +225,7 @@ export class SessionSearchCandidateInventory {
|
||||
}
|
||||
candidates.push(...selected)
|
||||
}
|
||||
this.updatePublications(previous, affectedPaths)
|
||||
for (const path of suppressed) {
|
||||
const owner = this.byPath.get(path)
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
import { SessionInputFrontier } from '../ai-vault/session-input-frontier'
|
||||
import { SessionObservationRetries } from '../ai-vault/session-observation-retries'
|
||||
import { fileIdentity } from './session-search-file-cursor'
|
||||
import type { SessionSearchFileRow, SessionSearchIndexStore } from './session-search-store'
|
||||
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
|
||||
export type Row = { candidate: SessionFileCandidate; root: string }
|
||||
export type Group = Row | Map<string, Row>
|
||||
type Publication = {
|
||||
revision: number
|
||||
required: Set<string>
|
||||
withdrawals?: SessionInputFrontier<string>
|
||||
}
|
||||
export type SessionSearchPublicationLease = {
|
||||
signal: AbortSignal
|
||||
valid(): boolean
|
||||
committed(): void
|
||||
release(): void
|
||||
}
|
||||
export function rows(group: Group | undefined): Row[] {
|
||||
return group ? (group instanceof Map ? [...group.values()] : [group]) : []
|
||||
}
|
||||
export abstract class SessionSearchCandidatePublications {
|
||||
protected readonly groups = new Map<string, Group>()
|
||||
protected readonly byPath = new Map<string, string | Set<string>>()
|
||||
protected readonly selected = new Map<string, Set<string>>()
|
||||
private readonly publications = new Map<string, Publication>()
|
||||
private readonly dirtyPublications = new SessionInputFrontier<Publication>()
|
||||
private readonly withdrawalRetries = new SessionObservationRetries()
|
||||
private readonly readers = new Map<string, Set<AbortController>>()
|
||||
private revision = 0
|
||||
|
||||
private owners(path: string): readonly string[] {
|
||||
const owner = this.byPath.get(path)
|
||||
return typeof owner === 'string' ? [owner] : [...(owner ?? [])]
|
||||
}
|
||||
private isSelected(path: string): boolean {
|
||||
return this.owners(path).some((key) => {
|
||||
const group = this.groups.get(key)
|
||||
return group instanceof Map
|
||||
? this.selected.get(key)?.has(path)
|
||||
: group?.candidate.file.path === path
|
||||
})
|
||||
}
|
||||
beginPublications(wake: boolean): void {
|
||||
this.withdrawalRetries.begin(wake)
|
||||
if (wake) {
|
||||
for (const key of this.withdrawalRetries.keys()) {
|
||||
const publication = this.publications.get(key)
|
||||
if (publication) {
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
get pendingPublications(): boolean {
|
||||
return this.dirtyPublications.size > 0
|
||||
}
|
||||
requirePublication(path: string): void {
|
||||
for (const key of this.owners(path)) {
|
||||
const publication = this.publications.get(key)
|
||||
if (publication && this.selected.get(key)?.has(path)) {
|
||||
publication.required.add(path)
|
||||
publication.withdrawals = undefined
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
}
|
||||
rowChanged(path: string, row: SessionSearchFileRow | undefined): void {
|
||||
for (const key of this.owners(path)) {
|
||||
const publication = this.publications.get(key)
|
||||
if (publication) {
|
||||
if (this.selected.get(key)?.has(path)) {
|
||||
publication.withdrawals = undefined
|
||||
} else if (row) {
|
||||
publication.withdrawals?.set(path, path)
|
||||
}
|
||||
if (!this.withdrawalRetries.heldOut(key)) {
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
beginPublication(candidate: SessionFileCandidate): SessionSearchPublicationLease | null {
|
||||
const path = candidate.file.path
|
||||
if (this.byPath.has(path) && !this.isSelected(path)) {
|
||||
return null
|
||||
}
|
||||
const owner = new AbortController()
|
||||
const readers = this.readers.get(path) ?? new Set<AbortController>()
|
||||
readers.add(owner)
|
||||
this.readers.set(path, readers)
|
||||
const revisions = new Map(
|
||||
this.owners(path).map((key) => [key, this.publications.get(key)?.revision])
|
||||
)
|
||||
const valid = () =>
|
||||
!owner.signal.aborted &&
|
||||
(!this.byPath.has(path) || this.isSelected(path)) &&
|
||||
this.owners(path).every(
|
||||
(key) => revisions.has(key) && revisions.get(key) === this.publications.get(key)?.revision
|
||||
)
|
||||
return {
|
||||
signal: owner.signal,
|
||||
valid,
|
||||
committed: () => {
|
||||
if (!valid()) {
|
||||
return
|
||||
}
|
||||
for (const key of this.owners(path)) {
|
||||
const publication = this.publications.get(key)
|
||||
if (publication && this.selected.get(key)?.has(path)) {
|
||||
publication.required.delete(path)
|
||||
publication.withdrawals = undefined
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
},
|
||||
release: () => {
|
||||
readers.delete(owner)
|
||||
if (!readers.size) {
|
||||
this.readers.delete(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
async reconcilePublications(
|
||||
store: SessionSearchIndexStore,
|
||||
overdue: () => boolean,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const available = this.dirtyPublications.size
|
||||
let consumed = 0
|
||||
while (this.dirtyPublications.size && consumed < available && (consumed === 0 || !overdue())) {
|
||||
if (signal?.aborted) {
|
||||
return
|
||||
}
|
||||
const [key, publication] = this.dirtyPublications.firstEntry()!
|
||||
this.dirtyPublications.delete(key)
|
||||
consumed++
|
||||
if (!this.withdrawalRetries.mayAttempt(key)) {
|
||||
if (!this.withdrawalRetries.heldOut(key)) {
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const members = rows(this.groups.get(key))
|
||||
const selected = this.selected.get(key)
|
||||
if (
|
||||
!selected?.size ||
|
||||
!members
|
||||
.filter((row) => selected.has(row.candidate.file.path))
|
||||
.every(
|
||||
({ candidate }) =>
|
||||
!publication.required.has(candidate.file.path) && this.covered(store, candidate)
|
||||
)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
if (!publication.withdrawals) {
|
||||
publication.withdrawals = new SessionInputFrontier<string>()
|
||||
for (const { candidate } of members) {
|
||||
const path = candidate.file.path
|
||||
if (!selected.has(path) && store.fileRow(path)) {
|
||||
publication.withdrawals.set(path, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
let attempted = 0
|
||||
while (publication.withdrawals.size && (attempted === 0 || !overdue())) {
|
||||
if (signal?.aborted || this.publications.get(key) !== publication) {
|
||||
return
|
||||
}
|
||||
const path = publication.withdrawals.first!
|
||||
// Another source can still select this physical path.
|
||||
if (this.isSelected(path)) {
|
||||
publication.withdrawals.delete(path)
|
||||
continue
|
||||
}
|
||||
attempted++
|
||||
const accepted = await store.removeFile(path)
|
||||
if (
|
||||
signal?.aborted ||
|
||||
this.publications.get(key) !== publication ||
|
||||
this.isSelected(path)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (!accepted) {
|
||||
this.withdrawalRetries.fail(key)
|
||||
break
|
||||
}
|
||||
publication.withdrawals.delete(path)
|
||||
this.withdrawalRetries.delete(key)
|
||||
}
|
||||
this.dirtyPublications.delete(key)
|
||||
if (publication.withdrawals.size && !this.withdrawalRetries.heldOut(key)) {
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
}
|
||||
private covered(store: SessionSearchIndexStore, candidate: SessionFileCandidate): boolean {
|
||||
const row = store.fileRow(candidate.file.path)
|
||||
const identity = fileIdentity(candidate.file)
|
||||
return (
|
||||
store.acceptingReads !== false &&
|
||||
!!row &&
|
||||
row.state === 'current' &&
|
||||
row.mtimeMs === candidate.file.mtimeMs &&
|
||||
row.byteOffset !== null &&
|
||||
row.byteOffset !== undefined &&
|
||||
row.byteOffset >= 0 &&
|
||||
(candidate.file.sizeBytes === undefined ||
|
||||
(row.sizeBytes === candidate.file.sizeBytes &&
|
||||
row.byteOffset === candidate.file.sizeBytes)) &&
|
||||
(!identity ||
|
||||
(!!row.identity && identity.dev === row.identity.dev && identity.ino === row.identity.ino))
|
||||
)
|
||||
}
|
||||
closePublications(): void {
|
||||
for (const readers of this.readers.values()) {
|
||||
for (const reader of readers) {
|
||||
reader.abort()
|
||||
}
|
||||
}
|
||||
this.readers.clear()
|
||||
this.publications.clear()
|
||||
}
|
||||
protected updatePublications(
|
||||
previous: ReadonlyMap<string, ReadonlySet<string>>,
|
||||
affectedPaths: Set<string>
|
||||
): void {
|
||||
for (const key of previous.keys()) {
|
||||
for (const { candidate } of rows(this.groups.get(key))) {
|
||||
affectedPaths.add(candidate.file.path)
|
||||
}
|
||||
if (!(this.groups.get(key) instanceof Map)) {
|
||||
this.publications.delete(key)
|
||||
this.dirtyPublications.delete(key)
|
||||
this.withdrawalRetries.delete(key)
|
||||
}
|
||||
}
|
||||
const changedGroups = new Set(previous.keys())
|
||||
for (const path of affectedPaths) {
|
||||
for (const reader of this.readers.get(path) ?? []) {
|
||||
reader.abort()
|
||||
}
|
||||
for (const key of this.owners(path)) {
|
||||
changedGroups.add(key)
|
||||
}
|
||||
}
|
||||
for (const key of changedGroups) {
|
||||
if (this.groups.get(key) instanceof Map) {
|
||||
const old = this.publications.get(key)
|
||||
const publication = {
|
||||
revision: ++this.revision,
|
||||
required: new Set(
|
||||
[...(old?.required ?? [])].filter((path) => this.selected.get(key)?.has(path))
|
||||
)
|
||||
}
|
||||
this.publications.set(key, publication)
|
||||
this.withdrawalRetries.delete(key)
|
||||
this.dirtyPublications.set(key, publication)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { recordSessionScanIssue } from '../ai-vault/session-scan-issues'
|
||||
import { SessionObservationRetries } from '../ai-vault/session-observation-retries'
|
||||
import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
|
||||
@@ -20,6 +21,11 @@ type Family = {
|
||||
|
||||
/** Recent observations add facts; only a successful exhaustive observation can withdraw them. */
|
||||
export class SessionSearchContainerInventory {
|
||||
private readonly pathLists = new Map<
|
||||
string,
|
||||
{ paths: readonly string[]; complete: boolean; issues: AiVaultScanIssue[] }
|
||||
>()
|
||||
private seenPathLists = new Set<string>()
|
||||
private readonly families = new Map<string, Family>()
|
||||
private seen = new Set<string>()
|
||||
private recentLimit = 12
|
||||
@@ -29,6 +35,7 @@ export class SessionSearchContainerInventory {
|
||||
|
||||
begin(full: boolean, recentLimit: number): void {
|
||||
this.seen = new Set()
|
||||
this.seenPathLists = new Set()
|
||||
this.retries.begin(full)
|
||||
this.recentLimit = recentLimit
|
||||
this.removed.clear()
|
||||
@@ -41,7 +48,15 @@ export class SessionSearchContainerInventory {
|
||||
}
|
||||
}
|
||||
get pending(): boolean {
|
||||
for (const [root, list] of this.pathLists) {
|
||||
if (!list.complete && !this.retries.heldOut(`paths:${root}`)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for (const [key, family] of this.families) {
|
||||
if (this.retries.heldOut(`paths:${key.slice(key.indexOf(':') + 1)}`)) {
|
||||
continue
|
||||
}
|
||||
for (const [path, physical] of family.physical) {
|
||||
if (physical.fullDue && !this.retries.heldOut(`${key}:${path}`)) {
|
||||
return true
|
||||
@@ -50,6 +65,37 @@ export class SessionSearchContainerInventory {
|
||||
}
|
||||
return false
|
||||
}
|
||||
async containerPaths(
|
||||
root: string,
|
||||
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
|
||||
issues: AiVaultScanIssue[]
|
||||
): Promise<{ paths: readonly string[]; complete: boolean }> {
|
||||
this.seenPathLists.add(root)
|
||||
const key = `paths:${root}`
|
||||
const previous = this.pathLists.get(root)
|
||||
if (!this.retries.mayAttempt(key)) {
|
||||
for (const issue of previous?.issues ?? []) {
|
||||
recordSessionScanIssue(issues, issue)
|
||||
}
|
||||
return previous ?? { paths: [], complete: false }
|
||||
}
|
||||
const ownIssues: AiVaultScanIssue[] = []
|
||||
const paths = await read(ownIssues)
|
||||
const complete = !ownIssues.some((issue) => issue.kind !== 'notice')
|
||||
const result = {
|
||||
paths: complete ? paths : [...new Set([...(previous?.paths ?? []), ...paths])],
|
||||
complete,
|
||||
issues: ownIssues
|
||||
}
|
||||
this.pathLists.set(root, result)
|
||||
issues.push(...ownIssues)
|
||||
if (complete) {
|
||||
this.retries.delete(key)
|
||||
} else {
|
||||
this.retries.fail(key)
|
||||
}
|
||||
return result
|
||||
}
|
||||
async discover(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
@@ -95,7 +141,10 @@ export class SessionSearchContainerInventory {
|
||||
this.release(key, path, family, physical)
|
||||
continue
|
||||
}
|
||||
const full = physical.fullDue && this.retries.mayAttempt(retryKey)
|
||||
const full =
|
||||
physical.fullDue &&
|
||||
!this.retries.heldOut(`paths:${key.slice(key.indexOf(':') + 1)}`) &&
|
||||
this.retries.mayAttempt(retryKey)
|
||||
const discovery = await read([path], full ? Infinity : this.recentLimit)
|
||||
family.discovery = discovery
|
||||
throwIfAiVaultScanCancelled(signal)
|
||||
@@ -162,6 +211,12 @@ export class SessionSearchContainerInventory {
|
||||
return { ...family.discovery, files: changes, sourceOrder: family.sourceOrder }
|
||||
}
|
||||
end(): void {
|
||||
for (const root of this.pathLists.keys()) {
|
||||
if (!this.seenPathLists.has(root)) {
|
||||
this.pathLists.delete(root)
|
||||
this.retries.delete(`paths:${root}`)
|
||||
}
|
||||
}
|
||||
for (const [key, family] of this.families) {
|
||||
if (!this.seen.has(key)) {
|
||||
for (const [path, physical] of family.physical) {
|
||||
@@ -172,6 +227,7 @@ export class SessionSearchContainerInventory {
|
||||
}
|
||||
}
|
||||
clear(): void {
|
||||
this.pathLists.clear()
|
||||
this.families.clear()
|
||||
this.retries.clear()
|
||||
this.proofs.clear()
|
||||
|
||||
@@ -213,6 +213,14 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
}
|
||||
}
|
||||
|
||||
containerPaths(
|
||||
root: string,
|
||||
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
|
||||
issues: AiVaultScanIssue[]
|
||||
) {
|
||||
return this.containers.containerPaths(root, read, issues)
|
||||
}
|
||||
|
||||
container(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
|
||||
@@ -318,7 +318,7 @@ it.each(['first-file', 'accepted-file', 'directory'] as const)(
|
||||
async (kind) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-refused-observation')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const directory = join(harness.claudeProjectDir, 'refused')
|
||||
const directory = join(harness.roots.claudeProjectsDir, 'refused')
|
||||
const path = join(directory, 'history.jsonl')
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
@@ -532,3 +532,57 @@ it.each([40, 400])(
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.each([40, 400])(
|
||||
'charges refused directory attempts to the expired %s-directory discovery budget',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-refused-budget')
|
||||
const root = harness.claudeProjectDir
|
||||
for (let index = 0; index < population; index++) {
|
||||
await mkdir(join(root, `child-${index}`), { recursive: true })
|
||||
}
|
||||
const inventory = new SessionFileInventory(root)
|
||||
const args = {
|
||||
rootDir: root,
|
||||
agent: 'claude' as const,
|
||||
limit: Infinity,
|
||||
extensions: ['.jsonl'],
|
||||
issues: []
|
||||
}
|
||||
let quanta = 0
|
||||
await inventory.drain(args, () => ++quanta >= Math.ceil(population / 16), 12)
|
||||
const actual = filesystem.wslGatedReaddir
|
||||
let refused = true
|
||||
const reads = vi.spyOn(filesystem, 'wslGatedReaddir').mockImplementation((path, ...rest) => {
|
||||
if (refused && path.startsWith(`${root}/`)) {
|
||||
return Promise.reject(
|
||||
Object.assign(new Error('fixture refused directory'), { code: 'EACCES' })
|
||||
)
|
||||
}
|
||||
return actual(path, ...rest)
|
||||
})
|
||||
try {
|
||||
inventory.begin(false, args)
|
||||
await inventory.drain(args, () => true, 12)
|
||||
// An expired slice admits one attempted input, including a failed observation.
|
||||
expect(reads.mock.calls).toHaveLength(1)
|
||||
expect(inventory.pending).toBe(true)
|
||||
for (let cycle = 0; cycle < population * 3 && inventory.pending; cycle++) {
|
||||
inventory.begin(false, args)
|
||||
inventory.retryFailures()
|
||||
await inventory.drain(args, () => true, 12)
|
||||
}
|
||||
expect(inventory.pending).toBe(false)
|
||||
refused = false
|
||||
const recovered = join(root, 'child-0', 'recovered.jsonl')
|
||||
await writeFile(recovered, '{}\n')
|
||||
inventory.invalidate(join(root, 'child-0'))
|
||||
inventory.begin(false, args)
|
||||
const result = await inventory.drain(args, () => false, 12)
|
||||
expect(result.changes.map((change) => change.file.path)).toContain(recovered)
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import type {
|
||||
SessionSearchCandidatePublications,
|
||||
SessionSearchPublicationLease
|
||||
} from './session-search-candidate-publication'
|
||||
import { inSessionParseFileLane } from '../ai-vault/parse-cache/session-parse-file-lane'
|
||||
import { requestWholeTranscriptRead } from '../ai-vault/transcripts/session-transcript-reader'
|
||||
import { traceSessionSearch } from './session-search-trace'
|
||||
@@ -47,7 +51,8 @@ export const SELF_CONTAINED_TRANSCRIPT_AGENTS = {
|
||||
export class SessionSearchIndexConsumer implements TranscriptConsumer {
|
||||
constructor(
|
||||
private readonly store: SessionSearchIndexStore,
|
||||
private readonly onCommitted?: (write: SessionSearchFileWrite) => void
|
||||
private readonly onCommitted?: (write: SessionSearchFileWrite) => void,
|
||||
private readonly publications?: SessionSearchCandidatePublications
|
||||
) {}
|
||||
|
||||
resumableOffset(candidate: SessionFileCandidate): number | null {
|
||||
@@ -87,6 +92,15 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
|
||||
}
|
||||
// A restart dropped the parser's resume point; append past our cursor once the reread proves the prefix.
|
||||
const resume = start.mode === 'replace' ? this.provableCursor(start) : null
|
||||
const publication = this.publications?.beginPublication(candidate)
|
||||
if (this.publications && !publication) {
|
||||
return null
|
||||
}
|
||||
const signal = publication
|
||||
? start.signal
|
||||
? AbortSignal.any([start.signal, publication.signal])
|
||||
: publication.signal
|
||||
: start.signal
|
||||
const write = resume
|
||||
? this.store.beginWrite(
|
||||
candidate,
|
||||
@@ -94,7 +108,7 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
|
||||
resume.byteOffset,
|
||||
start.identity,
|
||||
start.indexing,
|
||||
start.signal
|
||||
signal
|
||||
)
|
||||
: this.store.beginWrite(
|
||||
candidate,
|
||||
@@ -102,12 +116,23 @@ export class SessionSearchIndexConsumer implements TranscriptConsumer {
|
||||
start.previousByteOffset,
|
||||
start.identity,
|
||||
start.indexing,
|
||||
start.signal
|
||||
signal
|
||||
)
|
||||
if (!write) {
|
||||
publication?.release()
|
||||
return null
|
||||
}
|
||||
return new SessionSearchReadConsumer(this.store, start, write, resume, this.onCommitted)
|
||||
if (publication) {
|
||||
publication.signal.addEventListener('abort', () => write.discard(), { once: true })
|
||||
}
|
||||
return new SessionSearchReadConsumer(
|
||||
this.store,
|
||||
start,
|
||||
write,
|
||||
resume,
|
||||
this.onCommitted,
|
||||
publication ?? undefined
|
||||
)
|
||||
}
|
||||
|
||||
private provableCursor(start: TranscriptReadStart): SessionSearchResumableCursor | null {
|
||||
@@ -128,7 +153,8 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
|
||||
private readonly start: TranscriptReadStart,
|
||||
private readonly write: SessionSearchFileWrite,
|
||||
private readonly resume: SessionSearchResumableCursor | null = null,
|
||||
private readonly onCommitted?: (write: SessionSearchFileWrite) => void
|
||||
private readonly onCommitted?: (write: SessionSearchFileWrite) => void,
|
||||
private readonly publication?: SessionSearchPublicationLease
|
||||
) {}
|
||||
|
||||
async drain(): Promise<void> {
|
||||
@@ -157,9 +183,15 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
|
||||
}
|
||||
|
||||
finish(outcome: TranscriptReadOutcome): void {
|
||||
if (outcome.cancelled || this.write.cancelled?.() || this.write.failed?.()) {
|
||||
if (
|
||||
outcome.cancelled ||
|
||||
this.publication?.valid() === false ||
|
||||
this.write.cancelled?.() ||
|
||||
this.write.failed?.()
|
||||
) {
|
||||
// Cancellation must not count toward the failure limit for a healthy file.
|
||||
this.write.discard()
|
||||
this.publication?.release()
|
||||
return
|
||||
}
|
||||
const { candidate } = this.start
|
||||
@@ -172,6 +204,7 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
|
||||
rescueRejections: 1,
|
||||
rescueRejection: 'prefix-mismatch-or-unreached'
|
||||
})
|
||||
this.publication?.release()
|
||||
return
|
||||
}
|
||||
let committed = false
|
||||
@@ -185,14 +218,29 @@ class SessionSearchReadConsumer implements TranscriptReadConsumer {
|
||||
}
|
||||
if (committed) {
|
||||
this.store.writeCommitted(candidate)
|
||||
this.onCommitted?.(this.write)
|
||||
const acknowledge = (accepted: boolean) => {
|
||||
if (accepted && this.publication?.valid() !== false) {
|
||||
this.publication?.committed()
|
||||
this.onCommitted?.(this.write)
|
||||
}
|
||||
this.publication?.release()
|
||||
}
|
||||
if (this.write.settled) {
|
||||
void this.write.settled().then(acknowledge, () => acknowledge(false))
|
||||
} else {
|
||||
acknowledge(true)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Stamp and count repeated read failures so unchanged unreadable files can be held out.
|
||||
this.store.setFileState(candidate.file.path, 'failed', candidate.file.mtimeMs)
|
||||
this.publication?.release()
|
||||
}
|
||||
}
|
||||
|
||||
export function registerSessionSearchIndexConsumer(store: SessionSearchIndexStore): () => void {
|
||||
return registerTranscriptConsumer(new SessionSearchIndexConsumer(store))
|
||||
export function registerSessionSearchIndexConsumer(
|
||||
store: SessionSearchIndexStore,
|
||||
publications?: SessionSearchCandidatePublications
|
||||
): () => void {
|
||||
return registerTranscriptConsumer(new SessionSearchIndexConsumer(store, undefined, publications))
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SessionSearchCandidateInventory } from './session-search-candidate-inventory'
|
||||
import {
|
||||
getSessionParseCacheEntry,
|
||||
sessionParseCacheKey
|
||||
@@ -18,7 +19,11 @@ import type { SessionSearchWorkQueue } from './session-search-work-queue'
|
||||
export async function runSessionSearchIndexPass(
|
||||
store: SessionSearchIndexStore,
|
||||
queue: SessionSearchWorkQueue,
|
||||
options: { signal?: AbortSignal; overdue?: () => boolean }
|
||||
options: {
|
||||
signal?: AbortSignal
|
||||
overdue?: () => boolean
|
||||
publications?: SessionSearchCandidateInventory
|
||||
}
|
||||
): Promise<{ stats: SessionParseStats; outOfTime: boolean; left: number }> {
|
||||
const stats = createSessionParseStats()
|
||||
// Retry once per cycle; rotating the failed path cannot monopolize a pass.
|
||||
@@ -65,9 +70,13 @@ export async function runSessionSearchIndexPass(
|
||||
true
|
||||
)
|
||||
let published: SessionSearchFileWrite | undefined
|
||||
const consumer = new SessionSearchIndexConsumer(store, (write) => {
|
||||
published = write
|
||||
}).beginRead({
|
||||
const consumer = new SessionSearchIndexConsumer(
|
||||
store,
|
||||
(write) => {
|
||||
published = write
|
||||
},
|
||||
options.publications
|
||||
).beginRead({
|
||||
candidate,
|
||||
mode: 'append',
|
||||
previousByteOffset: row.byteOffset,
|
||||
|
||||
@@ -43,6 +43,8 @@ type FileCursor = Pick<FileRow, 'session_row_id' | 'byte_offset'>
|
||||
export type SessionSearchFileWrite = {
|
||||
/** Commit named chunks at the ceiling; spill anonymous reads privately until atomic publication. */
|
||||
add(message: TranscriptMessage): void
|
||||
/** Resolves only after the writer acknowledges final publication. */
|
||||
settled?(): Promise<boolean>
|
||||
failed?(): boolean
|
||||
cancelled?(): boolean
|
||||
drain?(): Promise<void>
|
||||
|
||||
@@ -101,11 +101,15 @@ export class SessionSearchIndexer {
|
||||
() => {
|
||||
this.sweepNext = true
|
||||
},
|
||||
(path) => this.work.queue.rowChanged(path, this.store.fileRow(path))
|
||||
(path) => {
|
||||
const row = this.store.fileRow(path)
|
||||
this.work.queue.rowChanged(path, row)
|
||||
this.work.candidates.rowChanged(path, row)
|
||||
}
|
||||
)
|
||||
liveIndexerPaths.add(this.ownershipPath)
|
||||
this.store.setRetentionCutoffMs(this.cutoffMs())
|
||||
this.unregister = registerSessionSearchIndexConsumer(this.store)
|
||||
this.unregister = registerSessionSearchIndexConsumer(this.store, this.work.candidates)
|
||||
}
|
||||
|
||||
/** Begin inventory capture, then consume changes and retained debt at the normal cadence. */
|
||||
|
||||
@@ -67,6 +67,8 @@ vi.mock(
|
||||
}
|
||||
}
|
||||
)
|
||||
import { readWholeTranscript } from '../ai-vault/transcripts/session-transcript-reader'
|
||||
import { buildOpenCodeSqliteCandidatePath } from '../../shared/opencode-sqlite-session-path'
|
||||
import { resetSessionParseCacheForTests } from '../ai-vault/parse-cache/session-parse-cache-store'
|
||||
import { resetTranscriptConsumersForTests } from '../ai-vault/transcripts/session-transcript-consumers'
|
||||
import {
|
||||
@@ -421,6 +423,22 @@ it('chooses the original earlier database when an equal-time alias is imported l
|
||||
'ses_shared'
|
||||
)
|
||||
expect(openEngine().search({ query: 'laterdatabasecopy' }).hits).toHaveLength(0)
|
||||
// Opening the retained raw alias still parses it for History, without republishing search output.
|
||||
const foreground = await readWholeTranscript({
|
||||
candidate: {
|
||||
agent: 'opencode',
|
||||
codexHome: null,
|
||||
file: {
|
||||
path: buildOpenCodeSqliteCandidatePath(alternative, 'ses_shared'),
|
||||
mtimeMs: 1740000060000,
|
||||
modifiedAt: '2025-02-19T21:21:00.000Z'
|
||||
}
|
||||
},
|
||||
platform: process.platform
|
||||
})
|
||||
await subject.settled()
|
||||
expect(foreground?.sessionId).toBe('ses_shared')
|
||||
expect(openEngine().search({ query: 'laterdatabasecopy' }).hits).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('settles a healthy physical full list when another family reports a concurrent refusal', async () => {
|
||||
@@ -505,7 +523,11 @@ it('retains a restored old row when a family listing loses the earlier absence p
|
||||
source.close()
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: { ...harness.roots, opencodeStorageDir: join(dirname(dbPath()), 'storage') },
|
||||
roots: {
|
||||
...harness.roots,
|
||||
opencodeDbPaths: undefined,
|
||||
opencodeStorageDir: join(dirname(dbPath()), 'storage')
|
||||
},
|
||||
historyDays: null,
|
||||
clock,
|
||||
discoveryDeadlineMs: 0,
|
||||
@@ -551,3 +573,52 @@ it('retains a restored old row when a family listing loses the earlier absence p
|
||||
listing.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('settles a full attempt blocked by persistent family enumeration while retaining recovery', async () => {
|
||||
writeOpenCodeSqliteDatabase(dbPath(), [
|
||||
{ id: 'ses_family', turns: [{ role: 'user', parts: ['retainedfamilyneedle'] }] }
|
||||
])
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: {
|
||||
...harness.roots,
|
||||
opencodeDbPaths: undefined,
|
||||
opencodeStorageDir: join(dirname(dbPath()), 'storage')
|
||||
},
|
||||
historyDays: null,
|
||||
clock,
|
||||
fullSweepEveryCycles: 1000
|
||||
})
|
||||
indexer = subject
|
||||
await subject.start()
|
||||
const completed = subject.status().lastSweepCompletedAt
|
||||
expect(openEngine().search({ query: 'retainedfamilyneedle' }).hits).toHaveLength(1)
|
||||
const actual = filesystem.wslGatedReaddir
|
||||
let refused = true
|
||||
const reads = vi.spyOn(filesystem, 'wslGatedReaddir').mockImplementation((path, ...rest) => {
|
||||
if (refused && path === dirname(dbPath())) {
|
||||
return Promise.reject(Object.assign(new Error('fixture family refusal'), { code: 'EACCES' }))
|
||||
}
|
||||
return actual(path, ...rest)
|
||||
})
|
||||
try {
|
||||
clock.costPerNowMs = 1
|
||||
await subject.reconcile({ full: true })
|
||||
for (let cycle = 0; cycle < 6; cycle++) {
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
}
|
||||
expect(subject.status().lastSweepCompletedAt).not.toBe(completed)
|
||||
expect(openEngine().search({ query: 'retainedfamilyneedle' }).hits).toHaveLength(1)
|
||||
refused = false
|
||||
appendOpenCodeSqliteTurn(dbPath(), 'ses_family', {
|
||||
role: 'assistant',
|
||||
parts: ['recoveredfamilyneedle']
|
||||
})
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: 'recoveredfamilyneedle' }).hits).toHaveLength(1)
|
||||
expect(subject.status().phase).toBe('current')
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -40,11 +40,13 @@ export class SessionSearchPassState {
|
||||
return (
|
||||
this.discovery.pending ||
|
||||
this.queue.size > 0 ||
|
||||
this.candidates.pendingPublications ||
|
||||
this.retirement !== null ||
|
||||
this.retire.size > 0
|
||||
)
|
||||
}
|
||||
close(): Promise<void> {
|
||||
this.candidates.closePublications()
|
||||
return this.discovery.close()
|
||||
}
|
||||
}
|
||||
@@ -82,6 +84,7 @@ export async function runSessionSearchPass(
|
||||
await store.purgeOlderThan(store.retentionCutoff, signal)
|
||||
args.onPurged?.()
|
||||
}
|
||||
state.candidates.beginPublications(args.full || !!args.purge)
|
||||
state.discovery.begin({
|
||||
full: args.full,
|
||||
maintenance: args.purge ?? args.full,
|
||||
@@ -125,6 +128,13 @@ export async function runSessionSearchPass(
|
||||
}
|
||||
for (const candidate of update.candidates) {
|
||||
state.parkedRetire.delete(candidate.file.path)
|
||||
if (
|
||||
state.discovery.forced.has(candidate.file.path) ||
|
||||
update.forced.has(candidate.file.path) ||
|
||||
state.discovery.metadata.has(candidate.file.path)
|
||||
) {
|
||||
state.candidates.requirePublication(candidate.file.path)
|
||||
}
|
||||
state.queue.enqueue(
|
||||
candidate,
|
||||
store,
|
||||
@@ -133,10 +143,13 @@ export async function runSessionSearchPass(
|
||||
state.discovery.metadata.has(candidate.file.path)
|
||||
)
|
||||
}
|
||||
const readOverdue = args.startReadDeadline?.() ?? (() => false)
|
||||
const read = await runSessionSearchIndexPass(store, state.queue, {
|
||||
signal,
|
||||
overdue: args.startReadDeadline?.()
|
||||
overdue: readOverdue,
|
||||
publications: state.candidates
|
||||
})
|
||||
await state.candidates.reconcilePublications(store, readOverdue, signal)
|
||||
const listings = state.discovery.listings()
|
||||
const roots = listings.map((listing) => listing.root)
|
||||
const rootsWithFiles = new Set(
|
||||
|
||||
@@ -41,12 +41,12 @@ export type SessionSearchIndexStore = Pick<
|
||||
| 'fileRows'
|
||||
| 'setFileState'
|
||||
| 'stateCounts'
|
||||
| 'removeFile'
|
||||
| 'purgeOlderThan'
|
||||
| 'close'
|
||||
| 'retentionCutoff'
|
||||
| 'setRetentionCutoffMs'
|
||||
> & {
|
||||
removeFile(path: string): boolean | Promise<boolean>
|
||||
readonly acceptingReads?: boolean
|
||||
prepare?(): Promise<void>
|
||||
flush?(): Promise<void>
|
||||
|
||||
@@ -276,11 +276,13 @@ export class SessionSearchStore {
|
||||
}
|
||||
|
||||
/** Only proven deletion permits removal; unreadable sources retain their indexed rows. */
|
||||
removeFile(path: string): void {
|
||||
removeFile(path: string): boolean {
|
||||
try {
|
||||
this.writer.removeFile(path)
|
||||
return true
|
||||
} catch (error) {
|
||||
this.onError(error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { SessionInputFrontier } from '../ai-vault/session-input-frontier'
|
||||
import type { SessionFileCandidate } from '../ai-vault/session-scanner-types'
|
||||
import { fileIdentity } from './session-search-file-cursor'
|
||||
import { sessionSearchReadDecision } from './session-search-read-decision'
|
||||
@@ -12,8 +13,8 @@ export type SessionSearchWork = {
|
||||
|
||||
/** A unique path owns its debt even when it leaves the next recent list. */
|
||||
export class SessionSearchWorkQueue {
|
||||
private readonly backlog = new Map<string, SessionSearchWork>()
|
||||
private readonly changed = new Map<string, SessionSearchWork>()
|
||||
private readonly backlog = new SessionInputFrontier<SessionSearchWork>()
|
||||
private readonly changed = new SessionInputFrontier<SessionSearchWork>()
|
||||
private version = 0
|
||||
private chooseChanged = true
|
||||
private due = new Set<string>()
|
||||
@@ -64,10 +65,7 @@ export class SessionSearchWorkQueue {
|
||||
next(): SessionSearchWork | undefined {
|
||||
const preferred = this.chooseChanged ? this.changed : this.backlog
|
||||
this.chooseChanged = !this.chooseChanged
|
||||
return (
|
||||
preferred.values().next().value ??
|
||||
(preferred === this.changed ? this.backlog : this.changed).values().next().value
|
||||
)
|
||||
return preferred.first ?? (preferred === this.changed ? this.backlog : this.changed).first
|
||||
}
|
||||
|
||||
acknowledge(work: SessionSearchWork, retry: boolean): void {
|
||||
|
||||
@@ -230,7 +230,7 @@ async function execute({ id, command: c }: SearchWorkerRequest): Promise<SearchW
|
||||
path = c.path
|
||||
break
|
||||
case 'remove':
|
||||
store.removeFile(c.path)
|
||||
accepted = store.removeFile(c.path)
|
||||
path = c.path
|
||||
break
|
||||
case 'purge': {
|
||||
|
||||
@@ -146,9 +146,24 @@ export class SessionSearchWorkerStoreBase extends SessionSearchCursorSnapshot {
|
||||
void this.send({ op: 'state', path, state, mtime })
|
||||
}
|
||||
}
|
||||
removeFile(path: string): void {
|
||||
if (this.acceptingReads) {
|
||||
void this.send({ op: 'remove', path })
|
||||
async removeFile(path: string): Promise<boolean> {
|
||||
if (!this.acceptingReads) {
|
||||
return false
|
||||
}
|
||||
const epoch = this.client.epoch,
|
||||
snapshotEpoch = this.snapshotEpoch
|
||||
try {
|
||||
const response = await this.send({ op: 'remove', path })
|
||||
return (
|
||||
this.acceptingReads &&
|
||||
epoch === this.client.epoch &&
|
||||
snapshotEpoch === this.snapshotEpoch &&
|
||||
response.accepted === true &&
|
||||
!response.failure &&
|
||||
!this.committed.has(path)
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
async purgeOlderThan(cutoff: number | null, signal?: AbortSignal): Promise<void> {
|
||||
|
||||
@@ -43,6 +43,7 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
|
||||
const owner = new AbortController()
|
||||
this.invalidations.set(owner, candidate.file.path)
|
||||
const readSignal = signal ? AbortSignal.any([owner.signal, signal]) : owner.signal
|
||||
let settlement: Promise<boolean> = Promise.resolve(false)
|
||||
let ended = false
|
||||
let cancelled = false
|
||||
let failed = false
|
||||
@@ -124,6 +125,7 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
|
||||
.catch(() => undefined)
|
||||
}
|
||||
return {
|
||||
settled: () => settlement,
|
||||
failed: () => failed,
|
||||
cancelled: () => !active(),
|
||||
drain: async () => {
|
||||
@@ -186,14 +188,19 @@ export class SessionSearchWorkerStore extends SessionSearchWorkerStoreBase {
|
||||
contentHash: hash
|
||||
}
|
||||
this.speculative.set(candidate.file.path, speculative)
|
||||
void post({ op: 'finish', handle, outcome, hash })
|
||||
.then(() => {
|
||||
settlement = post({ op: 'finish', handle, outcome, hash })
|
||||
.then((response) => {
|
||||
const accepted =
|
||||
response.accepted === true &&
|
||||
valid() &&
|
||||
this.committed.get(candidate.file.path)?.state === 'current'
|
||||
if (this.speculative.get(candidate.file.path) === speculative) {
|
||||
this.speculative.delete(candidate.file.path)
|
||||
}
|
||||
this.invalidations.delete(owner)
|
||||
return accepted
|
||||
})
|
||||
.catch(() => undefined)
|
||||
.catch(() => false)
|
||||
return true
|
||||
},
|
||||
discard: () => {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { currentSessionDiscovery, discoverSessionContainer } from '../../session-discovery-context'
|
||||
import {
|
||||
currentSessionDiscovery,
|
||||
discoverSessionContainer,
|
||||
discoverSessionContainerPaths
|
||||
} from '../../session-discovery-context'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import type { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error'
|
||||
@@ -28,16 +32,12 @@ export function opencodeDiscoveries(
|
||||
): Promise<SessionFileDiscovery>[] {
|
||||
return opencodeSources(options, wslHomeDirs).flatMap((source) => {
|
||||
const { storageDir } = source
|
||||
const enumerationIssues: AiVaultScanIssue[] = []
|
||||
const paths = source.dbPaths(enumerationIssues).then((dbPaths) => {
|
||||
issues.push(...enumerationIssues)
|
||||
return dbPaths
|
||||
})
|
||||
const paths = discoverSessionContainerPaths(storageDir, source.dbPaths, issues)
|
||||
return [
|
||||
paths.then((dbPaths) =>
|
||||
paths.then(({ paths: dbPaths, complete }) =>
|
||||
discoverOpenCodeSessions({
|
||||
storageDir,
|
||||
enumerationComplete: !enumerationIssues.some((issue) => issue.kind !== 'notice'),
|
||||
enumerationComplete: complete,
|
||||
dbPaths: dbPaths.filter((path) => !isOpenCodeV2DatabaseName(basename(path))),
|
||||
limitPerAgent: limit,
|
||||
issues,
|
||||
@@ -46,15 +46,8 @@ export function opencodeDiscoveries(
|
||||
})
|
||||
),
|
||||
// Current releases share opencode.db with v1; the worker checks for v2 tables.
|
||||
paths.then((dbPaths) =>
|
||||
discoverOpenCode2Sessions(
|
||||
storageDir,
|
||||
dbPaths,
|
||||
limit,
|
||||
issues,
|
||||
options,
|
||||
!enumerationIssues.some((issue) => issue.kind !== 'notice')
|
||||
)
|
||||
paths.then(({ paths: dbPaths, complete }) =>
|
||||
discoverOpenCode2Sessions(storageDir, dbPaths, limit, issues, options, complete)
|
||||
)
|
||||
]
|
||||
})
|
||||
|
||||
@@ -14,6 +14,11 @@ export type SessionDiscoveryDependency = {
|
||||
}
|
||||
|
||||
export type SessionDiscoveryContext = {
|
||||
containerPaths(
|
||||
root: string,
|
||||
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
|
||||
issues: AiVaultScanIssue[]
|
||||
): Promise<{ paths: readonly string[]; complete: boolean }>
|
||||
files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery>
|
||||
container(
|
||||
key: string,
|
||||
@@ -49,3 +54,15 @@ export function discoverSessionContainer(
|
||||
read(paths, limit)
|
||||
)
|
||||
}
|
||||
|
||||
export async function discoverSessionContainerPaths(
|
||||
root: string,
|
||||
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
|
||||
issues: AiVaultScanIssue[]
|
||||
): Promise<{ paths: readonly string[]; complete: boolean }> {
|
||||
const context = currentSessionDiscovery()
|
||||
if (context) {
|
||||
return context.containerPaths(root, read, issues)
|
||||
}
|
||||
return { paths: await read(issues), complete: true }
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
type SessionInventoryDirectory as Directory
|
||||
} from './session-file-inventory-directories'
|
||||
import type { Dirent } from 'node:fs'
|
||||
import { SessionInputFrontier } from './session-input-frontier'
|
||||
import { SessionObservationRetries } from './session-observation-retries'
|
||||
import { dirname, join, relative } from 'node:path'
|
||||
import { wslGatedReaddir, wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
@@ -35,8 +36,8 @@ export class SessionFileInventory {
|
||||
return this.observations.dependencies
|
||||
}
|
||||
private readonly directories: SessionInventoryDirectories
|
||||
private readonly dirty = new Map<string, boolean>()
|
||||
private readonly walks = new Map<string, Walk>()
|
||||
private readonly dirty = new SessionInputFrontier<boolean>()
|
||||
private readonly walks = new SessionInputFrontier<Walk>()
|
||||
private readonly trailing = new Set<string>()
|
||||
private readonly failures = new SessionObservationRetries()
|
||||
private reconcileAgain = false
|
||||
@@ -133,15 +134,15 @@ export class SessionFileInventory {
|
||||
): Promise<{ changes: SessionFileChange[]; removed: string[]; removedDirectories: string[] }> {
|
||||
let consumed = 0
|
||||
while ((this.dirty.size || this.walks.size) && (consumed === 0 || !overdue())) {
|
||||
consumed++
|
||||
if (this.dirty.size && (!this.walks.size || this.chooseDirty)) {
|
||||
this.chooseDirty = false
|
||||
await this.drainDirty(args, recentLimit)
|
||||
consumed++
|
||||
continue
|
||||
}
|
||||
this.chooseDirty = true
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
const walk = this.walks.values().next().value!
|
||||
const walk = this.walks.first!
|
||||
if (!this.failures.mayAttempt(walk.path)) {
|
||||
this.walks.delete(walk.path)
|
||||
continue
|
||||
@@ -201,7 +202,6 @@ export class SessionFileInventory {
|
||||
}
|
||||
walk.offset++
|
||||
}
|
||||
consumed++
|
||||
if (walk.offset === walk.entries.length) {
|
||||
this.walks.delete(walk.path)
|
||||
if (this.trailing.delete(walk.path)) {
|
||||
@@ -226,7 +226,7 @@ export class SessionFileInventory {
|
||||
}
|
||||
|
||||
private async drainDirty(args: DiscoverSessionFilesArgs, recentLimit: number): Promise<void> {
|
||||
const first = this.dirty.entries().next().value
|
||||
const first = this.dirty.firstEntry()
|
||||
if (!first) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
type Link<T> = { key: string; value: T; previous?: Link<T>; next?: Link<T> }
|
||||
|
||||
/** Keyed obligations retain their FIFO position until acknowledged or cancelled. */
|
||||
export class SessionInputFrontier<T> {
|
||||
private readonly members = new Map<string, Link<T>>()
|
||||
private head: Link<T> | undefined
|
||||
private tail: Link<T> | undefined
|
||||
|
||||
get size(): number {
|
||||
return this.members.size
|
||||
}
|
||||
get first(): T | undefined {
|
||||
return this.head?.value
|
||||
}
|
||||
firstEntry(): readonly [string, T] | undefined {
|
||||
return this.head ? [this.head.key, this.head.value] : undefined
|
||||
}
|
||||
has(key: string): boolean {
|
||||
return this.members.has(key)
|
||||
}
|
||||
get(key: string): T | undefined {
|
||||
return this.members.get(key)?.value
|
||||
}
|
||||
set(key: string, value: T): void {
|
||||
const current = this.members.get(key)
|
||||
if (current) {
|
||||
current.value = value
|
||||
return
|
||||
}
|
||||
const link: Link<T> = { key, value, previous: this.tail }
|
||||
if (this.tail) {
|
||||
this.tail.next = link
|
||||
} else {
|
||||
this.head = link
|
||||
}
|
||||
this.tail = link
|
||||
this.members.set(key, link)
|
||||
}
|
||||
delete(key: string): void {
|
||||
const link = this.members.get(key)
|
||||
if (!link) {
|
||||
return
|
||||
}
|
||||
if (link.previous) {
|
||||
link.previous.next = link.next
|
||||
} else {
|
||||
this.head = link.next
|
||||
}
|
||||
if (link.next) {
|
||||
link.next.previous = link.previous
|
||||
} else {
|
||||
this.tail = link.previous
|
||||
}
|
||||
this.members.delete(key)
|
||||
link.previous = undefined
|
||||
link.next = undefined
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user