mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
Retain discovery recovery and physical source obligations
Resume accepted observation frontiers fairly, scope dependency invalidation to changed inputs, and preserve raw per-database membership across bounded recent observations and scheduled complete observations. Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
co-authored by
Jinjing
parent
8a6d1827ed
commit
fb5dea11b0
@@ -11,55 +11,99 @@ import type {
|
||||
SessionFileDiscovery
|
||||
} from '../ai-vault/session-scanner-types'
|
||||
|
||||
/** Keep suppressed aliases so deleting a preferred path can promote the surviving source. */
|
||||
type Row = { candidate: SessionFileCandidate; root: string }
|
||||
type Group = Row | Map<string, Row>
|
||||
function rows(group: Group | undefined): Row[] {
|
||||
return group ? (group instanceof Map ? [...group.values()] : [group]) : []
|
||||
}
|
||||
function rowId(row: Row): string {
|
||||
return `${row.candidate.agent}:${row.candidate.file.path}`
|
||||
}
|
||||
function groupKey(candidate: SessionFileCandidate, root: string): string {
|
||||
const path = candidate.file.path
|
||||
return candidate.agent === 'codex'
|
||||
? `codex:${sessionFileExecutionNamespace(path)}:${basename(path)}`
|
||||
: candidate.agent === 'antigravity'
|
||||
? `antigravity:${dirname(path)}`
|
||||
: candidate.agent === 'opencode' ||
|
||||
candidate.agent === 'opencode2' ||
|
||||
candidate.agent === 'zcode'
|
||||
? `${candidate.agent}:${root}:${splitOpenCodeSqliteCandidate(path, candidate.agent)?.sessionId ?? basename(path, extname(path))}`
|
||||
: path
|
||||
}
|
||||
|
||||
/** Ordinary paths own one record; only competing aliases allocate a group and selected set. */
|
||||
export class SessionSearchCandidateInventory {
|
||||
private readonly groups = new Map<
|
||||
string,
|
||||
Map<string, { candidate: SessionFileCandidate; root: string }>
|
||||
>()
|
||||
private readonly byPath = new Map<string, Set<string>>()
|
||||
private readonly groups = new Map<string, Group>()
|
||||
private readonly byPath = new Map<string, string | Set<string>>()
|
||||
private readonly selected = new Map<string, Set<string>>()
|
||||
|
||||
has(path: string): boolean {
|
||||
return this.byPath.has(path)
|
||||
has(path: string, agent?: string | null): boolean {
|
||||
const owner = this.byPath.get(path)
|
||||
if (!agent) {
|
||||
return owner !== undefined
|
||||
}
|
||||
return (typeof owner === 'string' ? [owner] : [...(owner ?? [])]).some((key) =>
|
||||
rows(this.groups.get(key)).some(
|
||||
(row) => row.candidate.file.path === path && row.candidate.agent === agent
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
async update(
|
||||
discoveries: readonly SessionFileDiscovery[],
|
||||
removed: ReadonlySet<string>,
|
||||
options: AiVaultScanOptions,
|
||||
invalidated: ReadonlySet<string> = new Set()
|
||||
invalidated: ReadonlySet<string> = new Set(),
|
||||
removedCandidates: ReadonlyMap<string, ReadonlySet<string>> = new Map()
|
||||
): Promise<{ candidates: SessionFileCandidate[]; suppressed: Set<string>; forced: Set<string> }> {
|
||||
const changed = new Set<string>()
|
||||
const previous = new Map<string, Set<string>>()
|
||||
const forced = new Set<string>()
|
||||
for (const path of removed) {
|
||||
for (const key of this.byPath.get(path) ?? []) {
|
||||
const group = this.groups.get(key)
|
||||
if (group) {
|
||||
for (const [id, row] of group) {
|
||||
if (row.candidate.file.path === path) {
|
||||
group.delete(id)
|
||||
}
|
||||
}
|
||||
const touch = (key: string) => {
|
||||
if (previous.has(key)) {
|
||||
return
|
||||
}
|
||||
const group = this.groups.get(key)
|
||||
previous.set(
|
||||
key,
|
||||
this.selected.get(key) ??
|
||||
new Set(group && !(group instanceof Map) ? [group.candidate.file.path] : [])
|
||||
)
|
||||
}
|
||||
for (const path of new Set([...removed, ...removedCandidates.keys()])) {
|
||||
const owner = this.byPath.get(path)
|
||||
for (const key of typeof owner === 'string' ? [owner] : (owner ?? [])) {
|
||||
touch(key)
|
||||
const remaining = rows(this.groups.get(key)).filter(
|
||||
(row) =>
|
||||
row.candidate.file.path !== path ||
|
||||
(!removed.has(path) && !removedCandidates.get(path)?.has(row.candidate.agent))
|
||||
)
|
||||
if (remaining.length === 0) {
|
||||
this.groups.delete(key)
|
||||
} else if (remaining.length === 1) {
|
||||
this.groups.set(key, remaining[0])
|
||||
} else {
|
||||
this.groups.set(key, new Map(remaining.map((row) => [rowId(row), row])))
|
||||
}
|
||||
changed.add(key)
|
||||
}
|
||||
const retained = new Set(
|
||||
(typeof owner === 'string' ? [owner] : [...(owner ?? [])]).filter((key) =>
|
||||
rows(this.groups.get(key)).some((row) => row.candidate.file.path === path)
|
||||
)
|
||||
)
|
||||
if (retained.size === 0) {
|
||||
this.byPath.delete(path)
|
||||
} else {
|
||||
this.byPath.set(path, retained.size === 1 ? retained.values().next().value! : retained)
|
||||
}
|
||||
this.byPath.delete(path)
|
||||
}
|
||||
for (const discovery of discoveries) {
|
||||
for (const candidate of discoveryFileCandidates(discovery, discovery.files, options)) {
|
||||
const path = candidate.file.path
|
||||
const key =
|
||||
candidate.agent === 'codex'
|
||||
? `codex:${sessionFileExecutionNamespace(path)}:${basename(path)}`
|
||||
: candidate.agent === 'antigravity'
|
||||
? `antigravity:${dirname(path)}`
|
||||
: candidate.agent === 'opencode'
|
||||
? `opencode:${sessionFileExecutionNamespace(path)}:${splitOpenCodeSqliteCandidate(path)?.sessionId ?? basename(path, extname(path))}`
|
||||
: path
|
||||
const group = this.groups.get(key) ?? new Map()
|
||||
const path = candidate.file.path,
|
||||
key = groupKey(candidate, discovery.rootDir)
|
||||
touch(key)
|
||||
const group = this.groups.get(key)
|
||||
if (candidate.agent === 'codex' && candidate.file.nlink && candidate.file.nlink > 1) {
|
||||
for (const row of group.values()) {
|
||||
for (const row of rows(group)) {
|
||||
const file = row.candidate.file
|
||||
if (file.dev === candidate.file.dev && file.ino === candidate.file.ino) {
|
||||
row.candidate = { ...row.candidate, file: { ...candidate.file, path: file.path } }
|
||||
@@ -69,27 +113,53 @@ export class SessionSearchCandidateInventory {
|
||||
}
|
||||
}
|
||||
}
|
||||
group.set(`${candidate.agent}:${path}`, { candidate, root: discovery.rootDir })
|
||||
this.groups.set(key, group)
|
||||
const keys = this.byPath.get(path) ?? new Set()
|
||||
keys.add(key)
|
||||
this.byPath.set(path, keys)
|
||||
changed.add(key)
|
||||
const row = { candidate, root: discovery.rootDir }
|
||||
if (!group) {
|
||||
this.groups.set(key, row)
|
||||
} else if (group instanceof Map) {
|
||||
group.set(rowId(row), row)
|
||||
} else if (rowId(group) === rowId(row)) {
|
||||
this.groups.set(key, row)
|
||||
} else {
|
||||
this.groups.set(
|
||||
key,
|
||||
new Map([
|
||||
[rowId(group), group],
|
||||
[rowId(row), row]
|
||||
])
|
||||
)
|
||||
}
|
||||
const owner = this.byPath.get(path)
|
||||
if (!owner) {
|
||||
this.byPath.set(path, key)
|
||||
} else if (typeof owner === 'string') {
|
||||
if (owner !== key) {
|
||||
this.byPath.set(path, new Set([owner, key]))
|
||||
}
|
||||
} else {
|
||||
owner.add(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
const candidates: SessionFileCandidate[] = []
|
||||
const suppressed = new Set<string>()
|
||||
for (const key of changed) {
|
||||
const group = this.groups.get(key)!
|
||||
const rows = [...group.values()]
|
||||
const eligible = rows.some(
|
||||
(row) =>
|
||||
row.candidate.agent === 'opencode' &&
|
||||
splitOpenCodeSqliteCandidate(row.candidate.file.path)
|
||||
const candidates: SessionFileCandidate[] = [],
|
||||
suppressed = new Set<string>()
|
||||
for (const [key, oldPaths] of previous) {
|
||||
const group = this.groups.get(key)
|
||||
const members = rows(group)
|
||||
const eligible = members.some((row) =>
|
||||
splitOpenCodeSqliteCandidate(
|
||||
row.candidate.file.path,
|
||||
row.candidate.agent === 'zcode' ? 'zcode' : 'opencode'
|
||||
)
|
||||
)
|
||||
? rows.filter((row) => splitOpenCodeSqliteCandidate(row.candidate.file.path))
|
||||
: rows
|
||||
const selected = await sessionCandidatesFromDiscoveries(
|
||||
? members.filter((row) =>
|
||||
splitOpenCodeSqliteCandidate(
|
||||
row.candidate.file.path,
|
||||
row.candidate.agent === 'zcode' ? 'zcode' : 'opencode'
|
||||
)
|
||||
)
|
||||
: members
|
||||
const ranked = await sessionCandidatesFromDiscoveries(
|
||||
eligible.map(({ candidate, root }) => ({
|
||||
agent: candidate.agent,
|
||||
rootDir: root,
|
||||
@@ -97,23 +167,42 @@ export class SessionSearchCandidateInventory {
|
||||
})),
|
||||
options
|
||||
)
|
||||
// Each database source originally chose greatest mtime, with source order breaking ties.
|
||||
const selected = eligible.some((row) =>
|
||||
splitOpenCodeSqliteCandidate(
|
||||
row.candidate.file.path,
|
||||
row.candidate.agent === 'zcode' ? 'zcode' : 'opencode'
|
||||
)
|
||||
)
|
||||
? ranked.slice(0, 1)
|
||||
: ranked
|
||||
const paths = new Set(selected.map((candidate) => candidate.file.path))
|
||||
for (const path of this.selected.get(key) ?? []) {
|
||||
for (const path of oldPaths) {
|
||||
if (!paths.has(path)) {
|
||||
suppressed.add(path)
|
||||
}
|
||||
}
|
||||
this.selected.set(key, paths)
|
||||
candidates.push(...selected)
|
||||
if (group.size === 0) {
|
||||
this.groups.delete(key)
|
||||
if (group instanceof Map) {
|
||||
this.selected.set(key, paths)
|
||||
} else {
|
||||
this.selected.delete(key)
|
||||
}
|
||||
candidates.push(...selected)
|
||||
}
|
||||
return {
|
||||
candidates: candidates.sort((a, b) => b.file.mtimeMs - a.file.mtimeMs),
|
||||
suppressed,
|
||||
forced
|
||||
for (const path of suppressed) {
|
||||
const owner = this.byPath.get(path)
|
||||
if (
|
||||
(typeof owner === 'string' ? [owner] : [...(owner ?? [])]).some((key) => {
|
||||
const group = this.groups.get(key)
|
||||
return group instanceof Map
|
||||
? this.selected.get(key)?.has(path)
|
||||
: group?.candidate.file.path === path
|
||||
})
|
||||
) {
|
||||
suppressed.delete(path)
|
||||
}
|
||||
}
|
||||
candidates.sort((left, right) => right.file.mtimeMs - left.file.mtimeMs)
|
||||
return { candidates, suppressed, forced }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
import { dirname } from 'node:path'
|
||||
import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
|
||||
import type { FileWithMtime, SessionFileDiscovery } from '../ai-vault/session-scanner-types'
|
||||
import { isMissingTranscriptPathError } from '../ai-vault/session-scanner-discovery'
|
||||
import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
import { splitSyntheticSessionSource } from './session-search-synthetic-sources'
|
||||
|
||||
type Physical = {
|
||||
files: Map<string, FileWithMtime>
|
||||
signature?: string
|
||||
identity?: string
|
||||
fullDue: boolean
|
||||
}
|
||||
type Family = { discovery?: SessionFileDiscovery; physical: Map<string, Physical> }
|
||||
|
||||
/** Recent observations add facts; only a successful exhaustive observation can withdraw them. */
|
||||
export class SessionSearchContainerInventory {
|
||||
private readonly families = new Map<string, Family>()
|
||||
private seen = new Set<string>()
|
||||
private recentLimit = 12
|
||||
readonly removed = new Map<string, Set<string>>()
|
||||
readonly proofs = new Map<string, ReadonlySet<string>>()
|
||||
|
||||
begin(full: boolean, recentLimit: number): void {
|
||||
this.seen = new Set()
|
||||
this.recentLimit = recentLimit
|
||||
this.removed.clear()
|
||||
if (full) {
|
||||
for (const family of this.families.values()) {
|
||||
for (const physical of family.physical.values()) {
|
||||
physical.fullDue = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
get pending(): boolean {
|
||||
for (const family of this.families.values()) {
|
||||
for (const physical of family.physical.values()) {
|
||||
if (physical.fullDue) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
async discover(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
issues: readonly AiVaultScanIssue[],
|
||||
signal?: AbortSignal
|
||||
): Promise<SessionFileDiscovery> {
|
||||
this.seen.add(key)
|
||||
let family = this.families.get(key)
|
||||
if (!family) {
|
||||
family = { physical: new Map() }
|
||||
this.families.set(key, family)
|
||||
}
|
||||
const changes: FileWithMtime[] = []
|
||||
for (const path of paths) {
|
||||
let physical = family.physical.get(path)
|
||||
if (!physical) {
|
||||
physical = { files: new Map(), fullDue: true }
|
||||
family.physical.set(path, physical)
|
||||
}
|
||||
const before = await this.signature(path, signal)
|
||||
const proofKey = `${key.split(':', 1)[0]}:${path}`
|
||||
if (!before.known || physical.signature !== before.signature) {
|
||||
this.proofs.delete(proofKey)
|
||||
}
|
||||
if (physical.identity !== before.identity && before.known) {
|
||||
physical.fullDue = true
|
||||
}
|
||||
if (before.identity === '' && before.known && family.discovery) {
|
||||
this.release(key, path, family, physical)
|
||||
continue
|
||||
}
|
||||
const full = physical.fullDue
|
||||
const discovery = await read([path], full ? Infinity : this.recentLimit)
|
||||
family.discovery = discovery
|
||||
throwIfAiVaultScanCancelled(signal)
|
||||
const current = new Map<string, FileWithMtime>()
|
||||
for (const file of discovery.files) {
|
||||
current.set(file.path, file)
|
||||
const source = splitSyntheticSessionSource(file.path)
|
||||
const proof = this.proofs.get(proofKey)
|
||||
if (source && proof && !proof.has(source.id)) {
|
||||
this.proofs.delete(proofKey)
|
||||
}
|
||||
const previous = physical.files.get(file.path)
|
||||
if (
|
||||
!previous ||
|
||||
previous.mtimeMs !== file.mtimeMs ||
|
||||
previous.sizeBytes !== file.sizeBytes
|
||||
) {
|
||||
changes.push(file)
|
||||
}
|
||||
physical.files.set(file.path, file)
|
||||
}
|
||||
const after = await this.signature(path, signal)
|
||||
const complete = discovery.complete !== false && before.known && after.known
|
||||
const stable = complete && before.signature === after.signature
|
||||
if (full && complete) {
|
||||
physical.fullDue = !stable
|
||||
// Empty also means an unknown schema under the existing reader contract.
|
||||
if (stable && current.size) {
|
||||
for (const old of physical.files.keys()) {
|
||||
if (!current.has(old)) {
|
||||
this.remove(old, discovery.agent)
|
||||
physical.files.delete(old)
|
||||
}
|
||||
}
|
||||
const ids = new Set<string>()
|
||||
for (const file of current.values()) {
|
||||
const source = splitSyntheticSessionSource(file.path)
|
||||
if (source) {
|
||||
ids.add(source.id)
|
||||
}
|
||||
}
|
||||
this.proofs.set(proofKey, ids)
|
||||
}
|
||||
}
|
||||
if (!stable) {
|
||||
this.proofs.delete(proofKey)
|
||||
}
|
||||
physical.signature = after.signature
|
||||
physical.identity = after.known ? after.identity : physical.identity
|
||||
}
|
||||
// A refused family listing cannot certify that a physical source disappeared.
|
||||
const root = key.slice(key.indexOf(':') + 1)
|
||||
if (
|
||||
!issues.some(
|
||||
(issue) =>
|
||||
issue.kind !== 'notice' &&
|
||||
(issue.path === root || issue.path === dirname(root) || family.physical.has(issue.path))
|
||||
)
|
||||
) {
|
||||
for (const [path, physical] of family.physical) {
|
||||
if (!paths.includes(path)) {
|
||||
this.release(key, path, family, physical)
|
||||
}
|
||||
}
|
||||
}
|
||||
family.discovery ??= await read([], this.recentLimit)
|
||||
return { ...family.discovery, files: changes }
|
||||
}
|
||||
end(): void {
|
||||
for (const [key, family] of this.families) {
|
||||
if (!this.seen.has(key)) {
|
||||
for (const [path, physical] of family.physical) {
|
||||
this.release(key, path, family, physical)
|
||||
}
|
||||
this.families.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
clear(): void {
|
||||
this.families.clear()
|
||||
this.proofs.clear()
|
||||
this.removed.clear()
|
||||
}
|
||||
private remove(path: string, agent: string): void {
|
||||
const agents = this.removed.get(path) ?? new Set<string>()
|
||||
agents.add(agent)
|
||||
this.removed.set(path, agents)
|
||||
}
|
||||
private release(key: string, path: string, family: Family, physical: Physical): void {
|
||||
for (const old of physical.files.keys()) {
|
||||
if (family.discovery) {
|
||||
this.remove(old, family.discovery.agent)
|
||||
}
|
||||
}
|
||||
this.proofs.delete(`${key.split(':', 1)[0]}:${path}`)
|
||||
family.physical.delete(path)
|
||||
}
|
||||
private async signature(path: string, signal?: AbortSignal) {
|
||||
let known = true
|
||||
let identity = ''
|
||||
const facts = await Promise.all(
|
||||
[path, `${path}-wal`].map(async (input) => {
|
||||
try {
|
||||
const stat = await wslGatedStat(input, 'scan', signal)
|
||||
const id = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}`
|
||||
if (input === path) {
|
||||
identity = id
|
||||
}
|
||||
return `${id}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}`
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(signal)
|
||||
if (isMissingTranscriptPathError(error)) {
|
||||
return 'missing'
|
||||
}
|
||||
known = false
|
||||
return 'unknown'
|
||||
}
|
||||
})
|
||||
)
|
||||
return { known, identity, signature: facts.join('|') }
|
||||
}
|
||||
}
|
||||
@@ -79,7 +79,12 @@ export async function retireDeletedSessionSearchSources(
|
||||
signal
|
||||
})
|
||||
const proof = synthetic
|
||||
? proveSyntheticSource(synthetic, containerProof, args.enumeratedContainers)
|
||||
? proveSyntheticSource(
|
||||
synthetic,
|
||||
containerProof,
|
||||
args.enumeratedContainers,
|
||||
store.fileRow(path)?.agent
|
||||
)
|
||||
: containerProof
|
||||
if (proof.verdict === 'gone') {
|
||||
store.removeFile(path)
|
||||
@@ -146,12 +151,14 @@ async function proveSource(
|
||||
function proveSyntheticSource(
|
||||
synthetic: { container: string; id: string },
|
||||
containerProof: SessionSearchSourceVerdict,
|
||||
enumerated?: ReadonlyMap<string, ReadonlySet<string>>
|
||||
enumerated?: ReadonlyMap<string, ReadonlySet<string>>,
|
||||
agent?: string | null
|
||||
): SessionSearchSourceVerdict {
|
||||
if (containerProof.verdict !== 'present') {
|
||||
return containerProof
|
||||
}
|
||||
const ids = enumerated?.get(synthetic.container)
|
||||
const ids =
|
||||
enumerated?.get(`${agent}:${synthetic.container}`) ?? enumerated?.get(synthetic.container)
|
||||
// Unknown schemas can enumerate empty; require nonempty evidence before retiring container entries.
|
||||
if (!ids || ids.size === 0) {
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
|
||||
import { recordSessionScanIssue, scanIssueTextFromError } from '../ai-vault/session-scan-issues'
|
||||
import { SessionFileInventory } from '../ai-vault/session-file-inventory'
|
||||
import type { DiscoverSessionFilesArgs } from '../ai-vault/session-scanner-discovery'
|
||||
import type { SessionDiscoveryDependency } from '../ai-vault/session-discovery-context'
|
||||
import type { SessionSearchRootWatch } from './session-search-root-watchers'
|
||||
|
||||
/** Losing a subscription invalidates the input observation, not every transcript using that input. */
|
||||
export class SessionSearchDependencyInventory {
|
||||
private readonly inventory: SessionFileInventory
|
||||
private readonly observe: ((force: boolean) => Promise<ReadonlySet<string>>) | undefined
|
||||
private refused = false
|
||||
private readonly keys = new Map<string, string | Set<string>>()
|
||||
constructor(
|
||||
readonly dependency: SessionDiscoveryDependency,
|
||||
public polling = false
|
||||
) {
|
||||
this.inventory = new SessionFileInventory(dependency.path)
|
||||
this.observe = dependency.observe?.()
|
||||
}
|
||||
get pending(): boolean {
|
||||
return this.observe ? this.refused : this.inventory.pending
|
||||
}
|
||||
owners(key: string): Iterable<string> {
|
||||
const owners = this.keys.get(key)
|
||||
return typeof owners === 'string' ? [owners] : (owners ?? [])
|
||||
}
|
||||
addFile(path: string): void {
|
||||
const key = this.dependency.fileKey?.(path)
|
||||
if (!key) {
|
||||
return
|
||||
}
|
||||
const paths = this.keys.get(key)
|
||||
if (!paths) {
|
||||
this.keys.set(key, path)
|
||||
} else if (typeof paths === 'string') {
|
||||
if (paths !== path) {
|
||||
this.keys.set(key, new Set([paths, path]))
|
||||
}
|
||||
} else {
|
||||
paths.add(path)
|
||||
}
|
||||
}
|
||||
removeFile(path: string): void {
|
||||
const key = this.dependency.fileKey?.(path)
|
||||
if (!key) {
|
||||
return
|
||||
}
|
||||
const paths = this.keys.get(key)
|
||||
if (typeof paths === 'string') {
|
||||
if (paths === path) {
|
||||
this.keys.delete(key)
|
||||
}
|
||||
} else if (paths) {
|
||||
paths.delete(path)
|
||||
if (!paths.size) {
|
||||
this.keys.delete(key)
|
||||
} else if (paths.size === 1) {
|
||||
this.keys.set(key, paths.values().next().value!)
|
||||
}
|
||||
}
|
||||
}
|
||||
async changedKeys(
|
||||
watch: SessionSearchRootWatch | undefined,
|
||||
args: DiscoverSessionFilesArgs,
|
||||
overdue: () => boolean
|
||||
): Promise<Set<string>> {
|
||||
if (this.observe) {
|
||||
try {
|
||||
const delivered = [...(watch?.events.values() ?? [])].some(
|
||||
(event) => !!this.dependency.eventKey?.(event.path)
|
||||
)
|
||||
const keys = new Set(await this.observe(delivered))
|
||||
this.refused = false
|
||||
return keys
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
recordSessionScanIssue(args.issues, {
|
||||
agent: args.agent,
|
||||
path: this.dependency.path,
|
||||
...scanIssueTextFromError(error)
|
||||
})
|
||||
this.refused = true
|
||||
return new Set()
|
||||
}
|
||||
}
|
||||
const keys = new Set<string>()
|
||||
if ((!watch || watch.failed || watch.reconcile) && !this.inventory.walking) {
|
||||
this.inventory.reconcile()
|
||||
}
|
||||
for (const event of watch?.events.values() ?? []) {
|
||||
this.inventory.invalidate(event.path)
|
||||
}
|
||||
this.inventory.retryFailures()
|
||||
const delta = await this.inventory.drain(
|
||||
{
|
||||
...args,
|
||||
rootDir: this.dependency.path,
|
||||
extensions: ['*'],
|
||||
filePredicate: (path) => !!this.dependency.eventKey?.(path),
|
||||
directoryPredicate: this.dependency.mode === 'shallow' ? () => false : undefined,
|
||||
contentDependencyPath: undefined,
|
||||
searchDependencies: undefined,
|
||||
onDirectoryEntries: undefined
|
||||
},
|
||||
overdue,
|
||||
0
|
||||
)
|
||||
for (const path of [...delta.changes.map((change) => change.file.path), ...delta.removed]) {
|
||||
const key = this.dependency.eventKey?.(path)
|
||||
if (key) {
|
||||
keys.add(key)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import './session-search-worker.test-fixture'
|
||||
import {
|
||||
notifySessionSearchFileChange,
|
||||
notifySessionSearchOverflow
|
||||
} from './session-search-watcher.test-fixture'
|
||||
import { appendFile, mkdir, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import * as filesystem from '../native-chat/wsl-transcript-fs-access'
|
||||
import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache'
|
||||
import {
|
||||
codexFixture,
|
||||
CODEX_FIXTURE_SESSION_ID
|
||||
} from '../ai-vault/scanners/codex/session-scanner-codex-fixtures'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import {
|
||||
FakeSessionSearchClock,
|
||||
openSessionSearchIndexerHarness
|
||||
} from './session-search-indexer-test-fixture'
|
||||
|
||||
it.each([40, 400])(
|
||||
'bounds one title append independently of %s index records and transcript owners',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-title-input-work')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const root = harness.roots.codexSessionsDir!
|
||||
const index = join(dirname(root), 'session_index.jsonl')
|
||||
const fixture = codexFixture()
|
||||
const ids = Array.from(
|
||||
{ length: population },
|
||||
(_, id) => `019f0000-1111-7222-8333-${String(id).padStart(12, '0')}`
|
||||
)
|
||||
const records = ids.map((id) => `${JSON.stringify({ id, thread_name: 'Original title' })}\n`)
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
const streams = vi.spyOn(filesystem, 'openTranscriptReadStream')
|
||||
const slices = vi.spyOn(filesystem, 'readTranscriptSlice')
|
||||
const reads = vi.spyOn(parser, 'parseAgentSessionFileCached')
|
||||
try {
|
||||
await mkdir(root, { recursive: true })
|
||||
for (const id of ids) {
|
||||
await writeFile(
|
||||
join(root, fixture.fileName.replace(CODEX_FIXTURE_SESSION_ID, id)),
|
||||
`${fixture.seedLines.join('\n').replaceAll(CODEX_FIXTURE_SESSION_ID, id)}\n`
|
||||
)
|
||||
}
|
||||
await writeFile(index, records.join(''))
|
||||
await subject.start()
|
||||
streams.mockClear()
|
||||
slices.mockClear()
|
||||
reads.mockClear()
|
||||
// Lost delivery must re-observe the input, not reread it or force every transcript.
|
||||
notifySessionSearchOverflow(dirname(root))
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(streams.mock.calls.filter(([path]) => path === index)).toHaveLength(0)
|
||||
expect(reads).not.toHaveBeenCalled()
|
||||
const append = `${JSON.stringify({ id: ids[0], thread_name: 'Changed title' })}\n`
|
||||
const size = Buffer.byteLength(records.join('') + append)
|
||||
await appendFile(index, append)
|
||||
notifySessionSearchFileChange(index)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
const indexBytes =
|
||||
streams.mock.calls.reduce(
|
||||
(bytes, [path, options]) =>
|
||||
path !== index
|
||||
? bytes
|
||||
: bytes + Math.max(0, (options?.end ?? size - 1) - (options?.start ?? 0) + 1),
|
||||
0
|
||||
) +
|
||||
slices.mock.calls.reduce(
|
||||
(bytes, [path, , length]) => (path === index ? bytes + length : bytes),
|
||||
0
|
||||
)
|
||||
// Two independent consumers need the appended record plus one fixed continuity record each.
|
||||
expect(indexBytes).toBeLessThanOrEqual(
|
||||
2 * (Buffer.byteLength(append) + Buffer.byteLength(records[0]) + 1)
|
||||
)
|
||||
expect(reads).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db.prepare("SELECT count(*) AS n FROM sessions WHERE title='Changed title'").get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db.prepare("SELECT count(*) AS n FROM sessions WHERE title='Original title'").get()
|
||||
)
|
||||
).toEqual({ n: population - 1 })
|
||||
} finally {
|
||||
streams.mockRestore()
|
||||
slices.mockRestore()
|
||||
reads.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -1,62 +1,73 @@
|
||||
import { SessionSearchDependencyInventory } from './session-search-dependency-inventory'
|
||||
import { SessionSearchRootWatchers } from './session-search-root-watchers'
|
||||
import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
import { SessionSearchContainerInventory } from './session-search-container-inventory'
|
||||
import { SessionFileInventory } from '../ai-vault/session-file-inventory'
|
||||
import {
|
||||
discoverSessionFiles,
|
||||
isMissingTranscriptPathError,
|
||||
type DiscoverSessionFilesArgs
|
||||
} from '../ai-vault/session-scanner-discovery'
|
||||
import type { DiscoverSessionFilesArgs } from '../ai-vault/session-scanner-discovery'
|
||||
import type { SessionFileDiscovery } from '../ai-vault/session-scanner-types'
|
||||
import type { SessionDiscoveryContext } from '../ai-vault/session-discovery-context'
|
||||
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
|
||||
import { splitSyntheticSessionSource } from './session-search-synthetic-sources'
|
||||
import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types'
|
||||
|
||||
type Root = {
|
||||
inventory: SessionFileInventory
|
||||
args: DiscoverSessionFilesArgs
|
||||
identity?: string
|
||||
fallback: boolean
|
||||
fallbackCount?: number
|
||||
dependencyKeys: Map<string, Map<string, Set<string>>>
|
||||
dependencySnapshots: Map<string, ReadonlyMap<string, string>>
|
||||
dependencies: Map<string, SessionSearchDependencyInventory>
|
||||
}
|
||||
type Container = { signature: string; discovery: SessionFileDiscovery; membership: Set<string> }
|
||||
|
||||
/** Search owns its notifications and raw membership; ordinary History discovery stays stateless. */
|
||||
export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext {
|
||||
private readonly roots = new Map<string, Root>()
|
||||
private readonly artifactMaps = new Map<string, Map<string, string>>()
|
||||
private readonly containers = new Map<string, Container>()
|
||||
readonly containers = new SessionSearchContainerInventory()
|
||||
private readonly owner = new AbortController()
|
||||
private readonly watchers: SessionSearchRootWatchers
|
||||
private force = false
|
||||
private maintenance = false
|
||||
private overdue = () => false
|
||||
private recentLimit = 12
|
||||
private seen = new Set<string>()
|
||||
readonly forced = new Set<string>()
|
||||
readonly metadata = new Set<string>()
|
||||
readonly removed = new Set<string>()
|
||||
readonly enumeratedContainers = new Map<string, ReadonlySet<string>>()
|
||||
readonly removedCandidates = this.containers.removed
|
||||
readonly enumeratedContainers = this.containers.proofs
|
||||
|
||||
constructor(entryPath?: string) {
|
||||
this.watchers = new SessionSearchRootWatchers(entryPath)
|
||||
}
|
||||
|
||||
begin(options: { full: boolean; overdue: () => boolean; recentLimit: number }): void {
|
||||
begin(options: {
|
||||
full: boolean
|
||||
maintenance?: boolean
|
||||
overdue: () => boolean
|
||||
recentLimit: number
|
||||
}): void {
|
||||
this.force = options.full
|
||||
this.maintenance = options.maintenance === true
|
||||
this.overdue = options.overdue
|
||||
this.recentLimit = options.recentLimit
|
||||
this.watchers.begin()
|
||||
this.seen = new Set()
|
||||
this.containers.begin(options.full || this.maintenance, this.recentLimit)
|
||||
this.forced.clear()
|
||||
this.metadata.clear()
|
||||
this.removed.clear()
|
||||
this.enumeratedContainers.clear()
|
||||
}
|
||||
|
||||
get walking(): boolean {
|
||||
return [...this.roots.values()].some((root) => root.inventory.walking)
|
||||
}
|
||||
|
||||
get pending(): boolean {
|
||||
if (this.containers.pending) {
|
||||
return true
|
||||
}
|
||||
for (const root of this.roots.values()) {
|
||||
if (!root.fallback && root.inventory.pending) {
|
||||
if (
|
||||
root.inventory.pending ||
|
||||
[...root.dependencies.values()].some((dependency) => dependency.pending)
|
||||
) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -66,7 +77,7 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
listings(): { root: string; files: number }[] {
|
||||
return [...this.roots.values()].map((root) => ({
|
||||
root: root.inventory.root,
|
||||
files: root.fallback ? (root.fallbackCount ?? 0) : root.inventory.files.size
|
||||
files: root.inventory.files.size
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -79,10 +90,6 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
return map
|
||||
}
|
||||
|
||||
membership(key: string): ReadonlySet<string> | undefined {
|
||||
return this.containers.get(key)?.membership
|
||||
}
|
||||
|
||||
async files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery> {
|
||||
const key = `${args.agent}:${args.rootDir}`
|
||||
this.seen.add(key)
|
||||
@@ -93,92 +100,69 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
inventory: new SessionFileInventory(args.rootDir),
|
||||
args,
|
||||
fallback,
|
||||
dependencyKeys: new Map(),
|
||||
dependencySnapshots: new Map()
|
||||
dependencies: new Map()
|
||||
}
|
||||
this.roots.set(key, root)
|
||||
}
|
||||
root.args = args
|
||||
if (root.fallback) {
|
||||
const discovery = await discoverSessionFiles(args)
|
||||
root.fallbackCount = discovery.files.length
|
||||
return discovery
|
||||
if (this.maintenance) {
|
||||
root.fallback = await this.watchers.requiresPolling(args.rootDir)
|
||||
}
|
||||
const watch = await this.watchers.watch(args.rootDir, args.signal)
|
||||
if (!watch) {
|
||||
// Missing roots remain obligations; never replace their retained membership with an empty delta.
|
||||
root.inventory.reconcile()
|
||||
} else {
|
||||
if (this.force || watch.reconcile || root.identity !== watch.identity) {
|
||||
if (root.fallback) {
|
||||
if (!root.inventory.walking) {
|
||||
root.inventory.reconcile()
|
||||
}
|
||||
if (root.identity !== watch.identity && (await this.watchers.requiresPolling(args.rootDir))) {
|
||||
root.fallback = true
|
||||
const discovery = await discoverSessionFiles(args)
|
||||
root.fallbackCount = discovery.files.length
|
||||
return discovery
|
||||
}
|
||||
root.identity = watch.identity
|
||||
for (const event of watch.events.values()) {
|
||||
root.inventory.invalidate(event.path)
|
||||
} else {
|
||||
const watch = await this.watchers.watch(args.rootDir, args.signal)
|
||||
if (!watch) {
|
||||
if (!root.inventory.walking) {
|
||||
root.inventory.reconcile()
|
||||
}
|
||||
} else {
|
||||
if (
|
||||
(this.force || watch.reconcile || root.identity !== watch.identity) &&
|
||||
(root.identity !== undefined || !root.inventory.walking)
|
||||
) {
|
||||
root.inventory.reconcile()
|
||||
}
|
||||
if (
|
||||
root.identity !== watch.identity &&
|
||||
(await this.watchers.requiresPolling(args.rootDir))
|
||||
) {
|
||||
root.fallback = true
|
||||
}
|
||||
if (watch.failed && !root.inventory.walking) {
|
||||
root.inventory.reconcile()
|
||||
}
|
||||
root.identity = watch.identity
|
||||
for (const event of watch.events.values()) {
|
||||
root.inventory.invalidate(event.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
const auxiliaries = await Promise.all(
|
||||
(args.searchDependencies ?? []).map(async (dependency) => ({
|
||||
dependency,
|
||||
watch: await this.watchers.watch(dependency.path, args.signal)
|
||||
}))
|
||||
(args.searchDependencies ?? []).map(async (dependency) => {
|
||||
let observer = root.dependencies.get(dependency.path)
|
||||
if (!observer) {
|
||||
const polling = root.fallback || (await this.watchers.requiresPolling(dependency.path))
|
||||
observer = new SessionSearchDependencyInventory(dependency, polling)
|
||||
root.dependencies.set(dependency.path, observer)
|
||||
} else if (this.maintenance) {
|
||||
observer.polling = root.fallback || (await this.watchers.requiresPolling(dependency.path))
|
||||
}
|
||||
return {
|
||||
dependency,
|
||||
observer,
|
||||
watch: observer.polling
|
||||
? undefined
|
||||
: await this.watchers.watch(dependency.path, args.signal, dependency.mode)
|
||||
}
|
||||
})
|
||||
)
|
||||
for (const { dependency, watch: auxiliary } of auxiliaries) {
|
||||
let keys = root.dependencyKeys.get(dependency.path)
|
||||
if (!keys) {
|
||||
keys = new Map()
|
||||
root.dependencyKeys.set(dependency.path, keys)
|
||||
}
|
||||
if (!auxiliary) {
|
||||
continue
|
||||
}
|
||||
if (auxiliary.reconcile || auxiliary.failed) {
|
||||
for (const file of root.inventory.files.values()) {
|
||||
root.inventory.invalidate(file.path)
|
||||
if (dependency.metadata) {
|
||||
this.metadata.add(file.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
const eventKeys = new Set<string>()
|
||||
for (const event of auxiliary.events.values()) {
|
||||
for (const path of root.inventory.dependencies.get(event.path) ?? []) {
|
||||
root.inventory.invalidate(path)
|
||||
if (dependency.metadata) {
|
||||
this.metadata.add(path)
|
||||
}
|
||||
}
|
||||
const key = dependency.eventKey?.(event.path)
|
||||
if (key) {
|
||||
eventKeys.add(key)
|
||||
}
|
||||
}
|
||||
if (dependency.snapshot && (auxiliary.reconcile || eventKeys.size)) {
|
||||
const snapshot = await dependency.snapshot()
|
||||
const previous = root.dependencySnapshots.get(dependency.path)
|
||||
root.dependencySnapshots.set(dependency.path, snapshot)
|
||||
eventKeys.clear()
|
||||
for (const [key, value] of snapshot) {
|
||||
if (previous?.get(key) !== value) {
|
||||
eventKeys.add(key)
|
||||
}
|
||||
}
|
||||
for (const key of previous?.keys() ?? []) {
|
||||
if (!snapshot.has(key)) {
|
||||
eventKeys.add(key)
|
||||
}
|
||||
}
|
||||
// Nonstandard filenames cannot identify their thread until parsed.
|
||||
eventKeys.add('*')
|
||||
}
|
||||
for (const { dependency, observer, watch: auxiliary } of auxiliaries) {
|
||||
const eventKeys = await observer.changedKeys(auxiliary, args, this.overdue)
|
||||
for (const key of eventKeys) {
|
||||
for (const path of keys.get(key) ?? []) {
|
||||
for (const path of observer.owners(key)) {
|
||||
root.inventory.invalidate(path)
|
||||
if (dependency.metadata) {
|
||||
this.metadata.add(path)
|
||||
@@ -193,28 +177,16 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
if (change.force) {
|
||||
this.forced.add(change.file.path)
|
||||
}
|
||||
for (const { dependency } of auxiliaries) {
|
||||
const key = dependency.fileKey?.(change.file.path)
|
||||
if (!key) {
|
||||
continue
|
||||
}
|
||||
const keys = root.dependencyKeys.get(dependency.path)!
|
||||
const paths = keys.get(key) ?? new Set<string>()
|
||||
paths.add(change.file.path)
|
||||
keys.set(key, paths)
|
||||
for (const observer of root.dependencies.values()) {
|
||||
observer.addFile(change.file.path)
|
||||
}
|
||||
}
|
||||
for (const path of delta.removedDirectories) {
|
||||
this.artifactMaps.get(args.rootDir)?.delete(path)
|
||||
}
|
||||
for (const path of delta.removed) {
|
||||
for (const dependency of args.searchDependencies ?? []) {
|
||||
const key = dependency.fileKey?.(path)
|
||||
if (key) {
|
||||
const keys = root.dependencyKeys.get(dependency.path)
|
||||
const paths = keys?.get(key)
|
||||
paths?.delete(path)
|
||||
if (!paths?.size) {
|
||||
keys?.delete(key)
|
||||
}
|
||||
}
|
||||
for (const observer of root.dependencies.values()) {
|
||||
observer.removeFile(path)
|
||||
}
|
||||
this.removed.add(path)
|
||||
}
|
||||
@@ -225,65 +197,26 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
}
|
||||
}
|
||||
|
||||
async container(
|
||||
container(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: () => Promise<SessionFileDiscovery>
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
_limit: number,
|
||||
issues: readonly AiVaultScanIssue[]
|
||||
): Promise<SessionFileDiscovery> {
|
||||
const signature = (
|
||||
await Promise.all(
|
||||
paths
|
||||
.flatMap((path) => [path, `${path}-wal`])
|
||||
.map(async (path) => {
|
||||
try {
|
||||
const stat = await wslGatedStat(path, 'scan', this.owner.signal)
|
||||
return `${path}:${stat.dev}:${stat.ino}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}`
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(this.owner.signal)
|
||||
return `${path}:${isMissingTranscriptPathError(error) ? 'missing' : 'unknown'}`
|
||||
}
|
||||
})
|
||||
)
|
||||
).join('|')
|
||||
const old = this.containers.get(key)
|
||||
if (!this.force && old?.signature === signature && !signature.includes(':unknown')) {
|
||||
return { agent: old.discovery.agent, rootDir: old.discovery.rootDir, files: [] }
|
||||
}
|
||||
const discovery = await read()
|
||||
const membership = new Set<string>()
|
||||
const idsByContainer = new Map<string, Set<string>>()
|
||||
for (const file of discovery.files) {
|
||||
const synthetic = splitSyntheticSessionSource(file.path)
|
||||
if (!synthetic) {
|
||||
continue
|
||||
}
|
||||
membership.add(synthetic.id)
|
||||
const ids = idsByContainer.get(synthetic.container) ?? new Set<string>()
|
||||
ids.add(synthetic.id)
|
||||
idsByContainer.set(synthetic.container, ids)
|
||||
}
|
||||
if (discovery.complete === false) {
|
||||
return discovery
|
||||
}
|
||||
for (const [path, ids] of idsByContainer) {
|
||||
this.enumeratedContainers.set(path, ids)
|
||||
}
|
||||
if (old) {
|
||||
const current = new Set(discovery.files.map((file) => file.path))
|
||||
for (const file of old.discovery.files) {
|
||||
if (!current.has(file.path)) {
|
||||
this.removed.add(file.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
this.containers.set(key, { signature, discovery, membership })
|
||||
return discovery
|
||||
return this.containers.discover(key, paths, read, issues, this.owner.signal)
|
||||
}
|
||||
|
||||
async end(): Promise<void> {
|
||||
await this.watchers.end()
|
||||
this.containers.end()
|
||||
for (const key of this.roots.keys()) {
|
||||
if (!this.seen.has(key)) {
|
||||
const root = this.roots.get(key)!
|
||||
for (const path of root.inventory.files.keys()) {
|
||||
this.removed.add(path)
|
||||
}
|
||||
this.artifactMaps.delete(root.inventory.root)
|
||||
this.roots.delete(key)
|
||||
}
|
||||
}
|
||||
@@ -293,6 +226,7 @@ export class SessionSearchDiscoveryInventory implements SessionDiscoveryContext
|
||||
this.owner.abort()
|
||||
this.roots.clear()
|
||||
this.containers.clear()
|
||||
this.artifactMaps.clear()
|
||||
return this.watchers.close()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
import './session-search-worker.test-fixture'
|
||||
import {
|
||||
notifySessionSearchFileChange,
|
||||
notifySessionSearchOverflow
|
||||
} from './session-search-watcher.test-fixture'
|
||||
import { appendFile, mkdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import * as filesystem from '../native-chat/wsl-transcript-fs-access'
|
||||
import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache'
|
||||
import * as policy from './session-search-filesystem'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import {
|
||||
FakeSessionSearchClock,
|
||||
openSessionSearchIndexerHarness,
|
||||
writeClaudeTranscript,
|
||||
claudeLines
|
||||
} from './session-search-indexer-test-fixture'
|
||||
|
||||
it.each([100, 1000])(
|
||||
'reads appended bytes after a delivered update to a recent %s-turn transcript',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-recent-append')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const path = join(harness.claudeProjectDir, 'active.jsonl')
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
const reads = vi.spyOn(parser, 'parseAgentSessionFileCached')
|
||||
try {
|
||||
await writeClaudeTranscript(
|
||||
path,
|
||||
Array.from({ length: population }, () => 'archivedbody'),
|
||||
'active'
|
||||
)
|
||||
await subject.start()
|
||||
for (let cycle = 0; cycle < 3; cycle++) {
|
||||
reads.mockClear()
|
||||
const append = `${claudeLines(['appendedneedle'], 'active', population * 2 + cycle * 2).join('\n')}\n`
|
||||
await appendFile(path, append)
|
||||
notifySessionSearchFileChange(path)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
const call = reads.mock.calls.find(([candidate]) => candidate.file.path === path)
|
||||
expect(call).toBeDefined()
|
||||
// The parser stream needs only appended records; its fixed continuity probe does not grow with history.
|
||||
expect(call?.[2]?.bytesRead).toBeLessThanOrEqual(Buffer.byteLength(append) + 65536)
|
||||
expect(call?.[2]?.fullParses).toBe(0)
|
||||
}
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'appendedneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 6 })
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('retries a refused first observation after settlement without another filesystem event', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-stat-retry')
|
||||
const clock = new FakeSessionSearchClock(),
|
||||
path = join(harness.claudeProjectDir, 'recovered.jsonl')
|
||||
const actual = filesystem.wslGatedStat
|
||||
let refused = false
|
||||
const probe = vi.spyOn(filesystem, 'wslGatedStat').mockImplementation((observed, ...rest) => {
|
||||
if (observed === path && !refused) {
|
||||
refused = true
|
||||
return Promise.reject(
|
||||
Object.assign(new Error('fixture temporary stat refusal'), { code: 'EIO' })
|
||||
)
|
||||
}
|
||||
return actual(observed, ...rest)
|
||||
})
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
try {
|
||||
await writeClaudeTranscript(path, ['recoveredneedle'], 'recovered')
|
||||
await subject.start()
|
||||
expect(refused).toBe(true)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'recoveredneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 2 })
|
||||
} finally {
|
||||
probe.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it('advances the discovery suffix during continuous updates and repeated overflow', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-crawl-fairness')
|
||||
const clock = new FakeSessionSearchClock(),
|
||||
root = harness.roots.claudeProjectsDir!
|
||||
const active = join(harness.claudeProjectDir, 'active.jsonl')
|
||||
const listings = vi.spyOn(filesystem, 'wslGatedReaddir')
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock,
|
||||
discoveryDeadlineMs: 0
|
||||
})
|
||||
try {
|
||||
await writeClaudeTranscript(active, ['activebody'], 'active')
|
||||
for (let index = 0; index < 16; index++) {
|
||||
await writeClaudeTranscript(
|
||||
join(root, `bucket${index}`, 'history.jsonl'),
|
||||
['frontierneedle'],
|
||||
`history${index}`
|
||||
)
|
||||
}
|
||||
await subject.start()
|
||||
listings.mockClear()
|
||||
for (let cycle = 0; cycle < 64; cycle++) {
|
||||
await appendFile(
|
||||
active,
|
||||
`${claudeLines(['activeappend'], 'active', cycle * 2 + 10).join('\n')}\n`
|
||||
)
|
||||
notifySessionSearchFileChange(active)
|
||||
notifySessionSearchOverflow(root)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
}
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(DISTINCT session_row_id) AS n FROM messages JOIN messages_fts ON messages.id=messages_fts.rowid WHERE messages_fts MATCH 'frontierneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 16 })
|
||||
// An epoch covers sixteen child directories; sixty-four overflow signals cannot restart its root prefix.
|
||||
expect(listings.mock.calls.filter(([path]) => path === root).length).toBeLessThanOrEqual(4)
|
||||
} finally {
|
||||
listings.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it.each([40, 400])(
|
||||
'polls %s retained files without rereading unchanged transcripts and retires a deletion',
|
||||
async (population) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-polling-delta')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const path = join(harness.claudeProjectDir, 'changed.jsonl')
|
||||
const deleted = join(harness.claudeProjectDir, 'deleted.jsonl')
|
||||
const polling = vi.spyOn(policy, 'sessionSearchRequiresPolling').mockResolvedValue(true)
|
||||
const reads = vi.spyOn(parser, 'parseAgentSessionFileCached')
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
try {
|
||||
await writeClaudeTranscript(path, ['originalbody'], 'changed')
|
||||
await writeClaudeTranscript(deleted, ['deletedneedle'], 'deleted')
|
||||
for (let index = 0; index < population; index++) {
|
||||
await writeClaudeTranscript(
|
||||
join(harness.claudeProjectDir, `${index}.jsonl`),
|
||||
['archivebody'],
|
||||
`archive${index}`
|
||||
)
|
||||
}
|
||||
await subject.start()
|
||||
reads.mockClear()
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
// Polling must observe membership, but unchanged observations create no transcript read work.
|
||||
expect(reads).not.toHaveBeenCalled()
|
||||
await appendFile(path, `${claudeLines(['pollchangedneedle'], 'changed', 10).join('\n')}\n`)
|
||||
await rm(deleted)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(reads).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'pollchangedneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 2 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'deletedneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 0 })
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
polling.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('observes remote message-only edits through polling without a native notification', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-polling-content')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const storage = harness.roots.opencodeStorageDir!
|
||||
const primary = join(storage, 'session', 'project', 'ses_remote.json')
|
||||
const messages = join(storage, 'message', 'ses_remote')
|
||||
const polling = vi.spyOn(policy, 'sessionSearchRequiresPolling').mockResolvedValue(true)
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
try {
|
||||
await mkdir(join(storage, 'session', 'project'), { recursive: true })
|
||||
await mkdir(messages, { recursive: true })
|
||||
await writeFile(primary, JSON.stringify({ id: 'ses_remote', directory: '/repo/remote' }))
|
||||
await writeFile(
|
||||
join(messages, 'msg_1.json'),
|
||||
JSON.stringify({
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: 'originalremote' }],
|
||||
time: { created: 1777634000000 }
|
||||
})
|
||||
)
|
||||
await subject.start()
|
||||
await writeFile(
|
||||
join(messages, 'msg_2.json'),
|
||||
JSON.stringify({
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'remoteinputneedle' }],
|
||||
time: { created: 1777634001000 }
|
||||
})
|
||||
)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'remoteinputneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'originalremote'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
} finally {
|
||||
polling.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
@@ -2,20 +2,22 @@ import type SyncDatabase from '../sqlite/sync-database'
|
||||
import type { SessionSearchFileRow } from './session-search-store-types'
|
||||
|
||||
export function readSessionSearchFileRows(db: SyncDatabase, path?: string): SessionSearchFileRow[] {
|
||||
return (
|
||||
db
|
||||
.prepare(
|
||||
// Numeric stat IDs may exceed SQLite's safe INTEGER-to-number read range.
|
||||
`SELECT f.path, f.byte_offset AS byteOffset, f.prefix_hash AS prefixHash, s.agent, CAST(f.dev AS REAL) AS dev, CAST(f.ino AS REAL) AS ino,
|
||||
const statement = db.prepare(
|
||||
// Numeric stat IDs may exceed SQLite's safe INTEGER-to-number read range.
|
||||
`SELECT f.path, f.byte_offset AS byteOffset, f.prefix_hash AS prefixHash, s.agent, CAST(f.dev AS REAL) AS dev, CAST(f.ino AS REAL) AS ino,
|
||||
mtime_ms AS mtimeMs, size_bytes AS sizeBytes,
|
||||
state, fail_count AS failCount, failed_mtime_ms AS failedMtimeMs
|
||||
FROM files f LEFT JOIN sessions s ON s.id=f.session_row_id${path === undefined ? '' : ' WHERE f.path = ?'}`
|
||||
)
|
||||
.all(...(path === undefined ? [] : [path])) as (Omit<SessionSearchFileRow, 'identity'> & {
|
||||
dev: number | null
|
||||
ino: number | null
|
||||
})[]
|
||||
).map((row) => ({
|
||||
)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this query explicitly selects the file-row fields and nullable numeric identity columns.
|
||||
const rows = statement.all(...(path === undefined ? [] : [path])) as (Omit<
|
||||
SessionSearchFileRow,
|
||||
'identity'
|
||||
> & {
|
||||
dev: number | null
|
||||
ino: number | null
|
||||
})[]
|
||||
return rows.map((row) => ({
|
||||
path: row.path,
|
||||
byteOffset: row.byteOffset,
|
||||
prefixHash: row.prefixHash,
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { runProcess } from '@orca/process-host'
|
||||
import { statfs } from 'node:fs/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import { isWslUncPath } from '../../shared/wsl-paths'
|
||||
|
||||
const run = promisify(execFile)
|
||||
let mounts: { at: number; result: Promise<string[]> } | undefined
|
||||
let mounts: { at: number; result: Promise<string[] | undefined> } | undefined
|
||||
|
||||
/** Native notifications cannot establish freshness when another host writes a shared mount. */
|
||||
export async function sessionSearchRequiresPolling(path: string): Promise<boolean> {
|
||||
@@ -16,17 +14,20 @@ export async function sessionSearchRequiresPolling(path: string): Promise<boolea
|
||||
if (!mounts || Date.now() - mounts.at > 1_000) {
|
||||
mounts = {
|
||||
at: Date.now(),
|
||||
result: run('/sbin/mount', [], { timeout: 2_000, maxBuffer: 256 * 1024 })
|
||||
.then(({ stdout }) =>
|
||||
stdout.split('\n').flatMap((line) => {
|
||||
const match = / on (.+) \((?:nfs|smbfs|webdav|osxfuse|fusefs)[,)]/.exec(line)
|
||||
return match ? [match[1]] : []
|
||||
})
|
||||
result: runProcess({ program: '/sbin/mount', timeoutMs: 2_000, maxOutputBytes: 256 * 1024 })
|
||||
.then(({ code, outputTruncated, stdout }) =>
|
||||
code !== 0 || outputTruncated
|
||||
? undefined
|
||||
: stdout.split('\n').flatMap((line) => {
|
||||
const match = / on (.+) \((?:nfs|smbfs|webdav|osxfuse|fusefs)[,)]/.exec(line)
|
||||
return match ? [match[1]] : []
|
||||
})
|
||||
)
|
||||
.catch(() => [])
|
||||
.catch(() => undefined)
|
||||
}
|
||||
}
|
||||
return (await mounts.result).some((root) => path === root || path.startsWith(`${root}/`))
|
||||
const shared = await mounts.result
|
||||
return !shared || shared.some((root) => path === root || path.startsWith(`${root}/`))
|
||||
}
|
||||
if (process.platform === 'linux') {
|
||||
try {
|
||||
@@ -34,7 +35,7 @@ export async function sessionSearchRequiresPolling(path: string): Promise<boolea
|
||||
(await statfs(path)).type
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
codexFixture,
|
||||
CODEX_FIXTURE_SESSION_ID
|
||||
} from '../ai-vault/scanners/codex/session-scanner-codex-fixtures'
|
||||
import { subagentTranscriptsDirFor } from '../ai-vault/scanners/claude/session-scanner-subagent-transcripts'
|
||||
import { SessionFileInventory } from '../ai-vault/session-file-inventory'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import {
|
||||
@@ -66,7 +67,11 @@ it.each([40, 400])(
|
||||
stat.mock.calls.filter(([path]) => path === root || path.startsWith(`${root}/`)).length
|
||||
).toBeLessThanOrEqual(16)
|
||||
expect(
|
||||
readdir.mock.calls.filter(([path]) => path === root || path.startsWith(`${root}/`))
|
||||
readdir.mock.calls.filter(
|
||||
([observed]) =>
|
||||
observed !== subagentTranscriptsDirFor(path) &&
|
||||
(observed === root || observed.startsWith(`${root}/`))
|
||||
)
|
||||
).toHaveLength(0)
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
@@ -76,7 +81,7 @@ it.each([40, 400])(
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
).toEqual({ n: 2 })
|
||||
} finally {
|
||||
await subject.close()
|
||||
stat.mockRestore()
|
||||
@@ -157,7 +162,7 @@ it('indexes a delivered equal-stat rewrite outside the recent window', async ()
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
).toEqual({ n: 2 })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
|
||||
@@ -595,7 +595,7 @@ it('retains a deferred recent file after it leaves the top-N window', async () =
|
||||
const paths = ids.map((id) => transcriptPath(id))
|
||||
await writeClaudeTranscript(paths[0], ['deferredreviewneedle'], ids[0])
|
||||
await writeClaudeTranscript(paths[1], ['firstreviewneedle'], ids[1])
|
||||
const time = Date.now()
|
||||
const time = Date.UTC(2099, 0, 1)
|
||||
await utimes(paths[0], new Date(time), new Date(time))
|
||||
await utimes(paths[1], new Date(time + 1000), new Date(time + 1000))
|
||||
await nextCycle()
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import './session-search-worker.test-fixture'
|
||||
import { notifySessionSearchFileChange } from './session-search-watcher.test-fixture'
|
||||
import { mkdir, stat, utimes, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import * as parser from '../ai-vault/parse-cache/session-scanner-parse-cache'
|
||||
import * as databaseReader from '../ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-open'
|
||||
import { resetDevinSessionsIndexCacheForTests } from '../ai-vault/scanners/devin/session-scanner-devin-db'
|
||||
import {
|
||||
writeAntigravityHistory,
|
||||
writeAntigravityTranscript
|
||||
} from '../ai-vault/session-scanner-test-fixtures'
|
||||
import SyncDatabase from '../sqlite/sync-database'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import {
|
||||
FakeSessionSearchClock,
|
||||
openSessionSearchIndexerHarness
|
||||
} from './session-search-indexer-test-fixture'
|
||||
|
||||
it('does not add a database-only Devin search wake and still enriches an admitted transcript read', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-devin-admission')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const root = harness.roots.devinTranscriptsDir!
|
||||
const path = join(root, 'shown.json')
|
||||
const dbPath = join(dirname(root), 'sessions.db')
|
||||
await mkdir(root, { recursive: true })
|
||||
const db = new SyncDatabase(dbPath)
|
||||
db.exec(
|
||||
"CREATE TABLE sessions (id TEXT PRIMARY KEY, title TEXT, hidden INTEGER); INSERT INTO sessions VALUES ('shown', 'Original title', 0)"
|
||||
)
|
||||
const write = (text: string) =>
|
||||
writeFile(
|
||||
path,
|
||||
JSON.stringify({ session_id: 'shown', steps: [{ source: 'user', message: text }] })
|
||||
)
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
const reads = vi.spyOn(parser, 'parseAgentSessionFileCached')
|
||||
const accessor = vi.spyOn(databaseReader, 'readOpenCodeDatabase')
|
||||
try {
|
||||
await write('originaldevinneedle')
|
||||
await subject.start()
|
||||
reads.mockClear()
|
||||
accessor.mockClear()
|
||||
db.exec("UPDATE sessions SET title='Changed title'")
|
||||
const previous = await stat(dbPath)
|
||||
await utimes(dbPath, new Date(previous.mtimeMs + 1000), new Date(previous.mtimeMs + 1000))
|
||||
notifySessionSearchFileChange(dbPath)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(reads).not.toHaveBeenCalled()
|
||||
expect(accessor).not.toHaveBeenCalled()
|
||||
expect(
|
||||
harness.read((db) => db.prepare("SELECT title FROM sessions WHERE session_id='shown'").get())
|
||||
).toEqual({ title: 'Original title' })
|
||||
await write('changeddevinneedle')
|
||||
notifySessionSearchFileChange(path)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(
|
||||
harness.read((db) => db.prepare("SELECT title FROM sessions WHERE session_id='shown'").get())
|
||||
).toEqual({ title: 'Changed title' })
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'changeddevinneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
accessor.mockRestore()
|
||||
db.close()
|
||||
resetDevinSessionsIndexCacheForTests()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not fan out search reads for history metadata outside the Antigravity search parser', async () => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-history-admission')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const root = harness.roots.antigravityBrainDir!
|
||||
const id = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff'
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
const reads = vi.spyOn(parser, 'parseAgentSessionFileCached')
|
||||
try {
|
||||
const path = await writeAntigravityTranscript(root, id, [
|
||||
{
|
||||
source: 'USER_EXPLICIT',
|
||||
type: 'USER_INPUT',
|
||||
created_at: '2026-07-15T11:39:10.000Z',
|
||||
content: '<USER_REQUEST>historyadmissionneedle</USER_REQUEST>'
|
||||
}
|
||||
])
|
||||
await subject.start()
|
||||
reads.mockClear()
|
||||
await writeAntigravityHistory(root, [
|
||||
{ display: 'historyadmissionneedle', timestamp: 1784115550100, workspace: '/repo/changed' }
|
||||
])
|
||||
notifySessionSearchFileChange(join(dirname(root), 'history.jsonl'))
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(reads).not.toHaveBeenCalled()
|
||||
expect(harness.read((db) => db.prepare('SELECT cwd FROM sessions').get())).toEqual({
|
||||
cwd: null
|
||||
})
|
||||
await writeFile(
|
||||
path,
|
||||
`${JSON.stringify({ source: 'USER_EXPLICIT', type: 'USER_INPUT', created_at: '2026-07-15T11:39:10.000Z', content: '<USER_REQUEST>changedhistoryneedle</USER_REQUEST>' })}\n`
|
||||
)
|
||||
notifySessionSearchFileChange(path)
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'changedhistoryneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 1 })
|
||||
} finally {
|
||||
reads.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
})
|
||||
@@ -1,9 +1,12 @@
|
||||
import './session-search-worker.test-fixture'
|
||||
import { rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
|
||||
// Replace worker transport while exercising the real SQLite readers.
|
||||
export const openCodeReadCalls: string[] = []
|
||||
export const openCodeListObservations: { paths: readonly string[]; limit: number; rows: number }[] =
|
||||
[]
|
||||
vi.mock(
|
||||
'../ai-vault/scanners/opencode/reader-transport/session-scanner-opencode-sqlite-worker-spawn',
|
||||
async () => {
|
||||
@@ -16,12 +19,23 @@ vi.mock(
|
||||
const own = await import('./session-search-opencode-index.test')
|
||||
const { listOpenCode2SqliteSessions } =
|
||||
await import('../ai-vault/scanners/opencode2/session-scanner-opencode2-sqlite-list')
|
||||
const parse2 = await import('../ai-vault/scanners/opencode2/session-scanner-opencode2-sqlite')
|
||||
return {
|
||||
parseOpenCode2SqliteSessionViaWorker: parse2.parseOpenCode2SqliteSession,
|
||||
captureOpenCode2SqliteSessionViaWorker: parse2.captureOpenCode2SqliteSession,
|
||||
listOpenCode2SqliteSessionsViaWorker: listOpenCode2SqliteSessions,
|
||||
resolveOpenCodeSqliteWorkerEntryPath: () => null,
|
||||
listOpenCodeSqliteSessionsViaWorker: (
|
||||
args: Parameters<typeof list.listOpenCodeSqliteSessions>[0]
|
||||
) => list.listOpenCodeSqliteSessions(args),
|
||||
) =>
|
||||
list.listOpenCodeSqliteSessions(args).then((candidates) => {
|
||||
own.openCodeListObservations.push({
|
||||
paths: args.dbPaths,
|
||||
limit: args.limit,
|
||||
rows: candidates.length
|
||||
})
|
||||
return candidates
|
||||
}),
|
||||
parseOpenCodeSqliteSessionViaWorker: (
|
||||
args: Parameters<typeof parse.parseOpenCodeSqliteSession>[0]
|
||||
) => {
|
||||
@@ -43,7 +57,7 @@ import {
|
||||
appendOpenCodeSqliteTurn,
|
||||
writeOpenCodeSqliteDatabase
|
||||
} from '../ai-vault/scanners/opencode/sqlite/session-scanner-opencode-sqlite-fixture'
|
||||
import type SyncDatabase from '../sqlite/sync-database'
|
||||
import SyncDatabase from '../sqlite/sync-database'
|
||||
import { SessionSearchEngine } from './session-search-engine'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import { openSessionSearchDatabase } from './session-search-schema'
|
||||
@@ -73,6 +87,7 @@ beforeEach(async () => {
|
||||
indexer = null
|
||||
engineDbs = []
|
||||
openCodeReadCalls.length = 0
|
||||
openCodeListObservations.length = 0
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -176,3 +191,152 @@ it('re-reads a session that gained a message and replaces its rows', async () =>
|
||||
expect(engine.search({ query: ANSWER }).hits).toHaveLength(1)
|
||||
expect(openCodeReadCalls.filter((call) => call === `capture:${SESSION}`)).toHaveLength(2)
|
||||
})
|
||||
|
||||
it.each([40, 400])(
|
||||
'keeps recent database observations bounded at %s rows while old imports reach search on maintenance',
|
||||
async (population) => {
|
||||
writeOpenCodeSqliteDatabase(
|
||||
dbPath(),
|
||||
Array.from({ length: population }, (_, index) => ({
|
||||
id: `ses_${index}`,
|
||||
turns: [{ role: 'user', parts: ['retainedarchive'] }]
|
||||
}))
|
||||
)
|
||||
const subject = await startIndexer()
|
||||
appendOpenCodeSqliteTurn(dbPath(), 'ses_0', {
|
||||
role: 'assistant',
|
||||
parts: ['recentdatabaseneedle']
|
||||
})
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(openEngine().search({ query: 'recentdatabaseneedle' }).hits[0]?.sessionId).toBe('ses_0')
|
||||
expect(openCodeListObservations.at(-1)).toMatchObject({ limit: 12, rows: 12 })
|
||||
writeOpenCodeSqliteDatabase(dbPath(), [
|
||||
{ id: 'ses_imported', turns: [{ role: 'user', parts: ['oldimportneedle'] }] }
|
||||
])
|
||||
const source = new SyncDatabase(dbPath())
|
||||
source
|
||||
.prepare("UPDATE session SET time_created=1000, time_updated=2000 WHERE id='ses_imported'")
|
||||
.run()
|
||||
source.close()
|
||||
for (let cycle = 0; cycle < 14; cycle++) {
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
}
|
||||
expect(
|
||||
openCodeListObservations.filter((observation) => observation.limit === Infinity)
|
||||
).toHaveLength(2)
|
||||
expect(
|
||||
openCodeListObservations
|
||||
.filter((observation) => observation.limit === 12)
|
||||
.every((observation) => observation.rows === 12)
|
||||
).toBe(true)
|
||||
expect(openEngine().search({ query: 'oldimportneedle' }).hits[0]?.sessionId).toBe(
|
||||
'ses_imported'
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps a duplicate physical row and promotes it when the selected database disappears', async () => {
|
||||
const alternative = join(harness.root, 'opencode-db', 'opencode-alternative.db')
|
||||
writeOpenCodeSqliteDatabase(dbPath(), [
|
||||
{ id: 'ses_shared', turns: [{ role: 'user', parts: ['winningcopyneedle'] }] },
|
||||
{ id: 'ses_primary', turns: [{ role: 'user', parts: ['primaryonlyneedle'] }] }
|
||||
])
|
||||
writeOpenCodeSqliteDatabase(alternative, [
|
||||
{ id: 'ses_shared', turns: [{ role: 'user', parts: ['alternativecopyneedle'] }] },
|
||||
{ id: 'ses_alternative', turns: [{ role: 'user', parts: ['alternativeonlyneedle'] }] }
|
||||
])
|
||||
let paths = [dbPath(), alternative]
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: { ...harness.roots, opencodeDbPaths: paths },
|
||||
resolveRoots: async () => ({ ...harness.roots, opencodeDbPaths: paths }),
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
indexer = subject
|
||||
await subject.start()
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: 'winningcopyneedle' }).hits).toHaveLength(1)
|
||||
expect(openEngine().search({ query: 'alternativecopyneedle' }).hits).toHaveLength(0)
|
||||
await rm(dbPath())
|
||||
paths = [alternative]
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: 'alternativecopyneedle' }).hits[0]?.sessionId).toBe(
|
||||
'ses_shared'
|
||||
)
|
||||
expect(openEngine().search({ query: 'winningcopyneedle' }).hits).toHaveLength(0)
|
||||
expect(openEngine().search({ query: 'primaryonlyneedle' }).hits).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('retains refused database membership while healthy owners settle and retries only the refused full observation', async () => {
|
||||
const alternative = join(harness.root, 'opencode-db', 'opencode-alternative.db')
|
||||
writeOpenCodeSqliteDatabase(dbPath(), [
|
||||
{ id: 'ses_primary', turns: [{ role: 'user', parts: ['preservedneedle'] }] }
|
||||
])
|
||||
writeOpenCodeSqliteDatabase(alternative, [
|
||||
{ id: 'ses_alternative', turns: [{ role: 'user', parts: ['healthyneedle'] }] }
|
||||
])
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: { ...harness.roots, opencodeDbPaths: [dbPath(), alternative] },
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
indexer = subject
|
||||
await subject.start()
|
||||
await rename(dbPath(), `${dbPath()}.saved`)
|
||||
await writeFile(dbPath(), 'temporarily unreadable sqlite')
|
||||
appendOpenCodeSqliteTurn(alternative, 'ses_alternative', {
|
||||
role: 'assistant',
|
||||
parts: ['healthyupdateneedle']
|
||||
})
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: 'preservedneedle' }).hits).toHaveLength(1)
|
||||
expect(openEngine().search({ query: 'healthyupdateneedle' }).hits).toHaveLength(1)
|
||||
await rm(dbPath())
|
||||
await rename(`${dbPath()}.saved`, dbPath())
|
||||
openCodeListObservations.length = 0
|
||||
clock.advance(20000)
|
||||
await subject.settled()
|
||||
expect(
|
||||
openCodeListObservations.filter((observation) => observation.paths[0] === alternative)
|
||||
).toEqual([{ paths: [alternative], limit: 12, rows: 1 }])
|
||||
expect(
|
||||
openCodeListObservations.filter((observation) => observation.paths[0] === dbPath())
|
||||
).toEqual([{ paths: [dbPath()], limit: Infinity, rows: 1 }])
|
||||
expect(openEngine().search({ query: 'preservedneedle' }).hits).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('keeps full membership proofs separate when v1 and v2 share one physical database', async () => {
|
||||
writeVault()
|
||||
const subject = await startIndexer()
|
||||
expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1)
|
||||
const db = new SyncDatabase(dbPath())
|
||||
db.exec(`CREATE TABLE session_v2 (id TEXT PRIMARY KEY, time_created INTEGER, time_updated INTEGER);
|
||||
INSERT INTO session_v2 VALUES ('ses_capture', 1740000000000, 1740000010000)`)
|
||||
db.close()
|
||||
const changed = new SyncDatabase(dbPath())
|
||||
changed.prepare('DELETE FROM session WHERE id=?').run(SESSION)
|
||||
changed.close()
|
||||
await subject.reconcile({ full: true })
|
||||
// The v2 copy of this ID is not positive membership evidence for the removed v1 row.
|
||||
expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(0)
|
||||
expect(openEngine().search({ query: OTHER }).hits).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('keeps indexed rows when an existing physical database becomes empty or unrecognized', async () => {
|
||||
writeVault()
|
||||
const subject = await startIndexer()
|
||||
const db = new SyncDatabase(dbPath())
|
||||
db.exec('DELETE FROM session')
|
||||
db.close()
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1)
|
||||
const changed = new SyncDatabase(dbPath())
|
||||
changed.exec('DROP TABLE session')
|
||||
changed.close()
|
||||
await subject.reconcile({ full: true })
|
||||
expect(openEngine().search({ query: ANSWER }).hits).toHaveLength(1)
|
||||
})
|
||||
|
||||
@@ -17,6 +17,7 @@ import { runSessionSearchIndexPass } from './session-search-index-pass'
|
||||
import {
|
||||
discoverSessionSearchCandidates,
|
||||
sessionSearchEmptiedRoots,
|
||||
isUnderScanRoot,
|
||||
type SessionSearchScanRoots
|
||||
} from './session-search-scan-roots'
|
||||
import type { SessionSearchIndexStore, SessionSearchFileRow } from './session-search-store'
|
||||
@@ -30,6 +31,7 @@ export class SessionSearchPassState {
|
||||
readonly queue = new SessionSearchWorkQueue()
|
||||
retirement: Iterator<SessionSearchFileRow> | null = null
|
||||
readonly retire = new Set<string>()
|
||||
readonly parkedRetire = new Set<string>()
|
||||
needsRetirement = true
|
||||
constructor(entryPath?: string) {
|
||||
this.discovery = new SessionSearchDiscoveryInventory(entryPath)
|
||||
@@ -82,31 +84,47 @@ export async function runSessionSearchPass(
|
||||
}
|
||||
state.discovery.begin({
|
||||
full: args.full,
|
||||
maintenance: args.purge ?? args.full,
|
||||
recentLimit: args.recentPerAgent,
|
||||
overdue: args.discoveryOverdue ?? (() => false)
|
||||
})
|
||||
if (args.full) {
|
||||
state.needsRetirement = true
|
||||
}
|
||||
if (args.full || args.purge) {
|
||||
for (const path of state.parkedRetire) {
|
||||
state.retire.add(path)
|
||||
}
|
||||
state.parkedRetire.clear()
|
||||
}
|
||||
return withCursorChatMetaScan(async () => {
|
||||
const swept = await withSessionDiscovery(state.discovery, () =>
|
||||
discoverSessionSearchCandidates(args.roots, { limitPerAgent: Infinity, signal, raw: true })
|
||||
)
|
||||
await state.discovery.end()
|
||||
const issues: AiVaultScanIssue[] = [...swept.issues]
|
||||
const update = await state.candidates.update(
|
||||
swept.discoveries,
|
||||
state.discovery.removed,
|
||||
{ ...args.roots, signal },
|
||||
state.discovery.forced
|
||||
state.discovery.forced,
|
||||
state.discovery.removedCandidates
|
||||
)
|
||||
for (const path of update.suppressed) {
|
||||
state.queue.remove(path)
|
||||
}
|
||||
for (const path of state.discovery.removed) {
|
||||
for (const path of new Set([
|
||||
...state.discovery.removed,
|
||||
...state.discovery.removedCandidates.keys()
|
||||
])) {
|
||||
if (state.candidates.has(path, store.fileRow(path)?.agent)) {
|
||||
continue
|
||||
}
|
||||
state.queue.remove(path)
|
||||
state.retire.add(path)
|
||||
}
|
||||
for (const candidate of update.candidates) {
|
||||
state.parkedRetire.delete(candidate.file.path)
|
||||
state.queue.enqueue(
|
||||
candidate,
|
||||
store,
|
||||
@@ -128,7 +146,7 @@ export async function runSessionSearchPass(
|
||||
? sessionSearchEmptiedRoots(args.previousRootsWithFiles, rootsWithFiles)
|
||||
: new Set<string>()
|
||||
const degraded: SessionSearchDegradedRoot[] = []
|
||||
if (!state.discovery.pending && state.needsRetirement) {
|
||||
if (!state.discovery.walking && state.needsRetirement) {
|
||||
state.retirement ??= store.fileRows()
|
||||
state.needsRetirement = false
|
||||
}
|
||||
@@ -150,7 +168,7 @@ export async function runSessionSearchPass(
|
||||
state.retirement = null
|
||||
break
|
||||
}
|
||||
if (!state.candidates.has(row.value.path)) {
|
||||
if (!state.candidates.has(row.value.path, row.value.agent)) {
|
||||
paths.push(row.value.path)
|
||||
}
|
||||
}
|
||||
@@ -170,9 +188,16 @@ export async function runSessionSearchPass(
|
||||
directoryLimit: 512,
|
||||
signal
|
||||
})
|
||||
for (const path of [...retirement.unverifiable, ...retirement.unchecked]) {
|
||||
for (const path of retirement.unchecked) {
|
||||
retry.add(path)
|
||||
}
|
||||
for (const path of retirement.unverifiable) {
|
||||
if ([...emptiedRoots].some((root) => isUnderScanRoot(path, root))) {
|
||||
retry.add(path)
|
||||
} else {
|
||||
state.parkedRetire.add(path)
|
||||
}
|
||||
}
|
||||
degraded.push(...retirement.degradedRoots)
|
||||
// Retry refused paths next cycle, but keep advancing the older retirement frontier now.
|
||||
batches++
|
||||
@@ -192,7 +217,6 @@ export async function runSessionSearchPass(
|
||||
signal
|
||||
)
|
||||
await store.flush?.()
|
||||
await state.discovery.end()
|
||||
return {
|
||||
rootsWithFiles,
|
||||
degradedRoots: mergeDegradedRoots(
|
||||
|
||||
@@ -10,6 +10,8 @@ export type SessionSearchRootWatch = {
|
||||
failed: boolean
|
||||
events: Map<string, WatcherProcessEvent>
|
||||
reconcile: boolean
|
||||
attempts: number
|
||||
retryAt: number
|
||||
}
|
||||
|
||||
export class SessionSearchRootWatchers {
|
||||
@@ -36,10 +38,15 @@ export class SessionSearchRootWatchers {
|
||||
this.watches.clear()
|
||||
return this.supervisor.disposeAndWait()
|
||||
}
|
||||
watch(path: string, signal?: AbortSignal): Promise<SessionSearchRootWatch | undefined> {
|
||||
let probe = this.probes.get(path)
|
||||
watch(
|
||||
path: string,
|
||||
signal?: AbortSignal,
|
||||
mode: 'recursive' | 'shallow' = 'recursive'
|
||||
): Promise<SessionSearchRootWatch | undefined> {
|
||||
const key = `${mode}:${path}`
|
||||
let probe = this.probes.get(key)
|
||||
if (!probe) {
|
||||
probe = this.probeWatch(path, signal).then((watch) => {
|
||||
probe = this.probeWatch(path, key, mode, signal).then((watch) => {
|
||||
if (!watch) {
|
||||
return undefined
|
||||
}
|
||||
@@ -48,28 +55,38 @@ export class SessionSearchRootWatchers {
|
||||
watch.reconcile = false
|
||||
return delivery
|
||||
})
|
||||
this.probes.set(path, probe)
|
||||
this.probes.set(key, probe)
|
||||
}
|
||||
return probe
|
||||
}
|
||||
|
||||
private async probeWatch(
|
||||
path: string,
|
||||
key: string,
|
||||
mode: 'recursive' | 'shallow',
|
||||
signal?: AbortSignal
|
||||
): Promise<SessionSearchRootWatch | undefined> {
|
||||
const previous = this.watches.get(key)
|
||||
let identity: string
|
||||
try {
|
||||
const stat = await wslGatedStat(path, 'scan', signal)
|
||||
if (!stat.isDirectory()) {
|
||||
this.watches.delete(key)
|
||||
await previous?.subscription?.unsubscribe()
|
||||
return undefined
|
||||
}
|
||||
identity = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}`
|
||||
} catch {
|
||||
throwIfAiVaultScanCancelled(signal)
|
||||
this.watches.delete(key)
|
||||
await previous?.subscription?.unsubscribe().catch(() => undefined)
|
||||
return undefined
|
||||
}
|
||||
const previous = this.watches.get(path)
|
||||
if (previous && !previous.failed && previous.identity === identity) {
|
||||
if (
|
||||
previous &&
|
||||
previous.identity === identity &&
|
||||
(!previous.failed || Date.now() < previous.retryAt)
|
||||
) {
|
||||
return previous
|
||||
}
|
||||
await previous?.subscription?.unsubscribe().catch(() => undefined)
|
||||
@@ -77,10 +94,15 @@ export class SessionSearchRootWatchers {
|
||||
identity,
|
||||
failed: false,
|
||||
events: new Map(),
|
||||
reconcile: true
|
||||
reconcile: true,
|
||||
attempts: previous?.attempts ?? 0,
|
||||
retryAt: 0
|
||||
}
|
||||
this.watches.set(path, watch)
|
||||
this.watches.set(key, watch)
|
||||
const fail = () => {
|
||||
if (!watch.failed) {
|
||||
watch.retryAt = Date.now() + Math.min(300_000, 5_000 * 2 ** Math.min(watch.attempts++, 6))
|
||||
}
|
||||
watch.failed = true
|
||||
watch.reconcile = true
|
||||
}
|
||||
@@ -102,10 +124,12 @@ export class SessionSearchRootWatchers {
|
||||
watch.events.set(event.path, event)
|
||||
}
|
||||
},
|
||||
{ mode: 'recursive' },
|
||||
{ mode },
|
||||
{
|
||||
signal: this.owner.signal,
|
||||
onInterruption: fail,
|
||||
onInterruption: () => {
|
||||
watch.reconcile = true
|
||||
},
|
||||
onOverflow: () => {
|
||||
watch.reconcile = true
|
||||
},
|
||||
@@ -115,9 +139,11 @@ export class SessionSearchRootWatchers {
|
||||
)
|
||||
if (this.owner.signal.aborted) {
|
||||
await watch.subscription.unsubscribe()
|
||||
} else if (!watch.failed) {
|
||||
watch.attempts = 0
|
||||
}
|
||||
} catch {
|
||||
watch.failed = true
|
||||
fail()
|
||||
throwIfAiVaultScanCancelled(this.owner.signal)
|
||||
}
|
||||
return watch
|
||||
|
||||
@@ -1,16 +1,29 @@
|
||||
import { vi } from 'vitest'
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import type { WatcherProcessHooks } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription'
|
||||
import type { WatcherProcessCallback } from '../ipc/filesystem/watcher-process/parcel-watcher-process-subscription'
|
||||
|
||||
const watchers = vi.hoisted(() => new Map<string, WatcherProcessCallback>())
|
||||
const watchers = vi.hoisted(
|
||||
() =>
|
||||
new Map<
|
||||
object,
|
||||
{ path: string; callback: WatcherProcessCallback; hooks: WatcherProcessHooks }
|
||||
>()
|
||||
)
|
||||
vi.mock('../ipc/filesystem/watcher-process/parcel-watcher-process-supervisor', () => ({
|
||||
WatcherProcessSupervisor: class {
|
||||
async subscribe(path: string, callback: WatcherProcessCallback) {
|
||||
watchers.set(path, callback)
|
||||
async subscribe(
|
||||
path: string,
|
||||
callback: WatcherProcessCallback,
|
||||
_options: unknown,
|
||||
hooks: WatcherProcessHooks = {}
|
||||
) {
|
||||
const token = {}
|
||||
watchers.set(token, { path, callback, hooks })
|
||||
return {
|
||||
unsubscribe: async () => {
|
||||
watchers.delete(path)
|
||||
watchers.delete(token)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,7 +34,7 @@ vi.mock('../ipc/filesystem/watcher-process/parcel-watcher-process-supervisor', (
|
||||
}))
|
||||
|
||||
export function notifySessionSearchRootChanges(): void {
|
||||
for (const [path, callback] of watchers) {
|
||||
for (const { path, callback } of watchers.values()) {
|
||||
const directories = [path]
|
||||
for (let index = 0; index < directories.length; index++) {
|
||||
try {
|
||||
@@ -42,9 +55,17 @@ export function notifySessionSearchRootChanges(): void {
|
||||
}
|
||||
|
||||
export function notifySessionSearchFileChange(path: string): void {
|
||||
for (const [root, callback] of watchers) {
|
||||
for (const { path: root, callback } of watchers.values()) {
|
||||
if (path.startsWith(`${root}/`) || path.startsWith(`${root}\\`)) {
|
||||
callback(null, [{ path, type: 'update', isDirectory: false }])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function notifySessionSearchOverflow(path: string): void {
|
||||
for (const watch of watchers.values()) {
|
||||
if (watch.path === path) {
|
||||
watch.hooks.onOverflow?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// @orca-isolated-worker -- Its worker factory mock must be installed in each importing module graph.
|
||||
import type * as WorkerClient from './session-search-worker-client'
|
||||
import { createRequire } from 'node:module'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
@@ -3,7 +3,7 @@ import { completeAntigravityTranscriptPairs } from './antigravity-transcript-can
|
||||
import { recordAntigravityArtifactPath } from './antigravity-session-artifacts'
|
||||
import { ANTIGRAVITY_HISTORY_ROOTS } from '../../../../shared/antigravity-session-origin'
|
||||
import { homedir } from 'node:os'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { join } from 'node:path'
|
||||
import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import {
|
||||
isAntigravityTranscriptPath,
|
||||
@@ -49,15 +49,6 @@ export function antigravityDiscoveries(
|
||||
agent: 'antigravity',
|
||||
issues,
|
||||
extensions: ['.jsonl'],
|
||||
searchDependencies: [
|
||||
{
|
||||
path: dirname(rootDir),
|
||||
metadata: true,
|
||||
fileKey: () => 'history',
|
||||
eventKey: (path) =>
|
||||
['history.jsonl'].includes(basename(path)) || path.endsWith('.meta') ? 'history' : null
|
||||
}
|
||||
],
|
||||
filePredicate: isAntigravityTranscriptPath,
|
||||
directoryPredicate: shouldDescendAntigravityBrainDirectory,
|
||||
onDirectoryEntries: (dirPath, entries) =>
|
||||
|
||||
@@ -1,93 +1,20 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { Stats } from 'node:fs'
|
||||
import {
|
||||
openTranscriptReadStream,
|
||||
readTranscriptSlice
|
||||
} from '../../../native-chat/wsl-transcript-fs-access'
|
||||
import { consumeCompleteJsonlLines } from '../../session-scanner-jsonl-reader'
|
||||
import { readSessionIndexJsonl, type SessionIndexJsonlRead } from '../../session-index-jsonl-read'
|
||||
import { extractString, parseJsonObject } from '../../session-scanner-values'
|
||||
import { normalizeTitleText } from '../../session-scanner-text-normalization'
|
||||
|
||||
export type CodexIndexRead = {
|
||||
titles: Map<string, string>
|
||||
offset: number
|
||||
readThrough: number
|
||||
dev: number
|
||||
ino: number
|
||||
firstRecord: { length: number; hash: string } | null
|
||||
}
|
||||
|
||||
async function firstRecordSignature(path: string, end: number) {
|
||||
const hash = createHash('sha256')
|
||||
let length = 0
|
||||
for await (const chunk of openTranscriptReadStream(path, { end: end - 1 }, 'scan')) {
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
const newline = bytes.indexOf(0x0a)
|
||||
const prefix = newline === -1 ? bytes : bytes.subarray(0, newline + 1)
|
||||
hash.update(prefix)
|
||||
length += prefix.length
|
||||
if (newline !== -1) {
|
||||
return { length, hash: hash.digest('hex') }
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export type CodexIndexRead = SessionIndexJsonlRead & { titles: Map<string, string> }
|
||||
export async function readCodexIndexUpdate(
|
||||
path: string,
|
||||
stat: Stats,
|
||||
previous?: CodexIndexRead
|
||||
): Promise<CodexIndexRead> {
|
||||
let append = false
|
||||
if (
|
||||
previous?.firstRecord &&
|
||||
previous.offset === previous.readThrough &&
|
||||
stat.size > previous.readThrough &&
|
||||
stat.dev !== 0 &&
|
||||
stat.ino !== 0 &&
|
||||
previous.dev !== 0 &&
|
||||
previous.ino !== 0 &&
|
||||
stat.dev === previous.dev &&
|
||||
stat.ino === previous.ino
|
||||
) {
|
||||
try {
|
||||
const prefix = await firstRecordSignature(path, previous.firstRecord.length)
|
||||
const boundary = await readTranscriptSlice(path, previous.offset - 1, 1, 'scan')
|
||||
// Growth alone cannot distinguish an append from Codex replacing the index on removal.
|
||||
append = prefix?.hash === previous.firstRecord.hash && boundary[0] === 0x0a
|
||||
} catch {
|
||||
// A failed continuity check requires a complete reread.
|
||||
}
|
||||
}
|
||||
const titles = append ? new Map(previous!.titles) : new Map<string, string>()
|
||||
const applyLine = (line: string) => {
|
||||
const record = parseJsonObject(line)
|
||||
const sessionId = record && extractString(record.id)
|
||||
const title = record && normalizeTitleText(extractString(record.thread_name) ?? '')
|
||||
if (sessionId && title) {
|
||||
titles.set(sessionId, title)
|
||||
}
|
||||
}
|
||||
const start = append ? previous!.offset : 0
|
||||
const { consumedThrough, trailingPartialLine, bytesRead } = await consumeCompleteJsonlLines({
|
||||
path,
|
||||
start,
|
||||
onLine: applyLine
|
||||
})
|
||||
// EOF JSON remains readable; a tail forces the next growth to reread rather than retain it.
|
||||
if (trailingPartialLine) {
|
||||
applyLine(trailingPartialLine)
|
||||
}
|
||||
return {
|
||||
titles,
|
||||
offset: consumedThrough,
|
||||
readThrough: start + bytesRead,
|
||||
dev: stat.dev,
|
||||
ino: stat.ino,
|
||||
firstRecord: append
|
||||
? previous!.firstRecord
|
||||
: consumedThrough > 0
|
||||
? await firstRecordSignature(path, consumedThrough)
|
||||
: null
|
||||
}
|
||||
const read = await readSessionIndexJsonl(path, stat, previous, parseCodexTitleIndexRecord)
|
||||
return { ...read, titles: read.values }
|
||||
}
|
||||
export function parseCodexTitleIndexRecord(line: string): readonly [string, string] | null {
|
||||
const record = parseJsonObject(line)
|
||||
const id = record && extractString(record.id)
|
||||
const title = record && normalizeTitleText(extractString(record.thread_name) ?? '')
|
||||
return id && title ? [id, title] : null
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createSessionJsonlDependency } from '../../session-jsonl-dependency'
|
||||
import { parseCodexTitleIndexRecord } from './session-scanner-codex-index-read'
|
||||
import type { SessionDiscoveryDependency } from '../../session-discovery-context'
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
@@ -186,10 +188,15 @@ export const codexSearchDependencies: (root: string) => readonly SessionDiscover
|
||||
) => [
|
||||
{
|
||||
path: dirname(root),
|
||||
mode: 'shallow',
|
||||
eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'titles' : null),
|
||||
fileKey: (path) =>
|
||||
/([0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})\.jsonl$/i.exec(path)?.[1] ?? '*',
|
||||
snapshot: () => readCodexSessionIndexTitles(dirname(root)),
|
||||
observe: () =>
|
||||
createSessionJsonlDependency(
|
||||
join(dirname(root), CODEX_SESSION_INDEX_FILE),
|
||||
parseCodexTitleIndexRecord
|
||||
),
|
||||
metadata: true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import type { SessionDiscoveryDependency } from '../../session-discovery-context'
|
||||
import { parseWslUncPath } from '../../../../shared/wsl-paths'
|
||||
import { wasWslDistroLastSeenRunningSettled } from '../../../wsl/discovery/wsl-background-distro-gate'
|
||||
import { WSL_TRANSCRIPT_RUNNING_LEASE_MS } from '../../../native-chat/wsl-transcript-running-admission'
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { wslGatedStat } from '../../../native-chat/wsl-transcript-fs-access'
|
||||
import { columnExists, tableExists } from '../../../opencode-usage/schema-helpers'
|
||||
import { readOpenCodeDatabase } from '../opencode/sqlite/session-scanner-opencode-sqlite-open'
|
||||
@@ -177,14 +176,3 @@ function unixSecondsToIso(value: unknown): string | null {
|
||||
const date = new Date(seconds * 1000)
|
||||
return Number.isFinite(date.getTime()) ? date.toISOString() : null
|
||||
}
|
||||
|
||||
export const devinSearchDependencies: (root: string) => readonly SessionDiscoveryDependency[] = (
|
||||
root
|
||||
) => [
|
||||
{
|
||||
path: dirname(root),
|
||||
eventKey: (path) => (/^sessions\.db(?:-wal)?$/.test(basename(path)) ? 'db' : null),
|
||||
fileKey: () => 'db',
|
||||
metadata: true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,23 +1,25 @@
|
||||
export type KimiSessionIndexIdentity = {
|
||||
dev?: number
|
||||
ino?: number
|
||||
changeTimeMs: number
|
||||
mtimeMs: number
|
||||
sizeBytes: number
|
||||
}
|
||||
|
||||
type KimiSessionIndexCacheEntry = {
|
||||
type KimiSessionIndexCacheEntry<T> = {
|
||||
expiresAt: number
|
||||
generation: number
|
||||
identity: KimiSessionIndexIdentity
|
||||
timer: NodeJS.Timeout | null
|
||||
value: Promise<Map<string, string>>
|
||||
value: Promise<T>
|
||||
}
|
||||
|
||||
export const KIMI_WORK_DIR_CACHE_MAX_INDEX_PATHS = 64
|
||||
// Active scans refresh expiry; idle surfaces release parsed index maps.
|
||||
export const KIMI_WORK_DIR_CACHE_TTL_MS = 5 * 60_000
|
||||
|
||||
export class KimiSessionIndexCache {
|
||||
private readonly entries = new Map<string, KimiSessionIndexCacheEntry>()
|
||||
export class KimiSessionIndexCache<T = Map<string, string>> {
|
||||
private readonly entries = new Map<string, KimiSessionIndexCacheEntry<T>>()
|
||||
private minimumCacheGeneration = 0
|
||||
private nextGeneration = 0
|
||||
|
||||
@@ -48,10 +50,10 @@ export class KimiSessionIndexCache {
|
||||
indexPath: string,
|
||||
identity: KimiSessionIndexIdentity,
|
||||
generation: number,
|
||||
load: () => Promise<Map<string, string>>
|
||||
): Promise<Map<string, string>> {
|
||||
load: (previous: Promise<T> | undefined) => Promise<T>
|
||||
): Promise<T> {
|
||||
if (generation < this.minimumCacheGeneration) {
|
||||
return load()
|
||||
return load(undefined)
|
||||
}
|
||||
const cached = this.entries.get(indexPath)
|
||||
const now = Date.now()
|
||||
@@ -61,15 +63,15 @@ export class KimiSessionIndexCache {
|
||||
}
|
||||
if (cached && cached.generation > generation) {
|
||||
// An older stat must not overwrite a generation cached by a newer concurrent read.
|
||||
return load()
|
||||
return load(undefined)
|
||||
}
|
||||
|
||||
const entry: KimiSessionIndexCacheEntry = {
|
||||
const entry: KimiSessionIndexCacheEntry<T> = {
|
||||
expiresAt: now + KIMI_WORK_DIR_CACHE_TTL_MS,
|
||||
generation,
|
||||
identity,
|
||||
timer: null,
|
||||
value: load()
|
||||
value: load(cached?.value)
|
||||
}
|
||||
this.remember(indexPath, entry, now)
|
||||
return entry.value
|
||||
@@ -83,7 +85,7 @@ export class KimiSessionIndexCache {
|
||||
return this.entries.size
|
||||
}
|
||||
|
||||
private forget(indexPath: string, entry: KimiSessionIndexCacheEntry): void {
|
||||
private forget(indexPath: string, entry: KimiSessionIndexCacheEntry<T>): void {
|
||||
if (this.entries.get(indexPath) !== entry) {
|
||||
return
|
||||
}
|
||||
@@ -93,7 +95,7 @@ export class KimiSessionIndexCache {
|
||||
this.entries.delete(indexPath)
|
||||
}
|
||||
|
||||
private remember(indexPath: string, entry: KimiSessionIndexCacheEntry, now: number): void {
|
||||
private remember(indexPath: string, entry: KimiSessionIndexCacheEntry<T>, now: number): void {
|
||||
const replaced = this.entries.get(indexPath)
|
||||
if (replaced?.timer && replaced !== entry) {
|
||||
clearTimeout(replaced.timer)
|
||||
@@ -119,6 +121,8 @@ export class KimiSessionIndexCache {
|
||||
|
||||
function identitiesMatch(left: KimiSessionIndexIdentity, right: KimiSessionIndexIdentity): boolean {
|
||||
return (
|
||||
left.dev === right.dev &&
|
||||
left.ino === right.ino &&
|
||||
left.changeTimeMs === right.changeTimeMs &&
|
||||
left.mtimeMs === right.mtimeMs &&
|
||||
left.sizeBytes === right.sizeBytes
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import { createSessionJsonlDependency } from '../../session-jsonl-dependency'
|
||||
import type { SessionDiscoveryDependency } from '../../session-discovery-context'
|
||||
import { throwIfAiVaultScanCancelled, waitForAiVaultScan } from '../../ai-vault-scan-cancellation'
|
||||
import { homedir } from 'node:os'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { resolveAbsoluteDirOverride } from '../../../../shared/absolute-dir-override'
|
||||
import {
|
||||
openTranscriptReadStream,
|
||||
wslGatedStat
|
||||
} from '../../../native-chat/wsl-transcript-fs-access'
|
||||
import { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error'
|
||||
import { wslGatedStat } from '../../../native-chat/wsl-transcript-fs-access'
|
||||
import { readSessionIndexJsonl, type SessionIndexJsonlRead } from '../../session-index-jsonl-read'
|
||||
import { asRecord } from '../../session-scanner-record-value'
|
||||
import { extractString } from '../../session-scanner-values'
|
||||
import { KimiSessionIndexCache } from './session-scanner-kimi-index-cache'
|
||||
@@ -55,7 +52,7 @@ export function kimiPrimaryAgentWirePath(
|
||||
}
|
||||
|
||||
// Share parsed indexes by path and file identity; one home can hold many sessions.
|
||||
const workDirCacheByIndexPath = new KimiSessionIndexCache()
|
||||
const workDirCacheByIndexPath = new KimiSessionIndexCache<SessionIndexJsonlRead | undefined>()
|
||||
|
||||
export function clearKimiSessionIndexCache(): void {
|
||||
workDirCacheByIndexPath.clear()
|
||||
@@ -84,55 +81,42 @@ export async function readKimiWorkDirBySessionId(
|
||||
workDirCacheByIndexPath.get(
|
||||
indexPath,
|
||||
{
|
||||
dev: identity.dev,
|
||||
ino: identity.ino,
|
||||
changeTimeMs: identity.ctimeMs,
|
||||
mtimeMs: identity.mtimeMs,
|
||||
sizeBytes: identity.size
|
||||
},
|
||||
generation,
|
||||
() =>
|
||||
parseKimiSessionIndex(indexPath).then(({ map, refused }) => {
|
||||
// Evict inside the shared loader so every concurrent caller retries refused reads without a signature change.
|
||||
if (refused) {
|
||||
workDirCacheByIndexPath.delete(indexPath, generation)
|
||||
}
|
||||
return map
|
||||
})
|
||||
async (previous) => {
|
||||
try {
|
||||
return await readSessionIndexJsonl(
|
||||
indexPath,
|
||||
identity,
|
||||
await previous,
|
||||
parseKimiWorkDirRecord
|
||||
)
|
||||
} catch {
|
||||
// Refusal never becomes an accepted signature; the next caller must retry.
|
||||
workDirCacheByIndexPath.delete(indexPath, generation)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
),
|
||||
signal
|
||||
)
|
||||
).then((read) => read?.values ?? new Map<string, string>())
|
||||
}
|
||||
|
||||
async function parseKimiSessionIndex(
|
||||
indexPath: string
|
||||
): Promise<{ map: Map<string, string>; refused: boolean }> {
|
||||
const map = new Map<string, string>()
|
||||
// Share partial results after read failure so one broken index does not hide every session under its home.
|
||||
function parseKimiWorkDirRecord(line: string): readonly [string, string] | null {
|
||||
let record: Record<string, unknown> | null
|
||||
try {
|
||||
const lines = createInterface({
|
||||
input: openTranscriptReadStream(indexPath, { encoding: 'utf-8' }, 'scan'),
|
||||
crlfDelay: Infinity
|
||||
})
|
||||
for await (const line of lines) {
|
||||
if (!line.trim()) {
|
||||
continue
|
||||
}
|
||||
let record: Record<string, unknown> | null
|
||||
try {
|
||||
record = asRecord(JSON.parse(line) as unknown)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const sessionId = extractString(record?.sessionId)
|
||||
const workDir = extractString(record?.workDir)
|
||||
if (sessionId && workDir) {
|
||||
// Later lines win so a resumed session reflects its most recent workDir.
|
||||
map.set(sessionId, workDir)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return { map, refused: error instanceof WslTranscriptFsError }
|
||||
record = asRecord(JSON.parse(line) as unknown)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
return { map, refused: false }
|
||||
const id = extractString(record?.sessionId),
|
||||
cwd = extractString(record?.workDir)
|
||||
return id && cwd ? [id, cwd] : null
|
||||
}
|
||||
|
||||
export const kimiSearchDependencies: (root: string) => readonly SessionDiscoveryDependency[] = (
|
||||
@@ -142,11 +126,19 @@ export const kimiSearchDependencies: (root: string) => readonly SessionDiscovery
|
||||
path: root,
|
||||
fileKey: (path) => basename(dirname(path)),
|
||||
eventKey: (path) =>
|
||||
path.split(/[\\/]/).find((segment) => segment.startsWith('session_')) ?? null
|
||||
basename(path) === 'wire.jsonl'
|
||||
? (path.split(/[\\/]/).find((segment) => segment.startsWith('session_')) ?? null)
|
||||
: null
|
||||
},
|
||||
{
|
||||
path: dirname(root),
|
||||
fileKey: () => 'index',
|
||||
eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'index' : null)
|
||||
mode: 'shallow',
|
||||
fileKey: (path) => basename(dirname(path)),
|
||||
eventKey: (path) => (basename(path) === 'session_index.jsonl' ? 'index' : null),
|
||||
observe: () =>
|
||||
createSessionJsonlDependency(
|
||||
join(dirname(root), 'session_index.jsonl'),
|
||||
parseKimiWorkDirRecord
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { discoverSessionContainer } from '../../session-discovery-context'
|
||||
import { currentSessionDiscovery, discoverSessionContainer } from '../../session-discovery-context'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import type { AiVaultScanIssue } from '../../../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import type { WslTranscriptFsError } from '../../../native-chat/wsl-transcript-fs-error'
|
||||
@@ -12,6 +12,7 @@ import { discoverOpenCodeSessions } from './sqlite/session-scanner-opencode-sqli
|
||||
import { listOpenCode2SqliteSessionsViaWorker } from './reader-transport/session-scanner-opencode-sqlite-worker-spawn'
|
||||
import { isOpenCodeV2DatabaseName } from '../../../../shared/opencode-database-name'
|
||||
import type { AiVaultScanOptions, SessionFileDiscovery } from '../../session-scanner-types'
|
||||
import { isMissingTranscriptPathError } from '../../session-scanner-discovery'
|
||||
import { scopeWalksAgentFiles } from '../../session-scope-cwd'
|
||||
|
||||
type OpenCodeSource = {
|
||||
@@ -55,12 +56,17 @@ function opencodeSources(
|
||||
const onRefusal = (path: string, error: WslTranscriptFsError): void => {
|
||||
recordSessionScanIssue(issues, { agent: 'opencode', path, ...scanIssueTextFromError(error) })
|
||||
}
|
||||
const onFsError = (path: string, error: unknown): void => {
|
||||
if (currentSessionDiscovery() && !isMissingTranscriptPathError(error)) {
|
||||
recordSessionScanIssue(issues, { agent: 'opencode', path, ...scanIssueTextFromError(error) })
|
||||
}
|
||||
}
|
||||
const dataDirectorySource = (dataDirectory: string): OpenCodeSource => ({
|
||||
storageDir: join(dataDirectory, 'storage'),
|
||||
dbPaths: async () =>
|
||||
options.opencodeDbPaths
|
||||
? []
|
||||
: listOpenCodeDatabasesInDirectory(dataDirectory, onRefusal, options.signal)
|
||||
: listOpenCodeDatabasesInDirectory(dataDirectory, onRefusal, options.signal, onFsError)
|
||||
})
|
||||
const storageDir = options.opencodeStorageDir ?? resolveOpenCodeStorageDirectory()
|
||||
const primary: OpenCodeSource = {
|
||||
@@ -71,8 +77,13 @@ function opencodeSources(
|
||||
}
|
||||
// Custom storage roots still place SQLite databases in the parent data directory.
|
||||
return options.opencodeStorageDir
|
||||
? listOpenCodeDatabasesInDirectory(dirname(storageDir), onRefusal, options.signal)
|
||||
: listOpenCodeDatabases(onRefusal, undefined, options.signal)
|
||||
? listOpenCodeDatabasesInDirectory(
|
||||
dirname(storageDir),
|
||||
onRefusal,
|
||||
options.signal,
|
||||
onFsError
|
||||
)
|
||||
: listOpenCodeDatabases(onRefusal, onFsError, options.signal)
|
||||
}
|
||||
}
|
||||
// Orca-managed accounts relocate XDG_DATA_HOME, so their stores are invisible to the default lookup.
|
||||
@@ -95,24 +106,30 @@ async function discoverOpenCode2Sessions(
|
||||
issues: AiVaultScanIssue[],
|
||||
options: AiVaultScanOptions
|
||||
): Promise<SessionFileDiscovery> {
|
||||
const list = (scopePaths?: readonly string[]) =>
|
||||
const list = (scopePaths?: readonly string[], paths = dbPaths, count = limit) =>
|
||||
listOpenCode2SqliteSessionsViaWorker({
|
||||
dbPaths,
|
||||
limit,
|
||||
dbPaths: paths,
|
||||
limit: count,
|
||||
issues,
|
||||
scopePaths,
|
||||
signal: options.signal
|
||||
}).then((candidates) => candidates.map((candidate) => candidate.file))
|
||||
const [discovery, scopeFiles] = await Promise.all([
|
||||
discoverSessionContainer(`opencode2:${dbPaths.join('|')}`, dbPaths, async () => {
|
||||
const before = issues.length
|
||||
return {
|
||||
agent: 'opencode2',
|
||||
rootDir: storageDir,
|
||||
files: await list(),
|
||||
complete: !issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
}),
|
||||
discoverSessionContainer(
|
||||
`opencode2:${storageDir}`,
|
||||
dbPaths,
|
||||
async (paths, count) => {
|
||||
const before = issues.length
|
||||
return {
|
||||
agent: 'opencode2',
|
||||
rootDir: storageDir,
|
||||
files: await list(undefined, paths, count),
|
||||
complete: !issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
},
|
||||
limit,
|
||||
issues
|
||||
),
|
||||
scopeWalksAgentFiles('opencode2', options.scopePaths) ? list(options.scopePaths) : undefined
|
||||
])
|
||||
return {
|
||||
|
||||
+24
-15
@@ -24,15 +24,18 @@ export async function discoverOpenCodeSessions(args: {
|
||||
signal?: AbortSignal
|
||||
}): Promise<SessionFileDiscovery> {
|
||||
const walksScope = scopeWalksAgentFiles('opencode', args.scopePaths)
|
||||
const listSqlite = (scopePaths?: readonly string[]) =>
|
||||
const listSqlite = (
|
||||
scopePaths?: readonly string[],
|
||||
dbPaths = args.dbPaths,
|
||||
limit = args.limitPerAgent
|
||||
) =>
|
||||
listOpenCodeSqliteSessionsViaWorker({
|
||||
dbPaths: args.dbPaths,
|
||||
limit: args.limitPerAgent,
|
||||
dbPaths,
|
||||
limit,
|
||||
issues: args.issues,
|
||||
scopePaths,
|
||||
signal: args.signal
|
||||
})
|
||||
const containerKey = `opencode:${args.dbPaths.join('|')}`
|
||||
const [fileDiscovery, sqliteDiscovery, sqliteScopeCandidates] = await Promise.all([
|
||||
discoverFiles({
|
||||
rootDir: join(args.storageDir, 'session'),
|
||||
@@ -51,22 +54,28 @@ export async function discoverOpenCodeSessions(args: {
|
||||
keepScopeFiles: walksScope,
|
||||
signal: args.signal
|
||||
}),
|
||||
discoverSessionContainer(containerKey, args.dbPaths, async () => {
|
||||
const before = args.issues.length
|
||||
const candidates = await listSqlite()
|
||||
return {
|
||||
agent: 'opencode',
|
||||
rootDir: args.storageDir,
|
||||
files: candidates.map((candidate) => candidate.file),
|
||||
complete: !args.issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
}),
|
||||
discoverSessionContainer(
|
||||
`opencode:${args.storageDir}`,
|
||||
args.dbPaths,
|
||||
async (paths, limit) => {
|
||||
const before = args.issues.length
|
||||
const candidates = await listSqlite(undefined, paths, limit)
|
||||
return {
|
||||
agent: 'opencode',
|
||||
rootDir: args.storageDir,
|
||||
files: candidates.map((candidate) => candidate.file),
|
||||
complete: !args.issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
},
|
||||
args.limitPerAgent,
|
||||
args.issues
|
||||
),
|
||||
walksScope ? listSqlite(args.scopePaths) : undefined
|
||||
])
|
||||
|
||||
const sqliteFiles = sqliteDiscovery.files
|
||||
const sqliteScopeFiles = sqliteScopeCandidates?.map((c) => c.file)
|
||||
const sqliteSessionIds = currentSessionDiscovery()?.membership(containerKey) ?? new Set<string>()
|
||||
const sqliteSessionIds = new Set<string>()
|
||||
for (const file of [...sqliteFiles, ...(sqliteScopeFiles ?? [])]) {
|
||||
const parsed = splitOpenCodeSqliteCandidate(file.path)
|
||||
if (parsed) {
|
||||
|
||||
@@ -21,26 +21,47 @@ export function zcodeDiscoveries(
|
||||
: [homedir(), ...wslHomeDirs].map((home) => join(home, '.zcode', 'cli', 'db', 'db.sqlite'))
|
||||
return paths.map(async (dbPath) => {
|
||||
if (!(await zcodeDatabaseExists(dbPath, issues, options.signal))) {
|
||||
return { agent: 'zcode' as const, rootDir: dbPath, files: [] }
|
||||
}
|
||||
const list = (scopePaths?: readonly string[]) =>
|
||||
listZcodeSqliteSessionsViaWorker({
|
||||
dbPaths: [dbPath],
|
||||
return discoverSessionContainer(
|
||||
`zcode:${dbPath}`,
|
||||
[dbPath],
|
||||
async () => ({
|
||||
agent: 'zcode',
|
||||
rootDir: dbPath,
|
||||
files: [],
|
||||
complete: !issues.some((issue) => issue.path === dbPath && issue.kind !== 'notice')
|
||||
}),
|
||||
limit,
|
||||
issues
|
||||
)
|
||||
}
|
||||
const list = (
|
||||
scopePaths?: readonly string[],
|
||||
paths: readonly string[] = [dbPath],
|
||||
count = limit
|
||||
) =>
|
||||
listZcodeSqliteSessionsViaWorker({
|
||||
dbPaths: paths,
|
||||
limit: count,
|
||||
issues,
|
||||
scopePaths,
|
||||
signal: options.signal
|
||||
}).then((candidates) => candidates.map((candidate) => candidate.file))
|
||||
const [discovery, scopeFiles] = await Promise.all([
|
||||
discoverSessionContainer(`zcode:${dbPath}`, [dbPath], async () => {
|
||||
const before = issues.length
|
||||
return {
|
||||
agent: 'zcode',
|
||||
rootDir: dbPath,
|
||||
files: await list(),
|
||||
complete: !issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
}),
|
||||
discoverSessionContainer(
|
||||
`zcode:${dbPath}`,
|
||||
[dbPath],
|
||||
async (paths, count) => {
|
||||
const before = issues.length
|
||||
return {
|
||||
agent: 'zcode',
|
||||
rootDir: dbPath,
|
||||
files: await list(undefined, paths, count),
|
||||
complete: !issues.slice(before).some((issue) => issue.kind !== 'notice')
|
||||
}
|
||||
},
|
||||
limit,
|
||||
issues
|
||||
),
|
||||
scopeWalksAgentFiles('zcode', options.scopePaths) ? list(options.scopePaths) : undefined
|
||||
])
|
||||
return {
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import type { DiscoverSessionFilesArgs } from './session-scanner-discovery'
|
||||
import type { AiVaultScanIssue } from '../../shared/ai-vault/contracts/ai-vault-types'
|
||||
import type { SessionFileDiscovery } from './session-scanner-types'
|
||||
|
||||
export type SessionDiscoveryDependency = {
|
||||
path: string
|
||||
mode?: 'shallow' | 'recursive'
|
||||
fileKey?: (path: string) => string | null
|
||||
eventKey?: (path: string) => string | null
|
||||
metadata?: boolean
|
||||
snapshot?: () => Promise<ReadonlyMap<string, string>>
|
||||
observe?: () => (force: boolean) => Promise<ReadonlySet<string>>
|
||||
}
|
||||
|
||||
export type SessionDiscoveryContext = {
|
||||
@@ -15,10 +17,11 @@ export type SessionDiscoveryContext = {
|
||||
container(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: () => Promise<SessionFileDiscovery>
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
limit: number,
|
||||
issues: readonly AiVaultScanIssue[]
|
||||
): Promise<SessionFileDiscovery>
|
||||
artifacts(root: string): Map<string, string>
|
||||
membership(key: string): ReadonlySet<string> | undefined
|
||||
}
|
||||
|
||||
const discoveryContext = new AsyncLocalStorage<SessionDiscoveryContext>()
|
||||
@@ -34,7 +37,9 @@ export function withSessionDiscovery<T>(context: SessionDiscoveryContext, read:
|
||||
export function discoverSessionContainer(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: () => Promise<SessionFileDiscovery>
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
limit: number,
|
||||
issues: readonly AiVaultScanIssue[]
|
||||
): Promise<SessionFileDiscovery> {
|
||||
return currentSessionDiscovery()?.container(key, paths, read) ?? read()
|
||||
return currentSessionDiscovery()?.container(key, paths, read, limit, issues) ?? read(paths, limit)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { dirname, extname, relative } from 'node:path'
|
||||
import { dirname, extname, isAbsolute, relative } from 'node:path'
|
||||
import type { AiVaultAgentSource } from './session-scanner-types'
|
||||
|
||||
export function isDiscoverableSessionFile(
|
||||
@@ -6,7 +6,10 @@ export function isDiscoverableSessionFile(
|
||||
rootDir: string,
|
||||
filePath: string
|
||||
): boolean {
|
||||
if (!source.extensions.includes(extname(filePath).toLowerCase())) {
|
||||
if (
|
||||
!source.extensions.includes('*') &&
|
||||
!source.extensions.includes(extname(filePath).toLowerCase())
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (source.filePredicate && !source.filePredicate(filePath)) {
|
||||
@@ -28,5 +31,5 @@ export function pathSegments(filePath: string): string[] {
|
||||
|
||||
export function isInsideSessionRoot(root: string, path: string): boolean {
|
||||
const rel = relative(root, path)
|
||||
return rel !== '..' && !rel.startsWith('../') && !rel.startsWith('..\\') && !rel.startsWith('/')
|
||||
return !isAbsolute(rel) && rel !== '..' && !rel.startsWith('../') && !rel.startsWith('..\\')
|
||||
}
|
||||
|
||||
@@ -27,7 +27,11 @@ export class SessionInventoryDirectories extends Map<string, SessionInventoryDir
|
||||
}
|
||||
}
|
||||
|
||||
removeTree(path: string, removeFile: (path: string) => void): void {
|
||||
removeTree(
|
||||
path: string,
|
||||
removeFile: (path: string) => void,
|
||||
removeDirectory?: (path: string) => void
|
||||
): void {
|
||||
const directory = this.get(path)
|
||||
if (!directory) {
|
||||
return
|
||||
@@ -36,8 +40,9 @@ export class SessionInventoryDirectories extends Map<string, SessionInventoryDir
|
||||
removeFile(file)
|
||||
}
|
||||
for (const child of directory.children) {
|
||||
this.removeTree(child, removeFile)
|
||||
this.removeTree(child, removeFile, removeDirectory)
|
||||
}
|
||||
this.delete(path)
|
||||
removeDirectory?.(path)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import type { FileWithMtime } from './session-scanner-types'
|
||||
import { observeSessionFile, type DiscoverSessionFilesArgs } from './session-scanner-discovery'
|
||||
import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation'
|
||||
import { recordSessionScanIssue } from './session-scan-issues'
|
||||
import { retainRecentSessionFile } from './session-file-observation'
|
||||
import { sidecarEnrichesWithoutReparse } from './parse-cache/session-scanner-sidecar-enrichment'
|
||||
import { sidecarUnchanged } from './parse-cache/session-sidecar-stat'
|
||||
|
||||
export type SessionFileChange = { file: FileWithMtime; force: boolean }
|
||||
|
||||
/** Accepted file facts and retry debt survive both directory epochs and delivered events. */
|
||||
export class SessionInventoryFiles {
|
||||
readonly files = new Map<string, FileWithMtime>()
|
||||
readonly dependencies = new Map<string, Set<string>>()
|
||||
readonly failures = new Map<string, boolean>()
|
||||
readonly changes = new Map<string, SessionFileChange>()
|
||||
readonly removed = new Set<string>()
|
||||
readonly recent = new Map<string, FileWithMtime>()
|
||||
private readonly dependencyByFile = new Map<string, string>()
|
||||
constructor(
|
||||
private readonly addFile: (path: string) => void,
|
||||
private readonly scheduleParent: (path: string) => void,
|
||||
private readonly forgetDirty: (path: string) => void
|
||||
) {}
|
||||
async observe(path: string, args: DiscoverSessionFilesArgs, force: boolean, limit: number) {
|
||||
const observation = await observeSessionFile(
|
||||
path,
|
||||
!force && this.dependencyByFile.has(path)
|
||||
? () => this.dependencyByFile.get(path)
|
||||
: args.contentDependencyPath,
|
||||
args.signal
|
||||
)
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
if ('issue' in observation) {
|
||||
if (observation.missing && !this.files.has(path)) {
|
||||
this.failures.delete(path)
|
||||
return
|
||||
}
|
||||
if (!observation.missing) {
|
||||
recordSessionScanIssue(args.issues, { agent: args.agent, ...observation.issue })
|
||||
}
|
||||
this.failures.set(path, force || (this.failures.get(path) ?? false))
|
||||
// A successful parent listing, never a refused stat, can remove membership.
|
||||
this.scheduleParent(path)
|
||||
return
|
||||
}
|
||||
const metadataOnly = sidecarEnrichesWithoutReparse({
|
||||
agent: args.agent,
|
||||
file: observation.file,
|
||||
codexHome: null
|
||||
})
|
||||
this.failures.delete(path)
|
||||
if (!metadataOnly && observation.file.sidecar === 'unknown') {
|
||||
this.failures.set(path, force)
|
||||
}
|
||||
const file = {
|
||||
...observation.file,
|
||||
modifiedAt: new Date(observation.file.mtimeMs).toISOString()
|
||||
}
|
||||
const old = this.files.get(path)
|
||||
const dependency = metadataOnly ? undefined : observation.sidecarPath
|
||||
const previous = this.dependencyByFile.get(path)
|
||||
if (previous !== dependency) {
|
||||
if (previous) {
|
||||
this.removeDependency(previous, path)
|
||||
}
|
||||
if (dependency) {
|
||||
const owners = this.dependencies.get(dependency) ?? new Set<string>()
|
||||
owners.add(path)
|
||||
this.dependencies.set(dependency, owners)
|
||||
this.dependencyByFile.set(path, dependency)
|
||||
} else {
|
||||
this.dependencyByFile.delete(path)
|
||||
}
|
||||
}
|
||||
const dependencyChanged =
|
||||
!metadataOnly &&
|
||||
!!old &&
|
||||
file.sidecar !== 'unknown' &&
|
||||
!sidecarUnchanged(old.sidecar ?? 'none', file.sidecar ?? 'none')
|
||||
this.files.set(path, file)
|
||||
if (!old) {
|
||||
this.addFile(path)
|
||||
}
|
||||
if (
|
||||
force ||
|
||||
!old ||
|
||||
dependencyChanged ||
|
||||
old.mtimeMs !== file.mtimeMs ||
|
||||
old.sizeBytes !== file.sizeBytes ||
|
||||
old.dev !== file.dev ||
|
||||
old.ino !== file.ino
|
||||
) {
|
||||
this.changes.set(path, {
|
||||
file,
|
||||
force:
|
||||
dependencyChanged ||
|
||||
(force &&
|
||||
!!old &&
|
||||
old.mtimeMs === file.mtimeMs &&
|
||||
old.sizeBytes === file.sizeBytes &&
|
||||
old.dev === file.dev &&
|
||||
old.ino === file.ino)
|
||||
})
|
||||
}
|
||||
retainRecentSessionFile(this.recent, file, limit)
|
||||
}
|
||||
|
||||
forget(path: string): void {
|
||||
this.files.delete(path)
|
||||
this.failures.delete(path)
|
||||
this.forgetDirty(path)
|
||||
this.recent.delete(path)
|
||||
const dependency = this.dependencyByFile.get(path)
|
||||
if (dependency) {
|
||||
this.removeDependency(dependency, path)
|
||||
}
|
||||
this.dependencyByFile.delete(path)
|
||||
this.changes.delete(path)
|
||||
this.removed.add(path)
|
||||
}
|
||||
|
||||
private removeDependency(dependency: string, path: string): void {
|
||||
const owners = this.dependencies.get(dependency)
|
||||
owners?.delete(path)
|
||||
if (!owners?.size) {
|
||||
this.dependencies.delete(dependency)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,16 +8,13 @@ import { wslGatedReaddir, wslGatedStat } from '../native-chat/wsl-transcript-fs-
|
||||
import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation'
|
||||
import {
|
||||
isMissingTranscriptPathError,
|
||||
observeSessionFile,
|
||||
type DiscoverSessionFilesArgs
|
||||
} from './session-scanner-discovery'
|
||||
import type { FileWithMtime } from './session-scanner-types'
|
||||
import { isDiscoverableSessionFile, isInsideSessionRoot } from './session-file-discovery'
|
||||
import { recordSessionScanIssue, scanIssueTextFromError } from './session-scan-issues'
|
||||
import { retainRecentSessionFile } from './session-file-observation'
|
||||
import { sidecarUnchanged } from './parse-cache/session-sidecar-stat'
|
||||
import { SessionInventoryFiles, type SessionFileChange } from './session-file-inventory-files'
|
||||
|
||||
export type SessionFileChange = { file: FileWithMtime; force: boolean }
|
||||
export type { SessionFileChange } from './session-file-inventory-files'
|
||||
type Walk = {
|
||||
path: string
|
||||
depth: number
|
||||
@@ -29,34 +26,56 @@ type Walk = {
|
||||
|
||||
/** Retains discovery position independently of the read queue and of watcher delivery. */
|
||||
export class SessionFileInventory {
|
||||
readonly files = new Map<string, FileWithMtime>()
|
||||
readonly dependencies = new Map<string, Set<string>>()
|
||||
private readonly observations: SessionInventoryFiles
|
||||
get files() {
|
||||
return this.observations.files
|
||||
}
|
||||
get dependencies() {
|
||||
return this.observations.dependencies
|
||||
}
|
||||
private readonly directories: SessionInventoryDirectories
|
||||
private readonly dirty = new Map<string, boolean>()
|
||||
private readonly walks: Walk[] = []
|
||||
private readonly scheduled = new Set<string>()
|
||||
private readonly trailing = new Set<string>()
|
||||
private readonly recent = new Map<string, FileWithMtime>()
|
||||
private readonly dependencyByFile = new Map<string, string>()
|
||||
private readonly changes = new Map<string, SessionFileChange>()
|
||||
private readonly removed = new Set<string>()
|
||||
private readonly failures = new Set<string>()
|
||||
private reconcileAgain = false
|
||||
private reconciling = false
|
||||
private chooseDirty = true
|
||||
private readonly removedDirectories = new Set<string>()
|
||||
|
||||
constructor(readonly root: string) {
|
||||
this.directories = new SessionInventoryDirectories(root)
|
||||
this.observations = new SessionInventoryFiles(
|
||||
(path) => this.directories.addFile(path),
|
||||
(path) => this.schedule(dirname(path)),
|
||||
(path) => this.dirty.delete(path)
|
||||
)
|
||||
this.reconcile()
|
||||
}
|
||||
|
||||
get pending(): boolean {
|
||||
return this.walks.length > 0 || this.dirty.size > 0 || this.reconcileAgain
|
||||
return (
|
||||
this.walks.length > 0 ||
|
||||
this.dirty.size > 0 ||
|
||||
this.reconcileAgain ||
|
||||
this.failures.size > 0 ||
|
||||
this.observations.failures.size > 0
|
||||
)
|
||||
}
|
||||
|
||||
get walking(): boolean {
|
||||
return this.walks.length > 0 || this.reconcileAgain
|
||||
}
|
||||
|
||||
reconcile(): void {
|
||||
// A burst during a crawl leaves one trailing obligation, never restarts its prefix.
|
||||
if (!this.scheduled.has(this.root)) {
|
||||
this.schedule(this.root, true)
|
||||
if (this.reconciling) {
|
||||
this.reconcileAgain = true
|
||||
return
|
||||
}
|
||||
this.reconciling = true
|
||||
this.schedule(this.root, true)
|
||||
}
|
||||
|
||||
invalidate(path: string, force = true): void {
|
||||
@@ -69,7 +88,7 @@ export class SessionFileInventory {
|
||||
}
|
||||
|
||||
private schedule(path: string, reconcile = false): void {
|
||||
if (this.scheduled.has(path)) {
|
||||
if (!isInsideSessionRoot(this.root, path) || this.scheduled.has(path)) {
|
||||
return
|
||||
}
|
||||
const rel = relative(this.root, path)
|
||||
@@ -79,9 +98,11 @@ export class SessionFileInventory {
|
||||
|
||||
async pollRecent(args: DiscoverSessionFilesArgs, limit: number): Promise<void> {
|
||||
// This set is bounded on admission; never project/sort the archive to find recent paths.
|
||||
const paths = [...this.recent.keys()]
|
||||
const paths = [...this.observations.recent.keys()]
|
||||
for (const path of paths) {
|
||||
await this.observe(path, args, false, limit)
|
||||
if (!this.dirty.has(path)) {
|
||||
await this.observations.observe(path, args, false, limit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,77 +110,52 @@ export class SessionFileInventory {
|
||||
args: DiscoverSessionFilesArgs,
|
||||
overdue: () => boolean,
|
||||
recentLimit: number
|
||||
): Promise<{ changes: SessionFileChange[]; removed: string[] }> {
|
||||
): Promise<{ changes: SessionFileChange[]; removed: string[]; removedDirectories: string[] }> {
|
||||
let consumed = 0
|
||||
// Events arriving during an await stay in dirty; deleting before reading preserves their version.
|
||||
for (let index = 0; index < 16_384 && this.dirty.size; index++) {
|
||||
if (consumed > 0 && overdue()) {
|
||||
break
|
||||
while ((this.dirty.size || this.walks.length) && (consumed === 0 || !overdue())) {
|
||||
if (this.dirty.size && (!this.walks.length || this.chooseDirty)) {
|
||||
this.chooseDirty = false
|
||||
await this.drainDirty(args, recentLimit)
|
||||
consumed++
|
||||
continue
|
||||
}
|
||||
const first = this.dirty.entries().next().value
|
||||
if (!first) {
|
||||
break
|
||||
}
|
||||
const [path, force] = first
|
||||
this.dirty.delete(path)
|
||||
const owners = this.dependencies.get(path)
|
||||
if (owners) {
|
||||
for (const owner of owners) {
|
||||
this.invalidate(owner, force)
|
||||
}
|
||||
}
|
||||
if (this.accepts(path, args)) {
|
||||
await this.observe(path, args, force, recentLimit)
|
||||
} else if (isInsideSessionRoot(this.root, path)) {
|
||||
try {
|
||||
if ((await wslGatedStat(path, 'scan', args.signal)).isDirectory()) {
|
||||
if (this.scheduled.has(path)) {
|
||||
this.trailing.add(path)
|
||||
} else {
|
||||
this.schedule(path)
|
||||
}
|
||||
} else if (this.directories.has(path)) {
|
||||
this.schedule(dirname(path))
|
||||
}
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
if (isMissingTranscriptPathError(error)) {
|
||||
this.schedule(dirname(path))
|
||||
} else {
|
||||
this.issue(path, error, args)
|
||||
}
|
||||
}
|
||||
}
|
||||
consumed++
|
||||
}
|
||||
while (this.walks.length && (consumed === 0 || !overdue())) {
|
||||
this.chooseDirty = true
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
const walk = this.walks.at(-1)!
|
||||
const walk = this.walks[0]
|
||||
if (!walk.entries) {
|
||||
try {
|
||||
walk.entries = await wslGatedReaddir(walk.path, 'scan', args.signal)
|
||||
args.onDirectoryEntries?.(walk.path, walk.entries)
|
||||
const names = new Set(walk.entries.map((entry) => entry.name))
|
||||
const entries = new Map(walk.entries.map((entry) => [entry.name, entry]))
|
||||
const old = this.directories.get(walk.path)
|
||||
walk.previous = old
|
||||
if (old) {
|
||||
for (const path of old.files) {
|
||||
if (!names.has(relative(walk.path, path))) {
|
||||
this.forget(path)
|
||||
if (!entries.get(relative(walk.path, path))?.isFile()) {
|
||||
this.observations.forget(path)
|
||||
old.files.delete(path)
|
||||
}
|
||||
}
|
||||
for (const child of old.children) {
|
||||
if (!names.has(relative(walk.path, child))) {
|
||||
this.directories.removeTree(child, (path) => this.forget(path))
|
||||
if (!entries.get(relative(walk.path, child))?.isDirectory()) {
|
||||
this.removeDirectory(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
this.directories.set(walk.path, { files: new Set(), children: new Set() })
|
||||
this.directories.set(walk.path, old ?? { files: new Set(), children: new Set() })
|
||||
this.failures.delete(walk.path)
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
this.issue(walk.path, error, args)
|
||||
this.walks.pop()
|
||||
if (isMissingTranscriptPathError(error)) {
|
||||
this.failures.delete(walk.path)
|
||||
this.removeDirectory(walk.path)
|
||||
} else {
|
||||
this.issue(walk.path, error, args)
|
||||
}
|
||||
const index = this.walks.indexOf(walk)
|
||||
if (index !== -1) {
|
||||
this.walks.splice(index, 1)
|
||||
}
|
||||
this.scheduled.delete(walk.path)
|
||||
continue
|
||||
}
|
||||
@@ -170,15 +166,17 @@ export class SessionFileInventory {
|
||||
const entry = walk.entries[walk.offset]
|
||||
const path = join(walk.path, entry.name)
|
||||
if (entry.isDirectory() && (args.directoryPredicate?.(entry.name, walk.depth) ?? true)) {
|
||||
const known = walk.previous?.children.has(path)
|
||||
node.children.add(path)
|
||||
if (walk.reconcile || !walk.previous?.children.has(path)) {
|
||||
if (walk.reconcile || !known) {
|
||||
this.schedule(path, walk.reconcile)
|
||||
}
|
||||
} else if (entry.isFile() && this.accepts(path, args)) {
|
||||
const known = walk.previous?.files.has(path)
|
||||
node.files.add(path)
|
||||
// Do not advance past an aborted observation.
|
||||
if (walk.reconcile || !walk.previous?.files.has(path)) {
|
||||
await this.observe(path, args, false, recentLimit)
|
||||
if (walk.reconcile || !known) {
|
||||
await this.observations.observe(path, args, false, recentLimit)
|
||||
}
|
||||
}
|
||||
walk.offset++
|
||||
@@ -193,20 +191,92 @@ export class SessionFileInventory {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!this.walks.length && this.reconcileAgain) {
|
||||
this.reconcileAgain = false
|
||||
this.schedule(this.root, true)
|
||||
if (!this.walks.length) {
|
||||
this.reconciling = false
|
||||
if (this.reconcileAgain) {
|
||||
this.reconcileAgain = false
|
||||
this.reconcile()
|
||||
}
|
||||
}
|
||||
const changes = [...this.changes.values()]
|
||||
const removed = [...this.removed]
|
||||
this.changes.clear()
|
||||
this.removed.clear()
|
||||
return { changes, removed }
|
||||
const changes = [...this.observations.changes.values()]
|
||||
const removed = [...this.observations.removed]
|
||||
this.observations.changes.clear()
|
||||
this.observations.removed.clear()
|
||||
const removedDirectories = [...this.removedDirectories]
|
||||
this.removedDirectories.clear()
|
||||
return { changes, removed, removedDirectories }
|
||||
}
|
||||
|
||||
private async drainDirty(args: DiscoverSessionFilesArgs, recentLimit: number): Promise<void> {
|
||||
const first = this.dirty.entries().next().value
|
||||
if (!first) {
|
||||
return
|
||||
}
|
||||
const [path, force] = first
|
||||
this.dirty.delete(path)
|
||||
const owners = this.dependencies.get(path)
|
||||
if (owners) {
|
||||
for (const owner of owners) {
|
||||
this.invalidate(owner, force)
|
||||
}
|
||||
}
|
||||
if (this.accepts(path, args)) {
|
||||
await this.observations.observe(path, args, force, recentLimit)
|
||||
} else if (isInsideSessionRoot(this.root, path)) {
|
||||
try {
|
||||
const stat = await wslGatedStat(path, 'scan', args.signal)
|
||||
this.failures.delete(path)
|
||||
if (stat.isDirectory()) {
|
||||
if (!isInsideSessionRoot(this.root, path) || this.scheduled.has(path)) {
|
||||
this.trailing.add(path)
|
||||
} else {
|
||||
this.schedule(path)
|
||||
}
|
||||
} else if (this.directories.has(path)) {
|
||||
this.schedule(dirname(path))
|
||||
}
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
if (isMissingTranscriptPathError(error)) {
|
||||
this.removeDirectory(path)
|
||||
this.schedule(dirname(path))
|
||||
} else {
|
||||
this.issue(path, error, args)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private removeDirectory(path: string): void {
|
||||
for (let index = this.walks.length - 1; index >= 0; index--) {
|
||||
const walk = this.walks[index]
|
||||
if (isInsideSessionRoot(path, walk.path)) {
|
||||
this.walks.splice(index, 1)
|
||||
this.scheduled.delete(walk.path)
|
||||
this.trailing.delete(walk.path)
|
||||
}
|
||||
}
|
||||
for (const failed of this.failures) {
|
||||
if (isInsideSessionRoot(path, failed)) {
|
||||
this.failures.delete(failed)
|
||||
}
|
||||
}
|
||||
this.directories.get(dirname(path))?.children.delete(path)
|
||||
this.directories.removeTree(
|
||||
path,
|
||||
(file) => this.observations.forget(file),
|
||||
(directory) => {
|
||||
this.removedDirectories.add(directory)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
retryFailures(): void {
|
||||
for (const path of this.failures) {
|
||||
this.schedule(path)
|
||||
this.schedule(path, true)
|
||||
}
|
||||
for (const [path, force] of this.observations.failures) {
|
||||
this.invalidate(path, force)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,94 +284,6 @@ export class SessionFileInventory {
|
||||
return isInsideSessionRoot(this.root, path) && isDiscoverableSessionFile(args, this.root, path)
|
||||
}
|
||||
|
||||
private async observe(
|
||||
path: string,
|
||||
args: DiscoverSessionFilesArgs,
|
||||
force: boolean,
|
||||
limit: number
|
||||
) {
|
||||
const observation = await observeSessionFile(
|
||||
path,
|
||||
!force && this.dependencyByFile.has(path)
|
||||
? () => this.dependencyByFile.get(path)
|
||||
: args.contentDependencyPath,
|
||||
args.signal
|
||||
)
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
if ('issue' in observation) {
|
||||
if (!observation.missing) {
|
||||
recordSessionScanIssue(args.issues, { agent: args.agent, ...observation.issue })
|
||||
}
|
||||
// Only a successful parent listing removes membership; an unreadable stat is never deletion proof.
|
||||
if (this.files.has(path)) {
|
||||
this.schedule(dirname(path))
|
||||
}
|
||||
return
|
||||
}
|
||||
const file = {
|
||||
...observation.file,
|
||||
modifiedAt: new Date(observation.file.mtimeMs).toISOString()
|
||||
}
|
||||
const old = this.files.get(path)
|
||||
const dependency = observation.sidecarPath
|
||||
const previous = this.dependencyByFile.get(path)
|
||||
if (previous !== dependency) {
|
||||
if (previous) {
|
||||
this.dependencies.get(previous)?.delete(path)
|
||||
}
|
||||
if (dependency) {
|
||||
const owners = this.dependencies.get(dependency) ?? new Set<string>()
|
||||
owners.add(path)
|
||||
this.dependencies.set(dependency, owners)
|
||||
this.dependencyByFile.set(path, dependency)
|
||||
} else {
|
||||
this.dependencyByFile.delete(path)
|
||||
}
|
||||
}
|
||||
const dependencyChanged =
|
||||
!!old &&
|
||||
file.sidecar !== 'unknown' &&
|
||||
!sidecarUnchanged(old.sidecar ?? 'none', file.sidecar ?? 'none')
|
||||
this.files.set(path, file)
|
||||
if (!old) {
|
||||
this.directories.addFile(path)
|
||||
}
|
||||
if (
|
||||
force ||
|
||||
!old ||
|
||||
dependencyChanged ||
|
||||
old.mtimeMs !== file.mtimeMs ||
|
||||
old.sizeBytes !== file.sizeBytes ||
|
||||
old.dev !== file.dev ||
|
||||
old.ino !== file.ino
|
||||
) {
|
||||
this.changes.set(path, {
|
||||
file,
|
||||
force:
|
||||
dependencyChanged ||
|
||||
(force &&
|
||||
!!old &&
|
||||
old.mtimeMs === file.mtimeMs &&
|
||||
old.sizeBytes === file.sizeBytes &&
|
||||
old.dev === file.dev &&
|
||||
old.ino === file.ino)
|
||||
})
|
||||
}
|
||||
retainRecentSessionFile(this.recent, file, limit)
|
||||
}
|
||||
|
||||
private forget(path: string): void {
|
||||
this.files.delete(path)
|
||||
this.recent.delete(path)
|
||||
const dependency = this.dependencyByFile.get(path)
|
||||
if (dependency) {
|
||||
this.dependencies.get(dependency)?.delete(path)
|
||||
}
|
||||
this.dependencyByFile.delete(path)
|
||||
this.changes.delete(path)
|
||||
this.removed.add(path)
|
||||
}
|
||||
|
||||
private issue(path: string, error: unknown, args: DiscoverSessionFilesArgs) {
|
||||
this.failures.add(path)
|
||||
recordSessionScanIssue(args.issues, {
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { Stats } from 'node:fs'
|
||||
import {
|
||||
openTranscriptReadStream,
|
||||
readTranscriptSlice
|
||||
} from '../native-chat/wsl-transcript-fs-access'
|
||||
import { consumeCompleteJsonlLines } from './session-scanner-jsonl-reader'
|
||||
|
||||
export type SessionIndexJsonlRead = {
|
||||
values: Map<string, string>
|
||||
changedKeys: ReadonlySet<string>
|
||||
offset: number
|
||||
readThrough: number
|
||||
dev: number
|
||||
ino: number
|
||||
firstRecord: { length: number; hash: string } | null
|
||||
}
|
||||
|
||||
async function firstRecordSignature(path: string, end: number) {
|
||||
const hash = createHash('sha256')
|
||||
let length = 0
|
||||
for await (const chunk of openTranscriptReadStream(path, { end: end - 1 }, 'scan')) {
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
const newline = bytes.indexOf(0x0a)
|
||||
const prefix = newline === -1 ? bytes : bytes.subarray(0, newline + 1)
|
||||
hash.update(prefix)
|
||||
length += prefix.length
|
||||
if (newline !== -1) {
|
||||
return { length, hash: hash.digest('hex') }
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export async function readSessionIndexJsonl(
|
||||
path: string,
|
||||
stat: Stats,
|
||||
previous: SessionIndexJsonlRead | undefined,
|
||||
parse: (line: string) => readonly [string, string] | null
|
||||
): Promise<SessionIndexJsonlRead> {
|
||||
let append = false
|
||||
if (
|
||||
previous?.firstRecord &&
|
||||
previous.offset === previous.readThrough &&
|
||||
stat.size > previous.readThrough &&
|
||||
stat.dev !== 0 &&
|
||||
stat.ino !== 0 &&
|
||||
previous.dev !== 0 &&
|
||||
previous.ino !== 0 &&
|
||||
stat.dev === previous.dev &&
|
||||
stat.ino === previous.ino
|
||||
) {
|
||||
try {
|
||||
const prefix = await firstRecordSignature(path, previous.firstRecord.length)
|
||||
const boundary = await readTranscriptSlice(path, previous.offset - 1, 1, 'scan')
|
||||
// Growth alone cannot distinguish an append from Codex replacing the index on removal.
|
||||
append = prefix?.hash === previous.firstRecord.hash && boundary[0] === 0x0a
|
||||
} catch {
|
||||
// A failed continuity check requires a complete reread.
|
||||
}
|
||||
}
|
||||
const updates = new Map<string, string>()
|
||||
const applyLine = (line: string) => {
|
||||
const entry = parse(line)
|
||||
if (entry) {
|
||||
updates.set(entry[0], entry[1])
|
||||
}
|
||||
}
|
||||
const start = append ? previous!.offset : 0
|
||||
const { consumedThrough, trailingPartialLine, bytesRead } = await consumeCompleteJsonlLines({
|
||||
path,
|
||||
start,
|
||||
onLine: applyLine
|
||||
})
|
||||
// EOF JSON remains readable; a tail forces the next growth to reread rather than retain it.
|
||||
if (trailingPartialLine) {
|
||||
applyLine(trailingPartialLine)
|
||||
}
|
||||
const firstRecord = append
|
||||
? previous!.firstRecord
|
||||
: consumedThrough > 0
|
||||
? await firstRecordSignature(path, consumedThrough)
|
||||
: null
|
||||
const values = append ? previous!.values : new Map<string, string>()
|
||||
const changedKeys = new Set<string>()
|
||||
for (const [key, value] of updates) {
|
||||
if (previous?.values.get(key) !== value) {
|
||||
changedKeys.add(key)
|
||||
}
|
||||
values.set(key, value)
|
||||
}
|
||||
if (!append) {
|
||||
for (const key of previous?.values.keys() ?? []) {
|
||||
if (!values.has(key)) {
|
||||
changedKeys.add(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
values,
|
||||
changedKeys,
|
||||
offset: consumedThrough,
|
||||
readThrough: start + bytesRead,
|
||||
dev: stat.dev,
|
||||
ino: stat.ino,
|
||||
firstRecord
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
import { isMissingTranscriptPathError } from './session-file-observation'
|
||||
import { readSessionIndexJsonl, type SessionIndexJsonlRead } from './session-index-jsonl-read'
|
||||
|
||||
/** Each owner keeps its own accepted cursor, independent of foreground History cache reads. */
|
||||
export function createSessionJsonlDependency(
|
||||
path: string,
|
||||
parse: (line: string) => readonly [string, string] | null
|
||||
): (force: boolean) => Promise<ReadonlySet<string>> {
|
||||
let accepted: SessionIndexJsonlRead | undefined
|
||||
let signature: string | undefined
|
||||
return async (force) => {
|
||||
let stat
|
||||
try {
|
||||
stat = await wslGatedStat(path, 'scan')
|
||||
} catch (error) {
|
||||
if (!isMissingTranscriptPathError(error)) {
|
||||
throw error
|
||||
}
|
||||
const removed = new Set(accepted?.values.keys())
|
||||
accepted = undefined
|
||||
signature = undefined
|
||||
return removed
|
||||
}
|
||||
const nextSignature = `${stat.dev}:${stat.ino}:${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}`
|
||||
if (!force && signature === nextSignature) {
|
||||
return new Set()
|
||||
}
|
||||
// Commit the cursor only after a complete successful read; refusal leaves the input owed.
|
||||
const next = await readSessionIndexJsonl(path, stat, accepted, parse)
|
||||
accepted = next
|
||||
signature = nextSignature
|
||||
return next.changedKeys
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,7 @@ import {
|
||||
kimiSearchDependencies,
|
||||
resolveKimiSessionsDir
|
||||
} from './scanners/kimi/session-scanner-kimi-paths'
|
||||
import {
|
||||
devinSearchDependencies,
|
||||
devinSessionsDbDependencyPath
|
||||
} from './scanners/devin/session-scanner-devin-db'
|
||||
import * as devin from './scanners/devin/session-scanner-devin-db'
|
||||
import {
|
||||
cursorSearchDependencies,
|
||||
cursorChatMetaPath
|
||||
@@ -78,7 +75,6 @@ const DROID_PROJECTS_DIR = join(homedir(), '.factory', 'projects')
|
||||
const CLINE_SESSIONS_DIR =
|
||||
process.env.CLINE_SESSION_DATA_DIR?.trim() || join(homedir(), '.cline', 'data', 'sessions')
|
||||
|
||||
// OpenCode and Antigravity use dedicated discovery for their storage layouts.
|
||||
export const AI_VAULT_AGENT_SOURCES: Record<AiVaultDeletableAgent, AiVaultAgentSource> &
|
||||
Partial<Record<AiVaultAgent, AiVaultAgentSource>> = {
|
||||
claude: {
|
||||
@@ -175,8 +171,7 @@ export const AI_VAULT_AGENT_SOURCES: Record<AiVaultDeletableAgent, AiVaultAgentS
|
||||
mergeRootDiscoveries: true,
|
||||
extensions: ['.json'],
|
||||
// Database-only title or visibility changes must refresh sessions without rereading transcripts.
|
||||
contentDependencyPath: devinSessionsDbDependencyPath,
|
||||
searchDependencies: devinSearchDependencies
|
||||
contentDependencyPath: devin.devinSessionsDbDependencyPath
|
||||
},
|
||||
hermes: {
|
||||
rootDirs: (options, wslHomeDirs) =>
|
||||
@@ -316,3 +311,5 @@ export function sessionRowRootDirs(
|
||||
}
|
||||
|
||||
export { isDiscoverableSessionFile } from './session-file-discovery'
|
||||
|
||||
export type { AiVaultAgentSource } from './session-scanner-types'
|
||||
|
||||
Reference in New Issue
Block a user