mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix: retain session changes until their handoffs complete
Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
This commit is contained in:
co-authored by
Jinjing
parent
836ea57135
commit
fa7d716ca3
@@ -64,6 +64,9 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli
|
||||
}
|
||||
return unowned
|
||||
}
|
||||
async changedForRoot(root: string, agent: string) {
|
||||
return (await this.request!({ kind: 'changes', root, agent })).rows
|
||||
}
|
||||
async admitChanges(
|
||||
receipts: { key: string; revision: number }[],
|
||||
recent: boolean,
|
||||
|
||||
@@ -40,7 +40,7 @@ export type SessionSearchCandidateCommand =
|
||||
| { kind: 'unowned-page'; rows: { path: string; agent?: string | null }[] }
|
||||
| { kind: 'path'; path: string; agent?: string | null }
|
||||
| { kind: 'group'; group: string; after?: string }
|
||||
| { kind: 'changes'; after?: string }
|
||||
| { kind: 'changes'; after?: string; root?: string; agent?: string }
|
||||
| { kind: 'rerank'; root: string; agent: string; order: [string, number][]; after?: string }
|
||||
| { kind: 'consumed'; key: string; revision: number }
|
||||
| SessionSearchPublicationCommand
|
||||
|
||||
@@ -43,6 +43,7 @@ export class SessionSearchCatalogCandidates {
|
||||
CREATE INDEX temp.search_candidate_rank ON search_candidates(group_key,sqlite_source,mtime DESC,source_order,key);
|
||||
CREATE INDEX temp.search_candidate_changes ON search_candidates(changed,mtime DESC,key);
|
||||
CREATE INDEX temp.search_candidate_source ON search_candidates(root,agent,group_key);
|
||||
CREATE INDEX temp.search_candidate_seed ON search_candidates(root,agent,changed,mtime DESC,key);
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -98,9 +99,11 @@ export class SessionSearchCatalogCandidates {
|
||||
])
|
||||
)
|
||||
case 'changes': {
|
||||
const rows = this.read('WHERE changed = 1 AND key > ? ORDER BY mtime DESC,key LIMIT 256', [
|
||||
command.after ?? ''
|
||||
])
|
||||
const scoped = command.root !== undefined && command.agent !== undefined
|
||||
const rows = this.read(
|
||||
`WHERE changed = 1 AND key > ?${scoped ? ' AND root=? AND agent=?' : ''} ORDER BY mtime DESC,key LIMIT 256`,
|
||||
scoped ? [command.after ?? '', command.root!, command.agent!] : [command.after ?? '']
|
||||
)
|
||||
return this.result(rows)
|
||||
}
|
||||
case 'rerank': {
|
||||
@@ -156,7 +159,7 @@ export class SessionSearchCatalogCandidates {
|
||||
`UPDATE temp.search_candidates SET required = ${command.metadataOnly ? 'force_read' : '0'},force_read=${command.metadataOnly ? 'force_read' : '0'},metadata=0 WHERE path = ? AND agent = ? AND selected = 1`
|
||||
)
|
||||
.run(command.path, command.agent)
|
||||
this.dirtyPath(command.path)
|
||||
this.rowChanged(command.path)
|
||||
}
|
||||
return this.result([])
|
||||
}
|
||||
@@ -196,7 +199,7 @@ export class SessionSearchCatalogCandidates {
|
||||
return this.read('WHERE key=?', [key])[0]
|
||||
}
|
||||
rowChanged(path: string): void {
|
||||
this.dirtyPath(path)
|
||||
this.publications.rowChanged(path)
|
||||
}
|
||||
|
||||
private put(row: SessionSearchCatalogCandidate): void {
|
||||
@@ -271,14 +274,10 @@ export class SessionSearchCatalogCandidates {
|
||||
.run(++this.revision, path)
|
||||
if (Number(this.db.prepare('SELECT changes() AS n').get()?.n ?? 0)) {
|
||||
this.pending += previouslyClean
|
||||
this.dirtyPath(path)
|
||||
this.rowChanged(path)
|
||||
}
|
||||
}
|
||||
|
||||
private dirtyPath(path: string): void {
|
||||
this.publications.rowChanged(path)
|
||||
}
|
||||
|
||||
private read(where: string, values: (string | number)[]): SessionSearchCandidateRow[] {
|
||||
return boundedSessionSearchPage(
|
||||
this.db.prepare(`SELECT * FROM temp.search_candidates ${where}`).iterate(...values),
|
||||
|
||||
@@ -42,7 +42,19 @@ export function transitionSessionSearchCatalog(
|
||||
return transitionSessionSearchContainer(catalog, command)
|
||||
case 'input-invalidate':
|
||||
case 'dirty-page':
|
||||
return transitionSessionSearchDirty(catalog, command)
|
||||
case 'dirty-ack': {
|
||||
catalog.connection.exec('SAVEPOINT search_dirty_transition')
|
||||
try {
|
||||
const result = transitionSessionSearchDirty(catalog, command)
|
||||
catalog.connection.exec('RELEASE search_dirty_transition')
|
||||
return result
|
||||
} catch (error) {
|
||||
catalog.connection.exec(
|
||||
'ROLLBACK TO search_dirty_transition; RELEASE search_dirty_transition'
|
||||
)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
case 'dependency-remove':
|
||||
case 'dependency-deliver':
|
||||
case 'dependency-fanout':
|
||||
|
||||
@@ -27,7 +27,8 @@ export type SessionSearchCatalogRow = {
|
||||
ready: number
|
||||
tie?: number
|
||||
}
|
||||
export type SessionSearchCatalogCommand =
|
||||
export type SessionSearchInputReceipt = { consumer: string; revision: number }
|
||||
export type SessionSearchCatalogCommand = (
|
||||
| SessionSearchRetryCommand
|
||||
| SessionSearchAuthorityCommand
|
||||
| SessionSearchDirtyCommand
|
||||
@@ -63,6 +64,7 @@ export type SessionSearchCatalogCommand =
|
||||
limit?: number
|
||||
prefix?: string
|
||||
}
|
||||
) & { receipt?: SessionSearchInputReceipt }
|
||||
export type SessionSearchCatalogResult = SessionSearchDirtyResult &
|
||||
SessionSearchInventoryResult &
|
||||
SessionSearchDirectoryResult & {
|
||||
|
||||
@@ -32,7 +32,45 @@ export class SessionSearchCatalog extends SessionSearchCatalogStorage {
|
||||
|
||||
execute(command: SessionSearchCatalogCommand): SessionSearchCatalogResult {
|
||||
this.touched.add(command.scope)
|
||||
return this.apply(command)
|
||||
if (!command.receipt) {
|
||||
return this.apply(command)
|
||||
}
|
||||
const scope = `${command.scope}:input-receipts`
|
||||
const previous = this.execute({ kind: 'get', scope, key: command.receipt.consumer }).rows?.[0]
|
||||
if (previous && previous.rank >= command.receipt.revision) {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only this receipt owner encodes the accepted response.
|
||||
const accepted = JSON.parse(previous.value) as {
|
||||
result: SessionSearchCatalogResult
|
||||
scopes: string[]
|
||||
}
|
||||
for (const touchedScope of accepted.scopes) {
|
||||
this.touched.add(touchedScope)
|
||||
}
|
||||
return accepted.result
|
||||
}
|
||||
this.db.exec('SAVEPOINT search_input_receipt')
|
||||
try {
|
||||
const result = this.apply({ ...command, receipt: undefined })
|
||||
this.execute({
|
||||
kind: 'put',
|
||||
scope,
|
||||
rows: [
|
||||
{
|
||||
key: command.receipt.consumer,
|
||||
value: JSON.stringify({ result, scopes: [...this.touched] }),
|
||||
parent: '',
|
||||
group: '',
|
||||
rank: command.receipt.revision,
|
||||
ready: 0
|
||||
}
|
||||
]
|
||||
})
|
||||
this.db.exec('RELEASE search_input_receipt')
|
||||
return result
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK TO search_input_receipt; RELEASE search_input_receipt')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
response(command: SessionSearchCatalogCommand): SessionSearchCatalogResult {
|
||||
const records =
|
||||
|
||||
@@ -21,6 +21,11 @@ export class SessionSearchDependencyInventory {
|
||||
private readonly observe: SessionDiscoveryDependencyObserver | undefined
|
||||
private readonly retries = new SessionObservationRetries()
|
||||
private readonly delivered: SessionSearchCatalogClient<{ after?: string }>
|
||||
private staging:
|
||||
| { iterator?: AsyncGenerator<string>; keys: string[]; serial: number; done: boolean }
|
||||
| undefined
|
||||
private stagingSequence = 0
|
||||
private inventoryAck: { path: string; revision: number }[] | undefined
|
||||
private deliveryAfter: string | undefined
|
||||
private discardAfter: string | undefined
|
||||
private readonly keys: SessionSearchCatalogClient<string>
|
||||
@@ -40,6 +45,8 @@ export class SessionSearchDependencyInventory {
|
||||
}
|
||||
get pending(): boolean {
|
||||
return (
|
||||
!!this.staging ||
|
||||
!!this.inventoryAck ||
|
||||
this.retries.pending ||
|
||||
this.delivered.size > 0 ||
|
||||
this.observe?.pending === true ||
|
||||
@@ -88,19 +95,46 @@ export class SessionSearchDependencyInventory {
|
||||
args: DiscoverSessionFilesArgs,
|
||||
overdue: () => boolean
|
||||
): Promise<void> {
|
||||
let keys: string[] = []
|
||||
const flush = async () => {
|
||||
await this.delivered.putPage(keys.map((key) => this.delivered.encode(key, {})))
|
||||
keys = []
|
||||
this.staging ??= {
|
||||
iterator: this.changedKeys(watch, args, overdue),
|
||||
keys: [],
|
||||
serial: ++this.stagingSequence,
|
||||
done: false
|
||||
}
|
||||
for await (const key of this.changedKeys(watch, args, overdue)) {
|
||||
keys.push(key)
|
||||
if (keys.length === 256) {
|
||||
await flush()
|
||||
const pending = this.staging
|
||||
while (!pending.done || pending.keys.length) {
|
||||
if (
|
||||
pending.keys.length &&
|
||||
(pending.done || pending.keys.length === 256 || !pending.iterator)
|
||||
) {
|
||||
await this.delivered.transition({
|
||||
kind: 'dependency-deliver',
|
||||
scope: this.scope,
|
||||
keys: pending.keys,
|
||||
receipt: { consumer: 'native-dependency', revision: pending.serial }
|
||||
})
|
||||
pending.keys = []
|
||||
pending.serial = ++this.stagingSequence
|
||||
}
|
||||
if (pending.done) {
|
||||
break
|
||||
}
|
||||
pending.iterator ??= this.changedKeys(watch, args, overdue)
|
||||
try {
|
||||
const next = await pending.iterator.next()
|
||||
if (next.done) {
|
||||
pending.done = true
|
||||
} else {
|
||||
pending.keys.push(next.value)
|
||||
}
|
||||
} catch (error) {
|
||||
pending.iterator = undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
await flush()
|
||||
this.staging = undefined
|
||||
}
|
||||
|
||||
async addFile(path: string): Promise<void> {
|
||||
await this.addFiles([path])
|
||||
}
|
||||
@@ -130,6 +164,10 @@ export class SessionSearchDependencyInventory {
|
||||
args: DiscoverSessionFilesArgs,
|
||||
overdue: () => boolean
|
||||
): AsyncGenerator<string> {
|
||||
if (this.inventoryAck) {
|
||||
await this.inventory!.acknowledge(this.inventoryAck)
|
||||
this.inventoryAck = undefined
|
||||
}
|
||||
if (this.dependency.eventsOnly) {
|
||||
const keys = new Set<string>()
|
||||
for (const event of watch?.events.values() ?? []) {
|
||||
@@ -176,10 +214,14 @@ export class SessionSearchDependencyInventory {
|
||||
const keys = new Set<string>()
|
||||
const inventory = this.inventory!
|
||||
if ((!watch || watch.failed || watch.reconcile) && !inventory.walking) {
|
||||
await inventory.reconcile()
|
||||
await inventory.reconcile(watch?.receipt('reconcile'))
|
||||
if (watch) {
|
||||
watch.reconcile = false
|
||||
}
|
||||
}
|
||||
for (const event of watch?.events.values() ?? []) {
|
||||
await inventory.invalidate(event.path)
|
||||
await inventory.invalidate(event.path, true, true, watch?.eventReceipt(event.path))
|
||||
watch?.acknowledgeEvent(event.path)
|
||||
}
|
||||
await inventory.retryFailures()
|
||||
const delta = await inventory.drain(
|
||||
@@ -202,7 +244,9 @@ export class SessionSearchDependencyInventory {
|
||||
keys.add(key)
|
||||
}
|
||||
}
|
||||
this.inventoryAck = delta.receipts
|
||||
yield* keys
|
||||
await inventory.acknowledge(delta.receipts)
|
||||
this.inventoryAck = undefined
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { SessionSearchAuthorityPage } from './session-search-authority-page'
|
||||
import { wakeSessionSearchPrimaryFailures } from './session-search-primary-retries'
|
||||
import { recordSessionSearchInput } from './session-search-observation-authority'
|
||||
import type { SessionSearchCatalog, SessionSearchCatalogRow } from './session-search-catalog'
|
||||
@@ -5,8 +6,11 @@ import type { SessionSearchObservationDebt } from './session-search-inventory-tr
|
||||
|
||||
export type SessionSearchDirtyCommand =
|
||||
| { kind: 'dirty-page'; scope: string }
|
||||
| { kind: 'dirty-ack'; scope: string; path: string; revision: number }
|
||||
| { kind: 'input-invalidate'; scope: string; path: string; force: boolean; subtree?: boolean }
|
||||
export type SessionSearchDirtyResult = { dirtyInputs?: { path: string; force: boolean }[] }
|
||||
export type SessionSearchDirtyResult = {
|
||||
dirtyInputs?: { path: string; force: boolean; dirtyRevision: number }[]
|
||||
}
|
||||
type Entry = { value: boolean; revision: number }
|
||||
function entry(row: SessionSearchCatalogRow | undefined): Entry | undefined {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the matching frontier owns its encoded value and revision.
|
||||
@@ -80,10 +84,17 @@ export function transitionSessionSearchDirty(
|
||||
)
|
||||
return {}
|
||||
}
|
||||
if (command.kind === 'dirty-ack') {
|
||||
acknowledgeSessionSearchDirty(catalog, command.scope, command.path, command.revision)
|
||||
const authority = new SessionSearchAuthorityPage(catalog, command.scope)
|
||||
authority.settle(command.path)
|
||||
authority.flush()
|
||||
return {}
|
||||
}
|
||||
const scope = `${command.scope}:dirty`,
|
||||
expansion = `${command.scope}:dirty-expansion`
|
||||
const page = catalog.execute({ kind: 'page', scope, group: '', orderRank: true })
|
||||
const dirtyInputs: { path: string; force: boolean }[] = []
|
||||
const dirtyInputs: { path: string; force: boolean; dirtyRevision: number }[] = []
|
||||
let remaining = 256
|
||||
for (const row of page.rows ?? []) {
|
||||
const fact = entry(row)!
|
||||
@@ -94,9 +105,17 @@ export function transitionSessionSearchDirty(
|
||||
continue
|
||||
}
|
||||
const stored = catalog.execute({ kind: 'get', scope: expansion, key: row.key }).rows?.[0]
|
||||
const cursor: { revision: number; after?: string } | undefined = stored
|
||||
const cursor: { revision: number; after?: string; complete?: boolean } | undefined = stored
|
||||
? JSON.parse(stored.value)
|
||||
: undefined
|
||||
if (cursor?.revision === fact.revision && cursor.complete) {
|
||||
dirtyInputs.push({ path: row.key, force: fact.value, dirtyRevision: fact.revision })
|
||||
remaining--
|
||||
if (!remaining) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
const owners = catalog.execute({
|
||||
kind: 'page',
|
||||
scope: `${command.scope}:files`,
|
||||
@@ -127,9 +146,13 @@ export function transitionSessionSearchDirty(
|
||||
entry(catalog.execute({ kind: 'get', scope, key: row.key }).rows?.[0])?.revision ===
|
||||
fact.revision
|
||||
) {
|
||||
catalog.execute({ kind: 'delete', scope: expansion, key: row.key })
|
||||
catalog.execute({ kind: 'delete', scope, key: row.key })
|
||||
dirtyInputs.push({ path: row.key, force: fact.value })
|
||||
catalog.execute({
|
||||
kind: 'put',
|
||||
scope: expansion,
|
||||
rows: [{ ...row, value: JSON.stringify({ revision: fact.revision, complete: true }) }]
|
||||
})
|
||||
dirtyInputs.push({ path: row.key, force: fact.value, dirtyRevision: fact.revision })
|
||||
remaining--
|
||||
}
|
||||
if (!remaining) {
|
||||
break
|
||||
@@ -137,3 +160,18 @@ export function transitionSessionSearchDirty(
|
||||
}
|
||||
return { dirtyInputs }
|
||||
}
|
||||
|
||||
export function acknowledgeSessionSearchDirty(
|
||||
catalog: SessionSearchCatalog,
|
||||
owner: string,
|
||||
path: string,
|
||||
revision: number
|
||||
): void {
|
||||
const scope = `${owner}:dirty`
|
||||
const current = entry(catalog.execute({ kind: 'get', scope, key: path }).rows?.[0])
|
||||
if (current?.revision !== revision) {
|
||||
return
|
||||
}
|
||||
catalog.execute({ kind: 'delete', scope, key: path })
|
||||
catalog.execute({ kind: 'delete', scope: `${owner}:dirty-expansion`, key: path })
|
||||
}
|
||||
|
||||
@@ -11,7 +11,10 @@ export class SessionSearchDiscoveryInventory
|
||||
extends SessionSearchDiscoverySources
|
||||
implements SessionDiscoveryContext
|
||||
{
|
||||
async files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery> {
|
||||
files(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery> {
|
||||
return this.track(this.discoverFiles(args))
|
||||
}
|
||||
private async discoverFiles(args: DiscoverSessionFilesArgs): Promise<SessionFileDiscovery> {
|
||||
const key = `${args.agent}:${args.rootDir}`
|
||||
this.seen.add(key)
|
||||
let root = this.roots.get(key)
|
||||
@@ -33,6 +36,18 @@ export class SessionSearchDiscoveryInventory
|
||||
await root.inventory.wakeFailures()
|
||||
root.wake = this.primaryWake
|
||||
}
|
||||
const admitPrimary = async () => {
|
||||
if (
|
||||
root.inventory.primarySeedWaiting &&
|
||||
root.inventory.seedDelivered &&
|
||||
(!this.admitSeed || (await this.admitSeed(args.rootDir, args.agent)))
|
||||
) {
|
||||
await root.inventory.retryFailures()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
await admitPrimary()
|
||||
let watching = false
|
||||
if (this.maintenance) {
|
||||
root.fallback = await this.watchers.requiresPolling(args.rootDir)
|
||||
@@ -42,7 +57,12 @@ export class SessionSearchDiscoveryInventory
|
||||
await root.inventory.reconcile()
|
||||
}
|
||||
} else {
|
||||
const watch = await this.watchers.watch(args.rootDir, args.signal)
|
||||
const watch = await this.watchers.watch(
|
||||
args.rootDir,
|
||||
args.signal,
|
||||
'recursive',
|
||||
root.inventory.scope
|
||||
)
|
||||
if (!watch) {
|
||||
root.unwatched = true
|
||||
if (!root.inventory.walking) {
|
||||
@@ -54,7 +74,8 @@ export class SessionSearchDiscoveryInventory
|
||||
(this.force || watch.reconcile || root.identity !== watch.identity) &&
|
||||
(root.identity !== undefined || !root.inventory.walking)
|
||||
) {
|
||||
await root.inventory.reconcile()
|
||||
await root.inventory.reconcile(watch.receipt('reconcile'))
|
||||
watch.reconcile = false
|
||||
}
|
||||
if (
|
||||
root.identity !== watch.identity &&
|
||||
@@ -72,13 +93,16 @@ export class SessionSearchDiscoveryInventory
|
||||
root.unwatched ||
|
||||
(root.identity !== undefined && root.identity !== watch.identity))
|
||||
) {
|
||||
await root.inventory.wakeFailures()
|
||||
await root.inventory.wakeFailures(watch.receipt('recovery'))
|
||||
watch.recovery = false
|
||||
}
|
||||
root.unwatched = watch.failed
|
||||
root.identity = watch.identity
|
||||
for (const event of watch.events.values()) {
|
||||
await root.inventory.invalidate(event.path)
|
||||
await root.inventory.invalidate(event.path, true, true, watch.eventReceipt(event.path))
|
||||
watch.acknowledgeEvent(event.path)
|
||||
}
|
||||
watch.acknowledge()
|
||||
}
|
||||
}
|
||||
const auxiliaries = await Promise.all(
|
||||
@@ -101,7 +125,12 @@ export class SessionSearchDiscoveryInventory
|
||||
observer,
|
||||
watch: observer.polling
|
||||
? undefined
|
||||
: await this.watchers.watch(dependency.path, args.signal, dependency.mode)
|
||||
: await this.watchers.watch(
|
||||
dependency.path,
|
||||
args.signal,
|
||||
dependency.mode,
|
||||
`dependency:${key}:${dependency.path}`
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
@@ -111,9 +140,10 @@ export class SessionSearchDiscoveryInventory
|
||||
args
|
||||
)
|
||||
await observer.stageChanges(auxiliary, args, this.overdue)
|
||||
auxiliary?.acknowledge()
|
||||
await observer.settleDelivered(root.inventory.scope, this.overdue)
|
||||
}
|
||||
await root.inventory.retryFailures()
|
||||
await root.inventory.retryFailures(false)
|
||||
if (root.fallback || !watching) {
|
||||
await root.inventory.pollRecent(args, this.recentLimit)
|
||||
}
|
||||
@@ -173,7 +203,8 @@ export class SessionSearchDiscoveryInventory
|
||||
}
|
||||
}
|
||||
await root.inventory.acknowledge(delta.receipts)
|
||||
if ((!delta.changes.length && !delta.removed.length) || this.overdue()) {
|
||||
const admittedPrimary = await admitPrimary()
|
||||
if ((!delta.changes.length && !delta.removed.length && !admittedPrimary) || this.overdue()) {
|
||||
break
|
||||
}
|
||||
} while (root.inventory.pending)
|
||||
@@ -185,10 +216,20 @@ export class SessionSearchDiscoveryInventory
|
||||
read: (issues: AiVaultScanIssue[]) => Promise<readonly string[]>,
|
||||
issues: AiVaultScanIssue[]
|
||||
) {
|
||||
return this.containers.containerPaths(root, read, issues)
|
||||
return this.track(this.containers.containerPaths(root, read, issues))
|
||||
}
|
||||
|
||||
async container(
|
||||
container(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
limit: number,
|
||||
issues: readonly AiVaultScanIssue[],
|
||||
enumerationComplete = true
|
||||
): Promise<SessionFileDiscovery> {
|
||||
return this.track(this.discoverContainer(key, paths, read, limit, issues, enumerationComplete))
|
||||
}
|
||||
private async discoverContainer(
|
||||
key: string,
|
||||
paths: readonly string[],
|
||||
read: (paths: readonly string[], limit: number) => Promise<SessionFileDiscovery>,
|
||||
|
||||
@@ -24,6 +24,20 @@ type Root = {
|
||||
|
||||
/** Search owns its notifications and raw membership; ordinary History discovery stays stateless. */
|
||||
export class SessionSearchDiscoverySources {
|
||||
private readonly active = new Set<Promise<unknown>>()
|
||||
protected track<T>(work: Promise<T>): Promise<T> {
|
||||
this.active.add(work)
|
||||
void work.then(
|
||||
() => this.active.delete(work),
|
||||
() => this.active.delete(work)
|
||||
)
|
||||
return work
|
||||
}
|
||||
async settled(): Promise<void> {
|
||||
while (this.active.size) {
|
||||
await Promise.allSettled(this.active)
|
||||
}
|
||||
}
|
||||
protected readonly roots = new Map<string, Root>()
|
||||
readonly containers = new SessionSearchContainerInventory()
|
||||
protected readonly owner = new AbortController()
|
||||
@@ -39,6 +53,10 @@ export class SessionSearchDiscoverySources {
|
||||
onRemoved(accept: NonNullable<SessionSearchDiscoverySources['acceptRemoved']>): void {
|
||||
this.acceptRemoved = accept
|
||||
}
|
||||
protected admitSeed: ((root: string, agent: string) => Promise<boolean>) | undefined
|
||||
onSeedAdmission(accept: NonNullable<SessionSearchDiscoverySources['admitSeed']>): void {
|
||||
this.admitSeed = accept
|
||||
}
|
||||
protected accept:
|
||||
| ((
|
||||
discovery: SessionFileDiscovery,
|
||||
@@ -91,7 +109,7 @@ export class SessionSearchDiscoverySources {
|
||||
}
|
||||
|
||||
get pending(): boolean {
|
||||
if (this.containers.pending) {
|
||||
if (this.containers.pending || this.watchers.pending) {
|
||||
return true
|
||||
}
|
||||
for (const root of this.roots.values()) {
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import './session-search-watcher.test-fixture'
|
||||
import './session-search-worker.test-fixture'
|
||||
import { join } from 'node:path'
|
||||
import { mkdir } from 'node:fs/promises'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { SessionSearchIndexer } from './session-search-indexer'
|
||||
import { SessionSearchWorkerClient } from './session-search-worker-client'
|
||||
import {
|
||||
notifySessionSearchFileChange,
|
||||
notifySessionSearchOverflow
|
||||
} from './session-search-watcher.test-fixture'
|
||||
import {
|
||||
FakeSessionSearchClock,
|
||||
openSessionSearchIndexerHarness,
|
||||
writeClaudeTranscript
|
||||
} from './session-search-indexer-test-fixture'
|
||||
|
||||
vi.mock('./session-search-filesystem', () => ({ sessionSearchRequiresPolling: async () => false }))
|
||||
|
||||
it.each([
|
||||
'event-before',
|
||||
'event-after',
|
||||
'prepare-before',
|
||||
'prepare-after',
|
||||
'directory-schedule'
|
||||
] as const)(
|
||||
'retains a delivered new conversation across %s rejection without a second notice',
|
||||
async (failure) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-input-handoff')
|
||||
const root = harness.roots.claudeProjectsDir!
|
||||
const directory = join(root, 'new-project')
|
||||
const first = join(harness.claudeProjectDir, 'first.jsonl')
|
||||
const second = join(
|
||||
failure === 'directory-schedule' ? directory : harness.claudeProjectDir,
|
||||
'second.jsonl'
|
||||
)
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const errors: unknown[] = []
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock,
|
||||
onError: (error) => errors.push(error)
|
||||
})
|
||||
const request = SessionSearchWorkerClient.prototype.request
|
||||
let armed = false,
|
||||
rejected = false
|
||||
const gate = vi
|
||||
.spyOn(SessionSearchWorkerClient.prototype, 'request')
|
||||
.mockImplementation(async function (this: SessionSearchWorkerClient, command) {
|
||||
const target =
|
||||
command.op === 'catalog' &&
|
||||
((failure.startsWith('event-') &&
|
||||
command.command.kind === 'input-invalidate' &&
|
||||
command.command.path === second) ||
|
||||
(failure.startsWith('prepare-') &&
|
||||
command.command.kind === 'observe-prepare' &&
|
||||
command.command.inputs.some((input) => input.path === second)) ||
|
||||
(failure === 'directory-schedule' &&
|
||||
command.command.kind === 'directory-schedule' &&
|
||||
command.command.path === directory))
|
||||
if (armed && !rejected && target) {
|
||||
rejected = true
|
||||
if (failure.endsWith('after')) {
|
||||
await request.call(this, command)
|
||||
}
|
||||
throw new Error('fixture catalog handoff rejected')
|
||||
}
|
||||
return request.call(this, command)
|
||||
})
|
||||
const hits = () =>
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare(
|
||||
"SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'handoffneedle'"
|
||||
)
|
||||
.get()
|
||||
)
|
||||
try {
|
||||
await mkdir(harness.claudeProjectDir, { recursive: true })
|
||||
await subject.start()
|
||||
await writeClaudeTranscript(first, ['handoffneedle'], 'handoff-first')
|
||||
await writeClaudeTranscript(second, ['handoffneedle'], 'handoff-second')
|
||||
if (failure === 'directory-schedule') {
|
||||
notifySessionSearchFileChange(directory)
|
||||
}
|
||||
armed = true
|
||||
await subject.reconcile()
|
||||
expect(rejected).toBe(true)
|
||||
expect(errors).toHaveLength(1)
|
||||
// Only resume the failed boundary; no new notification, full refresh or maintenance tick.
|
||||
await subject.reconcile()
|
||||
expect(hits()).toEqual({ n: 4 })
|
||||
expect(subject.status().phase).toBe('current')
|
||||
} finally {
|
||||
gate.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.each(['before', 'after'] as const)(
|
||||
'retains native recovery across an accepted-result %s rejection',
|
||||
async (completion) => {
|
||||
const harness = await openSessionSearchIndexerHarness('ss-native-recovery')
|
||||
const path = join(harness.claudeProjectDir, 'session.jsonl')
|
||||
const clock = new FakeSessionSearchClock()
|
||||
const subject = new SessionSearchIndexer({
|
||||
databasePath: harness.databasePath,
|
||||
roots: harness.roots,
|
||||
historyDays: null,
|
||||
clock
|
||||
})
|
||||
const request = SessionSearchWorkerClient.prototype.request
|
||||
let armed = false,
|
||||
rejected = false
|
||||
const gate = vi
|
||||
.spyOn(SessionSearchWorkerClient.prototype, 'request')
|
||||
.mockImplementation(async function (this: SessionSearchWorkerClient, command) {
|
||||
if (
|
||||
armed &&
|
||||
!rejected &&
|
||||
command.op === 'catalog' &&
|
||||
command.command.kind === 'retry-wake' &&
|
||||
command.command.all &&
|
||||
command.command.receipt?.consumer.endsWith(':recovery')
|
||||
) {
|
||||
rejected = true
|
||||
if (completion === 'after') {
|
||||
await request.call(this, command)
|
||||
}
|
||||
throw new Error('fixture native recovery rejected')
|
||||
}
|
||||
return request.call(this, command)
|
||||
})
|
||||
try {
|
||||
await writeClaudeTranscript(path, ['nativeold'], 'native-recovery')
|
||||
await subject.start()
|
||||
harness.write((db) =>
|
||||
db.exec(
|
||||
"CREATE TRIGGER refuse_native BEFORE UPDATE ON sessions BEGIN SELECT RAISE(ABORT,'fixture refused content'); END"
|
||||
)
|
||||
)
|
||||
await writeClaudeTranscript(path, ['nativenew'], 'native-recovery')
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
clock.advance(20_000)
|
||||
await subject.settled()
|
||||
}
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db.prepare('SELECT state,fail_count FROM files WHERE path=?').get(path)
|
||||
)
|
||||
).toEqual({ state: 'failed', fail_count: 3 })
|
||||
harness.write((db) => db.exec('DROP TRIGGER refuse_native'))
|
||||
armed = true
|
||||
expect(notifySessionSearchOverflow(harness.roots.claudeProjectsDir!)).toBe(1)
|
||||
await subject.reconcile()
|
||||
expect(rejected).toBe(true)
|
||||
await subject.reconcile()
|
||||
expect(
|
||||
harness.read((db) =>
|
||||
db
|
||||
.prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'nativenew'")
|
||||
.get()
|
||||
)
|
||||
).toEqual({ n: 2 })
|
||||
expect(subject.status().phase).toBe('current')
|
||||
} finally {
|
||||
gate.mockRestore()
|
||||
await subject.close()
|
||||
await harness.cleanup()
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -1,3 +1,4 @@
|
||||
import { acknowledgeSessionSearchDirty } from './session-search-dirty-transition'
|
||||
import { SessionSearchAuthorityPage } from './session-search-authority-page'
|
||||
import { SESSION_INPUT_FAILURES_BEFORE_HELD_OUT } from '../ai-vault/session-observation-retries'
|
||||
import { recordSessionSearchRetry } from './session-search-retry-transition'
|
||||
@@ -137,6 +138,10 @@ export function transitionSessionSearchInventory(
|
||||
continue
|
||||
}
|
||||
remove(observing, outcome.path)
|
||||
if (outcome.dirtyRevision !== undefined) {
|
||||
acknowledgeSessionSearchDirty(catalog, command.scope, outcome.path, outcome.dirtyRevision)
|
||||
authority.forget(`${command.scope}:dirty`, outcome.path)
|
||||
}
|
||||
const accepted = decode<SessionSearchObservedFacts>(current)
|
||||
const previous = decode<SessionSearchObservationDebt>(get(failures, outcome.path))
|
||||
if (previous?.attempted === command.generation) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import './session-search-watcher.test-fixture'
|
||||
import './session-search-worker.test-fixture'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
@@ -120,7 +121,7 @@ it.each(['file', 'subtree'] as const)(
|
||||
await rm(child, { recursive: true })
|
||||
}
|
||||
paused = false
|
||||
notifySessionSearchOverflow(harness.roots.claudeProjectsDir!)
|
||||
expect(notifySessionSearchOverflow(harness.roots.claudeProjectsDir!)).toBe(1)
|
||||
await subject.reconcile()
|
||||
expect(paused).toBe(true)
|
||||
refusing = variant === 'subtree'
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { acknowledgeSessionSearchDirty } from './session-search-dirty-transition'
|
||||
import { dirname } from 'node:path'
|
||||
import { SessionSearchAuthorityPage } from './session-search-authority-page'
|
||||
import { boundedSessionSearchPage } from './session-search-catalog-pages'
|
||||
@@ -37,6 +38,7 @@ export function prepareSessionSearchObservations(
|
||||
failures,
|
||||
`${failures}:eligible`,
|
||||
`${command.scope}:dirty`,
|
||||
`${command.scope}:dirty-expansion`,
|
||||
`${command.scope}:authority`,
|
||||
`${command.scope}:authority-root`,
|
||||
observing,
|
||||
@@ -52,8 +54,23 @@ export function prepareSessionSearchObservations(
|
||||
(input) => {
|
||||
const refusal = get(failures, input.path)
|
||||
const debt = authorityValue<SessionSearchObservationDebt>(refusal)
|
||||
const revision = catalog.nextRevision()
|
||||
const attempt = !debt || (debt.failures < 3 && debt.attempted !== command.generation)
|
||||
const dirty = get(`${command.scope}:dirty`, input.path)
|
||||
const stale =
|
||||
input.dirtyRevision !== undefined &&
|
||||
authorityValue<{ revision: number }>(dirty)?.revision !== input.dirtyRevision
|
||||
const previousToken = authorityValue<string>(get(observing, input.path))
|
||||
const previous: [string, string, number] | undefined = previousToken
|
||||
? JSON.parse(previousToken)
|
||||
: undefined
|
||||
const reuse =
|
||||
input.dirtyRevision !== undefined &&
|
||||
previous &&
|
||||
previous[0] === (get(files, input.path)?.value ?? '') &&
|
||||
previous[1] === (dirty?.value ?? '') &&
|
||||
authority.authority(input.path)?.self === previous[2]
|
||||
const revision = reuse ? previous[2] : catalog.nextRevision()
|
||||
const attempt =
|
||||
!stale && (!debt || (debt.failures < 3 && debt.attempted !== command.generation))
|
||||
const mergesDebt =
|
||||
!attempt &&
|
||||
debt &&
|
||||
@@ -99,6 +116,11 @@ export function prepareSessionSearchObservations(
|
||||
)
|
||||
authority.forget(failures, input.path)
|
||||
}
|
||||
if (!input.attempt && input.dirtyRevision !== undefined) {
|
||||
acknowledgeSessionSearchDirty(catalog, command.scope, input.path, input.dirtyRevision)
|
||||
authority.forget(`${command.scope}:dirty`, input.path)
|
||||
authority.settle(input.path)
|
||||
}
|
||||
if (input.attempt) {
|
||||
authority.record(input.path, input.authority, 0)
|
||||
put(observing, input.path, input.token)
|
||||
|
||||
@@ -17,6 +17,7 @@ export type SessionSearchObservationInput = {
|
||||
force: boolean
|
||||
readOwed?: boolean
|
||||
metadataOwed?: boolean
|
||||
dirtyRevision?: number
|
||||
}
|
||||
export type SessionSearchPreparedObservation = SessionSearchObservationInput & {
|
||||
accepted?: SessionSearchObservedFacts
|
||||
|
||||
@@ -116,11 +116,30 @@ export async function runSessionSearchPass(
|
||||
} while (state.candidates.pendingChanges)
|
||||
}
|
||||
let discoveryLane = Promise.resolve()
|
||||
const serializeDiscovery = (consume: () => Promise<void>): Promise<void> => {
|
||||
const serializeDiscovery = <T>(consume: () => Promise<T>): Promise<T> => {
|
||||
const accepted = discoveryLane.then(consume)
|
||||
discoveryLane = accepted
|
||||
discoveryLane = accepted.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
)
|
||||
return accepted
|
||||
}
|
||||
state.discovery.onSeedAdmission((root, agent) =>
|
||||
serializeDiscovery(async () => {
|
||||
const rows = await state.candidates.changedForRoot(root, agent)
|
||||
if (rows.length) {
|
||||
await state.parkedRetire.deletePage(rows.map((row) => row.candidate.file.path))
|
||||
state.queue.update(
|
||||
await state.candidates.admitChanges(
|
||||
rows.map(({ key, revision }) => ({ key, revision })),
|
||||
!args.full,
|
||||
state.recovering
|
||||
)
|
||||
)
|
||||
}
|
||||
return !(await state.candidates.changedForRoot(root, agent)).length
|
||||
})
|
||||
)
|
||||
state.discovery.onRemoved((paths, observations) =>
|
||||
serializeDiscovery(async () => {
|
||||
if (!paths.length) {
|
||||
@@ -139,7 +158,7 @@ export async function runSessionSearchPass(
|
||||
)
|
||||
const swept = await withSessionDiscovery(state.discovery, () =>
|
||||
discoverSessionSearchCandidates(args.roots, { limitPerAgent: Infinity, signal, raw: true })
|
||||
)
|
||||
).finally(() => state.discovery.settled())
|
||||
await state.discovery.end()
|
||||
const issues: AiVaultScanIssue[] = [...swept.issues]
|
||||
const update = await state.candidates.update(
|
||||
|
||||
@@ -33,8 +33,8 @@ export type SessionSearchRetryCommand =
|
||||
}
|
||||
| { kind: 'retry-remove'; scope: string; path: string }
|
||||
| { kind: 'retry-require'; scope: string; path: string }
|
||||
| { kind: 'retry-wake'; scope: string; prefix?: string }
|
||||
| { kind: 'retry-admit'; scope: string; root: string; generation: number }
|
||||
| { kind: 'retry-wake'; scope: string; prefix?: string; all?: boolean }
|
||||
| { kind: 'retry-admit'; scope: string; root: string; generation: number; primary?: boolean }
|
||||
|
||||
export function recordSessionSearchRetry(
|
||||
catalog: SessionSearchCatalog,
|
||||
@@ -153,7 +153,9 @@ export function transitionSessionSearchRetry(
|
||||
command: SessionSearchRetryCommand
|
||||
) {
|
||||
if (command.kind === 'retry-admit') {
|
||||
admitSessionSearchPrimaryFailures(catalog, command.scope)
|
||||
if (command.primary !== false) {
|
||||
admitSessionSearchPrimaryFailures(catalog, command.scope)
|
||||
}
|
||||
admit(
|
||||
catalog,
|
||||
command.scope,
|
||||
@@ -173,6 +175,16 @@ export function transitionSessionSearchRetry(
|
||||
return {}
|
||||
}
|
||||
if (command.kind === 'retry-wake') {
|
||||
if (command.all && command.prefix) {
|
||||
for (const suffix of ['primary-wakes', 'refusals', 'file-refusals']) {
|
||||
catalog.execute({
|
||||
kind: 'retry-wake',
|
||||
scope: `${command.scope}:${suffix}`,
|
||||
prefix: command.prefix
|
||||
})
|
||||
}
|
||||
return {}
|
||||
}
|
||||
if (command.scope.endsWith(':primary-wakes') && command.prefix) {
|
||||
wakeSessionSearchPrimaryFailures(
|
||||
catalog,
|
||||
|
||||
@@ -5,7 +5,9 @@ import type { WatcherProcessSubscription } from '../ipc/filesystem/watcher-proce
|
||||
import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol'
|
||||
import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation'
|
||||
import { RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS } from '../../shared/file-watch/runtime-file-watch-limits'
|
||||
export type SessionSearchRootWatch = {
|
||||
import { SessionSearchWatchDelivery } from './session-search-watch-delivery'
|
||||
export type { SessionSearchRootWatch } from './session-search-watch-delivery'
|
||||
type NativeWatch = {
|
||||
identity: string
|
||||
subscription?: WatcherProcessSubscription
|
||||
failed: boolean
|
||||
@@ -17,17 +19,36 @@ export type SessionSearchRootWatch = {
|
||||
}
|
||||
|
||||
export class SessionSearchRootWatchers {
|
||||
private readonly watches = new Map<string, SessionSearchRootWatch>()
|
||||
private readonly watches = new Map<string, NativeWatch>()
|
||||
private readonly owner = new AbortController()
|
||||
private readonly supervisor: WatcherProcessSupervisor
|
||||
private probes = new Map<string, Promise<SessionSearchRootWatch | undefined>>()
|
||||
private sequence = 0
|
||||
private readonly consumers = new Map<string, Map<string, SessionSearchWatchDelivery>>()
|
||||
private seen = new Set<string>()
|
||||
get pending(): boolean {
|
||||
return [...this.consumers.values()].some((consumers) =>
|
||||
[...consumers.values()].some((consumer) => consumer.pending)
|
||||
)
|
||||
}
|
||||
private probes = new Map<string, Promise<NativeWatch | undefined>>()
|
||||
constructor(entryPath?: string) {
|
||||
this.supervisor = new WatcherProcessSupervisor({ entryPath })
|
||||
}
|
||||
begin(): void {
|
||||
this.probes = new Map()
|
||||
this.seen.clear()
|
||||
}
|
||||
async end(): Promise<void> {
|
||||
for (const [key, consumers] of this.consumers) {
|
||||
for (const consumer of consumers.keys()) {
|
||||
if (!this.seen.has(`${key}:${consumer}`)) {
|
||||
consumers.delete(consumer)
|
||||
}
|
||||
}
|
||||
if (!consumers.size) {
|
||||
this.consumers.delete(key)
|
||||
}
|
||||
}
|
||||
for (const [path, watch] of this.watches) {
|
||||
if (!this.probes.has(path)) {
|
||||
this.watches.delete(path)
|
||||
@@ -38,29 +59,49 @@ export class SessionSearchRootWatchers {
|
||||
close(): Promise<void> {
|
||||
this.owner.abort()
|
||||
this.watches.clear()
|
||||
this.consumers.clear()
|
||||
return this.supervisor.disposeAndWait()
|
||||
}
|
||||
watch(
|
||||
async watch(
|
||||
path: string,
|
||||
signal?: AbortSignal,
|
||||
mode: 'recursive' | 'shallow' = 'recursive'
|
||||
): Promise<SessionSearchRootWatch | undefined> {
|
||||
mode: 'recursive' | 'shallow' = 'recursive',
|
||||
consumer = path
|
||||
) {
|
||||
const key = `${mode}:${path}`
|
||||
this.seen.add(`${key}:${consumer}`)
|
||||
let consumers = this.consumers.get(key)
|
||||
if (!consumers) {
|
||||
consumers = new Map()
|
||||
this.consumers.set(key, consumers)
|
||||
}
|
||||
let delivery = consumers.get(consumer)
|
||||
if (!delivery) {
|
||||
delivery = new SessionSearchWatchDelivery(consumer, () => ++this.sequence)
|
||||
consumers.set(consumer, delivery)
|
||||
}
|
||||
let probe = this.probes.get(key)
|
||||
if (!probe) {
|
||||
probe = this.probeWatch(path, key, mode, signal).then((watch) => {
|
||||
if (!watch) {
|
||||
return undefined
|
||||
}
|
||||
const delivery = { ...watch }
|
||||
watch.events = new Map()
|
||||
watch.reconcile = false
|
||||
watch.recovery = false
|
||||
return delivery
|
||||
})
|
||||
probe = this.probeWatch(path, key, mode, signal)
|
||||
this.probes.set(key, probe)
|
||||
}
|
||||
return probe
|
||||
const watch = await probe
|
||||
if (!watch) {
|
||||
if (!delivery.pending) {
|
||||
return undefined
|
||||
}
|
||||
const pending = delivery.capture()
|
||||
pending.failed = true
|
||||
pending.reconcile = true
|
||||
return pending
|
||||
}
|
||||
for (const target of consumers.values()) {
|
||||
target.stage(watch)
|
||||
}
|
||||
watch.events = new Map()
|
||||
watch.reconcile = false
|
||||
watch.recovery = false
|
||||
return delivery.capture()
|
||||
}
|
||||
|
||||
private async probeWatch(
|
||||
@@ -68,7 +109,7 @@ export class SessionSearchRootWatchers {
|
||||
key: string,
|
||||
mode: 'recursive' | 'shallow',
|
||||
signal?: AbortSignal
|
||||
): Promise<SessionSearchRootWatch | undefined> {
|
||||
): Promise<NativeWatch | undefined> {
|
||||
const previous = this.watches.get(key)
|
||||
let identity: string
|
||||
try {
|
||||
@@ -81,6 +122,11 @@ export class SessionSearchRootWatchers {
|
||||
identity = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}`
|
||||
} catch {
|
||||
throwIfAiVaultScanCancelled(signal)
|
||||
if (previous) {
|
||||
for (const target of this.consumers.get(key)?.values() ?? []) {
|
||||
target.stage(previous)
|
||||
}
|
||||
}
|
||||
this.watches.delete(key)
|
||||
await previous?.subscription?.unsubscribe().catch(() => undefined)
|
||||
return undefined
|
||||
@@ -92,8 +138,13 @@ export class SessionSearchRootWatchers {
|
||||
) {
|
||||
return previous
|
||||
}
|
||||
if (previous) {
|
||||
for (const target of this.consumers.get(key)?.values() ?? []) {
|
||||
target.stage(previous)
|
||||
}
|
||||
}
|
||||
await previous?.subscription?.unsubscribe().catch(() => undefined)
|
||||
const watch: SessionSearchRootWatch = {
|
||||
const watch: NativeWatch = {
|
||||
identity,
|
||||
failed: false,
|
||||
events: new Map(),
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { SessionSearchInputReceipt } from './session-search-catalog-types'
|
||||
import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol'
|
||||
|
||||
type Notice = {
|
||||
identity: string
|
||||
failed: boolean
|
||||
events: Map<string, WatcherProcessEvent>
|
||||
reconcile: boolean
|
||||
recovery: boolean
|
||||
}
|
||||
export type SessionSearchRootWatch = Notice & {
|
||||
receipt: (phase: string) => SessionSearchInputReceipt
|
||||
eventReceipt: (path: string) => SessionSearchInputReceipt
|
||||
acknowledgeEvent: (path: string) => void
|
||||
acknowledge: () => void
|
||||
}
|
||||
|
||||
/** Each consumer retains one unaccepted delivery beside coalesced later arrivals. */
|
||||
export class SessionSearchWatchDelivery {
|
||||
private events = new Map<string, WatcherProcessEvent>()
|
||||
private reconcile = false
|
||||
private recovery = false
|
||||
private active: SessionSearchRootWatch | undefined
|
||||
private identity = ''
|
||||
private failed = false
|
||||
constructor(
|
||||
private readonly consumer: string,
|
||||
private readonly next: () => number
|
||||
) {}
|
||||
get pending(): boolean {
|
||||
return !!this.active || this.events.size > 0 || this.reconcile || this.recovery
|
||||
}
|
||||
stage(notice: Notice): void {
|
||||
if (this.identity && this.identity !== notice.identity) {
|
||||
this.reconcile = true
|
||||
this.recovery = true
|
||||
}
|
||||
this.identity = notice.identity
|
||||
this.failed = notice.failed
|
||||
this.reconcile ||= notice.reconcile
|
||||
this.recovery ||= notice.recovery
|
||||
for (const [path, event] of notice.events) {
|
||||
if (this.events.size >= 16_384 && !this.events.has(path)) {
|
||||
this.reconcile = true
|
||||
this.recovery = true
|
||||
break
|
||||
}
|
||||
this.events.set(path, event)
|
||||
}
|
||||
}
|
||||
capture(): SessionSearchRootWatch {
|
||||
if (this.active) {
|
||||
return this.active
|
||||
}
|
||||
const revision = this.next()
|
||||
const events = this.events
|
||||
const receipts = new Map([...events.keys()].map((path) => [path, this.next()]))
|
||||
this.events = new Map()
|
||||
const active: SessionSearchRootWatch = {
|
||||
identity: this.identity,
|
||||
failed: this.failed,
|
||||
events,
|
||||
reconcile: this.reconcile,
|
||||
recovery: this.recovery,
|
||||
receipt: (phase) => ({ consumer: `${this.consumer}:${phase}`, revision }),
|
||||
eventReceipt: (path) => ({
|
||||
consumer: `${this.consumer}:events`,
|
||||
revision: receipts.get(path)!
|
||||
}),
|
||||
acknowledgeEvent: (path) => {
|
||||
events.delete(path)
|
||||
receipts.delete(path)
|
||||
},
|
||||
acknowledge: () => {
|
||||
if (this.active === active) {
|
||||
this.active = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
this.reconcile = false
|
||||
this.recovery = false
|
||||
this.active = active
|
||||
return active
|
||||
}
|
||||
}
|
||||
@@ -64,10 +64,15 @@ export function notifySessionSearchFileChange(path: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
export function notifySessionSearchOverflow(path: string): void {
|
||||
export function notifySessionSearchOverflow(path: string): number {
|
||||
let delivered = 0
|
||||
for (const watch of watchers.values()) {
|
||||
if (watch.path === path) {
|
||||
watch.hooks.onOverflow?.()
|
||||
if (watch.hooks.onOverflow) {
|
||||
watch.hooks.onOverflow()
|
||||
delivered++
|
||||
}
|
||||
}
|
||||
}
|
||||
return delivered
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import { SessionInventoryDirectoryWalks } from './session-file-inventory-walk'
|
||||
import { dirname } from 'node:path'
|
||||
import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation'
|
||||
import {
|
||||
isMissingTranscriptPathError,
|
||||
type DiscoverSessionFilesArgs
|
||||
} from './session-scanner-discovery'
|
||||
import { isInsideSessionRoot } from './session-file-discovery'
|
||||
|
||||
type DirtyInput = { path: string; force: boolean; dirtyRevision: number }
|
||||
type Probe = {
|
||||
input: DirtyInput
|
||||
serial: number
|
||||
revision?: number
|
||||
result?: { directory: boolean } | { error: unknown }
|
||||
phase: number
|
||||
valid?: boolean
|
||||
}
|
||||
|
||||
/** A successful probe retains its next handoff across catalog failures. */
|
||||
export class SessionInventoryDirtyInputs extends SessionInventoryDirectoryWalks {
|
||||
private probeSequence = 0
|
||||
private probe: Probe | undefined
|
||||
protected get dirtyProbing(): boolean {
|
||||
return this.probe !== undefined
|
||||
}
|
||||
protected async drainDirty(args: DiscoverSessionFilesArgs, limit: number): Promise<void> {
|
||||
if (this.probe) {
|
||||
await this.finishProbe(args)
|
||||
}
|
||||
const page = await this.request({ kind: 'dirty-page', scope: this.scope })
|
||||
const inputs = page.dirtyInputs ?? []
|
||||
await this.observations.observePage(
|
||||
inputs.filter(({ path }) => this.accepts(path, args)),
|
||||
args,
|
||||
limit
|
||||
)
|
||||
for (const input of inputs) {
|
||||
if (this.accepts(input.path, args)) {
|
||||
continue
|
||||
}
|
||||
if (!isInsideSessionRoot(this.root, input.path)) {
|
||||
await this.acknowledgeDirty(input)
|
||||
continue
|
||||
}
|
||||
this.probe = { input, serial: ++this.probeSequence, phase: 0 }
|
||||
await this.finishProbe(args)
|
||||
}
|
||||
}
|
||||
private acknowledgeDirty(input: DirtyInput): Promise<unknown> {
|
||||
return this.request({
|
||||
kind: 'dirty-ack',
|
||||
scope: this.scope,
|
||||
path: input.path,
|
||||
revision: input.dirtyRevision
|
||||
})
|
||||
}
|
||||
private async finishProbe(args: DiscoverSessionFilesArgs): Promise<void> {
|
||||
const probe = this.probe!
|
||||
const { path } = probe.input
|
||||
const receipt = (phase: string) => ({
|
||||
consumer: `dirty-probe:${phase}`,
|
||||
revision: probe.serial
|
||||
})
|
||||
if (probe.revision === undefined) {
|
||||
probe.revision = (
|
||||
await this.request({
|
||||
kind: 'authority-probe',
|
||||
scope: this.scope,
|
||||
path,
|
||||
receipt: receipt('begin')
|
||||
})
|
||||
).observationEpoch!
|
||||
}
|
||||
if (!probe.result) {
|
||||
try {
|
||||
const stat = await wslGatedStat(path, 'scan', args.signal)
|
||||
probe.result = { directory: stat.isDirectory() }
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
probe.result = { error }
|
||||
}
|
||||
}
|
||||
if (probe.valid === undefined) {
|
||||
probe.valid = !(
|
||||
await this.request({
|
||||
kind: 'authority-probe-end',
|
||||
scope: this.scope,
|
||||
path,
|
||||
revision: probe.revision,
|
||||
receipt: receipt('end')
|
||||
})
|
||||
).invalidProof
|
||||
}
|
||||
if (probe.valid) {
|
||||
if ('error' in probe.result) {
|
||||
if (isMissingTranscriptPathError(probe.result.error)) {
|
||||
if (probe.phase === 0) {
|
||||
await this.schedule(path, false, true, probe.revision)
|
||||
probe.phase = 1
|
||||
}
|
||||
await this.schedule(dirname(path))
|
||||
} else {
|
||||
await this.issue(path, probe.result.error, args)
|
||||
}
|
||||
} else {
|
||||
if (probe.phase === 0) {
|
||||
await this.failures.delete(path)
|
||||
probe.phase = 1
|
||||
}
|
||||
if (probe.result.directory) {
|
||||
await this.request({
|
||||
kind: 'directory-schedule',
|
||||
scope: this.scope,
|
||||
path,
|
||||
depth: this.depth(path),
|
||||
reconcile: false,
|
||||
trailing: true,
|
||||
receipt: receipt('schedule')
|
||||
})
|
||||
} else if (await this.directories.get(path)) {
|
||||
await this.schedule(dirname(path))
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.acknowledgeDirty(probe.input)
|
||||
this.probe = undefined
|
||||
}
|
||||
private depth(path: string): number {
|
||||
let depth = 0,
|
||||
key = path
|
||||
while (key !== this.root && dirname(key) !== key) {
|
||||
depth++
|
||||
key = dirname(key)
|
||||
}
|
||||
return depth
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,13 @@ export class SessionInventoryFiles {
|
||||
)
|
||||
}
|
||||
async observePage(
|
||||
inputs: { path: string; force: boolean; readOwed?: boolean; metadataOwed?: boolean }[],
|
||||
inputs: {
|
||||
path: string
|
||||
force: boolean
|
||||
readOwed?: boolean
|
||||
metadataOwed?: boolean
|
||||
dirtyRevision?: number
|
||||
}[],
|
||||
args: DiscoverSessionFilesArgs,
|
||||
limit: number
|
||||
): Promise<Map<string, FileWithMtime>> {
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
import type { SessionSearchInputReceipt } from '../ai-vault-search/session-search-catalog-types'
|
||||
import type { SessionSearchRemovalReceipt } from '../ai-vault-search/session-search-observation-authority'
|
||||
import { SessionInventoryDirectoryWalks } from './session-file-inventory-walk'
|
||||
import { dirname } from 'node:path'
|
||||
import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access'
|
||||
import { SessionInventoryDirtyInputs } from './session-file-inventory-dirty'
|
||||
import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation'
|
||||
import {
|
||||
isMissingTranscriptPathError,
|
||||
type DiscoverSessionFilesArgs
|
||||
} from './session-scanner-discovery'
|
||||
import { isInsideSessionRoot } from './session-file-discovery'
|
||||
import type { DiscoverSessionFilesArgs } from './session-scanner-discovery'
|
||||
import type { SessionFileChange } from './session-file-inventory-files'
|
||||
export type { SessionFileChange } from './session-file-inventory-files'
|
||||
|
||||
/** The writer retains queued directories, observations and membership across bounded passes. */
|
||||
export class SessionFileInventory extends SessionInventoryDirectoryWalks {
|
||||
export class SessionFileInventory extends SessionInventoryDirtyInputs {
|
||||
private wakeSequence = 0
|
||||
private waking: SessionSearchInputReceipt | undefined
|
||||
private reconcileAgain = false
|
||||
private reconciling = false
|
||||
private initialized = false
|
||||
@@ -48,6 +45,7 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks {
|
||||
}
|
||||
get pending(): boolean {
|
||||
return (
|
||||
this.dirtyProbing ||
|
||||
this.observations.applying ||
|
||||
this.authority.readySize > 0 ||
|
||||
this.garbage.size > 0 ||
|
||||
@@ -73,36 +71,75 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks {
|
||||
await this.reconcile()
|
||||
}
|
||||
}
|
||||
async wakeFailures(): Promise<void> {
|
||||
async wakeFailures(receipt?: SessionSearchInputReceipt): Promise<void> {
|
||||
const acceptedReceipt =
|
||||
receipt ?? (this.waking ??= { consumer: 'explicit-recovery', revision: ++this.wakeSequence })
|
||||
await this.request({
|
||||
kind: 'retry-wake',
|
||||
scope: `${this.scope}:primary-wakes`,
|
||||
prefix: this.root
|
||||
scope: this.scope,
|
||||
prefix: this.root,
|
||||
all: true,
|
||||
receipt: acceptedReceipt
|
||||
})
|
||||
await this.failures.wake()
|
||||
await this.observations.failures.wake()
|
||||
if (!receipt) {
|
||||
this.waking = undefined
|
||||
}
|
||||
}
|
||||
get refused(): boolean {
|
||||
return this.failures.size > 0 || this.observations.failures.size > 0
|
||||
}
|
||||
get walking(): boolean {
|
||||
return this.cancellations.size > 0 || this.walks.size > 0 || this.reconcileAgain
|
||||
get seedDelivered(): boolean {
|
||||
return !this.observations.delivery.size && !this.observations.applying
|
||||
}
|
||||
async reconcile(): Promise<void> {
|
||||
get primarySeedWaiting(): boolean {
|
||||
return this.primaryWakes.size > 0 && !this.cancellations.size && !this.walks.size
|
||||
}
|
||||
get walking(): boolean {
|
||||
return (
|
||||
this.cancellations.size > 0 ||
|
||||
this.walks.size > 0 ||
|
||||
this.reconcileAgain ||
|
||||
this.primarySeedWaiting
|
||||
)
|
||||
}
|
||||
async reconcile(receipt?: SessionSearchInputReceipt): Promise<void> {
|
||||
if (this.primarySeedWaiting) {
|
||||
this.reconcileAgain = true
|
||||
return
|
||||
}
|
||||
if (receipt) {
|
||||
await this.request({
|
||||
kind: 'directory-schedule',
|
||||
scope: this.scope,
|
||||
path: this.root,
|
||||
depth: 0,
|
||||
reconcile: true,
|
||||
trailing: true,
|
||||
receipt
|
||||
})
|
||||
this.reconciling = true
|
||||
return
|
||||
}
|
||||
if (this.reconciling) {
|
||||
this.reconcileAgain = true
|
||||
return
|
||||
}
|
||||
this.reconciling = true
|
||||
await this.schedule(this.root, true)
|
||||
this.reconciling = true
|
||||
}
|
||||
async invalidate(path: string, force = true, wake = true): Promise<void> {
|
||||
async invalidate(
|
||||
path: string,
|
||||
force = true,
|
||||
wake = true,
|
||||
receipt?: SessionSearchInputReceipt
|
||||
): Promise<void> {
|
||||
if (!wake) {
|
||||
await this.dirty.set(path, ((await this.dirty.get(path)) ?? false) || force)
|
||||
return
|
||||
}
|
||||
await this.request({
|
||||
kind: 'input-invalidate',
|
||||
receipt,
|
||||
scope: this.scope,
|
||||
path,
|
||||
force,
|
||||
@@ -197,7 +234,7 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks {
|
||||
await this.request({ kind: 'authority-cleanup', scope: this.scope })
|
||||
}
|
||||
}
|
||||
if (!this.walks.size && !this.cancellations.size) {
|
||||
if (!this.walks.size && !this.cancellations.size && !this.primaryWakes.size) {
|
||||
this.reconciling = false
|
||||
if (this.reconcileAgain) {
|
||||
this.reconcileAgain = false
|
||||
@@ -238,57 +275,20 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks {
|
||||
acknowledge(receipts: { path: string; revision: number }[]): Promise<void> {
|
||||
return this.observations.acknowledge(receipts)
|
||||
}
|
||||
private async drainDirty(args: DiscoverSessionFilesArgs, limit: number): Promise<void> {
|
||||
const page = await this.request({ kind: 'dirty-page', scope: this.scope })
|
||||
const inputs = page.dirtyInputs ?? []
|
||||
await this.observations.observePage(
|
||||
inputs.filter(({ path }) => this.accepts(path, args)),
|
||||
args,
|
||||
limit
|
||||
)
|
||||
for (const { path } of inputs) {
|
||||
if (this.accepts(path, args) || !isInsideSessionRoot(this.root, path)) {
|
||||
continue
|
||||
}
|
||||
const revision = (await this.request({ kind: 'authority-probe', scope: this.scope, path }))
|
||||
.observationEpoch!
|
||||
try {
|
||||
const stat = await wslGatedStat(path, 'scan', args.signal)
|
||||
if (
|
||||
(await this.request({ kind: 'authority-probe-end', scope: this.scope, path, revision }))
|
||||
.invalidProof
|
||||
) {
|
||||
continue
|
||||
}
|
||||
await this.failures.delete(path)
|
||||
if (stat.isDirectory()) {
|
||||
await ((await this.walks.has(path)) ? this.trailing.put(path, true) : this.schedule(path))
|
||||
} else if (await this.directories.get(path)) {
|
||||
await this.schedule(dirname(path))
|
||||
}
|
||||
} catch (error) {
|
||||
throwIfAiVaultScanCancelled(args.signal)
|
||||
if (
|
||||
(await this.request({ kind: 'authority-probe-end', scope: this.scope, path, revision }))
|
||||
.invalidProof
|
||||
) {
|
||||
continue
|
||||
}
|
||||
if (isMissingTranscriptPathError(error)) {
|
||||
await this.schedule(path, false, true, revision)
|
||||
await this.schedule(dirname(path))
|
||||
} else {
|
||||
await this.issue(path, error, args)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
async retryFailures(): Promise<void> {
|
||||
async retryFailures(primary = true): Promise<void> {
|
||||
await this.request({
|
||||
kind: 'retry-admit',
|
||||
primary,
|
||||
scope: this.scope,
|
||||
root: this.root,
|
||||
generation: this.failures.epoch
|
||||
})
|
||||
if (!this.primaryWakes.size && !this.walks.size && !this.cancellations.size) {
|
||||
this.reconciling = false
|
||||
if (this.reconcileAgain) {
|
||||
this.reconcileAgain = false
|
||||
await this.reconcile()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user