diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index b754cea4db7..e1d40054e61 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -64,6 +64,9 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli } return unowned } + async changedForRoot(root: string, agent: string) { + return (await this.request!({ kind: 'changes', root, agent })).rows + } async admitChanges( receipts: { key: string; revision: number }[], recent: boolean, diff --git a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts index 8dfb786874e..67a705a7278 100644 --- a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts @@ -40,7 +40,7 @@ export type SessionSearchCandidateCommand = | { kind: 'unowned-page'; rows: { path: string; agent?: string | null }[] } | { kind: 'path'; path: string; agent?: string | null } | { kind: 'group'; group: string; after?: string } - | { kind: 'changes'; after?: string } + | { kind: 'changes'; after?: string; root?: string; agent?: string } | { kind: 'rerank'; root: string; agent: string; order: [string, number][]; after?: string } | { kind: 'consumed'; key: string; revision: number } | SessionSearchPublicationCommand diff --git a/src/main/ai-vault-search/session-search-catalog-candidates.ts b/src/main/ai-vault-search/session-search-catalog-candidates.ts index 8ff8f0321c2..8d008a7a4b7 100644 --- a/src/main/ai-vault-search/session-search-catalog-candidates.ts +++ b/src/main/ai-vault-search/session-search-catalog-candidates.ts @@ -43,6 +43,7 @@ export class SessionSearchCatalogCandidates { CREATE INDEX temp.search_candidate_rank ON search_candidates(group_key,sqlite_source,mtime DESC,source_order,key); CREATE INDEX temp.search_candidate_changes ON search_candidates(changed,mtime DESC,key); CREATE INDEX temp.search_candidate_source ON search_candidates(root,agent,group_key); + CREATE INDEX temp.search_candidate_seed ON search_candidates(root,agent,changed,mtime DESC,key); `) } @@ -98,9 +99,11 @@ export class SessionSearchCatalogCandidates { ]) ) case 'changes': { - const rows = this.read('WHERE changed = 1 AND key > ? ORDER BY mtime DESC,key LIMIT 256', [ - command.after ?? '' - ]) + const scoped = command.root !== undefined && command.agent !== undefined + const rows = this.read( + `WHERE changed = 1 AND key > ?${scoped ? ' AND root=? AND agent=?' : ''} ORDER BY mtime DESC,key LIMIT 256`, + scoped ? [command.after ?? '', command.root!, command.agent!] : [command.after ?? ''] + ) return this.result(rows) } case 'rerank': { @@ -156,7 +159,7 @@ export class SessionSearchCatalogCandidates { `UPDATE temp.search_candidates SET required = ${command.metadataOnly ? 'force_read' : '0'},force_read=${command.metadataOnly ? 'force_read' : '0'},metadata=0 WHERE path = ? AND agent = ? AND selected = 1` ) .run(command.path, command.agent) - this.dirtyPath(command.path) + this.rowChanged(command.path) } return this.result([]) } @@ -196,7 +199,7 @@ export class SessionSearchCatalogCandidates { return this.read('WHERE key=?', [key])[0] } rowChanged(path: string): void { - this.dirtyPath(path) + this.publications.rowChanged(path) } private put(row: SessionSearchCatalogCandidate): void { @@ -271,14 +274,10 @@ export class SessionSearchCatalogCandidates { .run(++this.revision, path) if (Number(this.db.prepare('SELECT changes() AS n').get()?.n ?? 0)) { this.pending += previouslyClean - this.dirtyPath(path) + this.rowChanged(path) } } - private dirtyPath(path: string): void { - this.publications.rowChanged(path) - } - private read(where: string, values: (string | number)[]): SessionSearchCandidateRow[] { return boundedSessionSearchPage( this.db.prepare(`SELECT * FROM temp.search_candidates ${where}`).iterate(...values), diff --git a/src/main/ai-vault-search/session-search-catalog-transition.ts b/src/main/ai-vault-search/session-search-catalog-transition.ts index db38a82ac81..03a6df79596 100644 --- a/src/main/ai-vault-search/session-search-catalog-transition.ts +++ b/src/main/ai-vault-search/session-search-catalog-transition.ts @@ -42,7 +42,19 @@ export function transitionSessionSearchCatalog( return transitionSessionSearchContainer(catalog, command) case 'input-invalidate': case 'dirty-page': - return transitionSessionSearchDirty(catalog, command) + case 'dirty-ack': { + catalog.connection.exec('SAVEPOINT search_dirty_transition') + try { + const result = transitionSessionSearchDirty(catalog, command) + catalog.connection.exec('RELEASE search_dirty_transition') + return result + } catch (error) { + catalog.connection.exec( + 'ROLLBACK TO search_dirty_transition; RELEASE search_dirty_transition' + ) + throw error + } + } case 'dependency-remove': case 'dependency-deliver': case 'dependency-fanout': diff --git a/src/main/ai-vault-search/session-search-catalog-types.ts b/src/main/ai-vault-search/session-search-catalog-types.ts index 314719ea50f..921d608300f 100644 --- a/src/main/ai-vault-search/session-search-catalog-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-types.ts @@ -27,7 +27,8 @@ export type SessionSearchCatalogRow = { ready: number tie?: number } -export type SessionSearchCatalogCommand = +export type SessionSearchInputReceipt = { consumer: string; revision: number } +export type SessionSearchCatalogCommand = ( | SessionSearchRetryCommand | SessionSearchAuthorityCommand | SessionSearchDirtyCommand @@ -63,6 +64,7 @@ export type SessionSearchCatalogCommand = limit?: number prefix?: string } +) & { receipt?: SessionSearchInputReceipt } export type SessionSearchCatalogResult = SessionSearchDirtyResult & SessionSearchInventoryResult & SessionSearchDirectoryResult & { diff --git a/src/main/ai-vault-search/session-search-catalog.ts b/src/main/ai-vault-search/session-search-catalog.ts index e88623674cf..805346dfc1d 100644 --- a/src/main/ai-vault-search/session-search-catalog.ts +++ b/src/main/ai-vault-search/session-search-catalog.ts @@ -32,7 +32,45 @@ export class SessionSearchCatalog extends SessionSearchCatalogStorage { execute(command: SessionSearchCatalogCommand): SessionSearchCatalogResult { this.touched.add(command.scope) - return this.apply(command) + if (!command.receipt) { + return this.apply(command) + } + const scope = `${command.scope}:input-receipts` + const previous = this.execute({ kind: 'get', scope, key: command.receipt.consumer }).rows?.[0] + if (previous && previous.rank >= command.receipt.revision) { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only this receipt owner encodes the accepted response. + const accepted = JSON.parse(previous.value) as { + result: SessionSearchCatalogResult + scopes: string[] + } + for (const touchedScope of accepted.scopes) { + this.touched.add(touchedScope) + } + return accepted.result + } + this.db.exec('SAVEPOINT search_input_receipt') + try { + const result = this.apply({ ...command, receipt: undefined }) + this.execute({ + kind: 'put', + scope, + rows: [ + { + key: command.receipt.consumer, + value: JSON.stringify({ result, scopes: [...this.touched] }), + parent: '', + group: '', + rank: command.receipt.revision, + ready: 0 + } + ] + }) + this.db.exec('RELEASE search_input_receipt') + return result + } catch (error) { + this.db.exec('ROLLBACK TO search_input_receipt; RELEASE search_input_receipt') + throw error + } } response(command: SessionSearchCatalogCommand): SessionSearchCatalogResult { const records = diff --git a/src/main/ai-vault-search/session-search-dependency-inventory.ts b/src/main/ai-vault-search/session-search-dependency-inventory.ts index f409b4b6b8b..b6e8ca1f070 100644 --- a/src/main/ai-vault-search/session-search-dependency-inventory.ts +++ b/src/main/ai-vault-search/session-search-dependency-inventory.ts @@ -21,6 +21,11 @@ export class SessionSearchDependencyInventory { private readonly observe: SessionDiscoveryDependencyObserver | undefined private readonly retries = new SessionObservationRetries() private readonly delivered: SessionSearchCatalogClient<{ after?: string }> + private staging: + | { iterator?: AsyncGenerator; keys: string[]; serial: number; done: boolean } + | undefined + private stagingSequence = 0 + private inventoryAck: { path: string; revision: number }[] | undefined private deliveryAfter: string | undefined private discardAfter: string | undefined private readonly keys: SessionSearchCatalogClient @@ -40,6 +45,8 @@ export class SessionSearchDependencyInventory { } get pending(): boolean { return ( + !!this.staging || + !!this.inventoryAck || this.retries.pending || this.delivered.size > 0 || this.observe?.pending === true || @@ -88,19 +95,46 @@ export class SessionSearchDependencyInventory { args: DiscoverSessionFilesArgs, overdue: () => boolean ): Promise { - let keys: string[] = [] - const flush = async () => { - await this.delivered.putPage(keys.map((key) => this.delivered.encode(key, {}))) - keys = [] + this.staging ??= { + iterator: this.changedKeys(watch, args, overdue), + keys: [], + serial: ++this.stagingSequence, + done: false } - for await (const key of this.changedKeys(watch, args, overdue)) { - keys.push(key) - if (keys.length === 256) { - await flush() + const pending = this.staging + while (!pending.done || pending.keys.length) { + if ( + pending.keys.length && + (pending.done || pending.keys.length === 256 || !pending.iterator) + ) { + await this.delivered.transition({ + kind: 'dependency-deliver', + scope: this.scope, + keys: pending.keys, + receipt: { consumer: 'native-dependency', revision: pending.serial } + }) + pending.keys = [] + pending.serial = ++this.stagingSequence + } + if (pending.done) { + break + } + pending.iterator ??= this.changedKeys(watch, args, overdue) + try { + const next = await pending.iterator.next() + if (next.done) { + pending.done = true + } else { + pending.keys.push(next.value) + } + } catch (error) { + pending.iterator = undefined + throw error } } - await flush() + this.staging = undefined } + async addFile(path: string): Promise { await this.addFiles([path]) } @@ -130,6 +164,10 @@ export class SessionSearchDependencyInventory { args: DiscoverSessionFilesArgs, overdue: () => boolean ): AsyncGenerator { + if (this.inventoryAck) { + await this.inventory!.acknowledge(this.inventoryAck) + this.inventoryAck = undefined + } if (this.dependency.eventsOnly) { const keys = new Set() for (const event of watch?.events.values() ?? []) { @@ -176,10 +214,14 @@ export class SessionSearchDependencyInventory { const keys = new Set() const inventory = this.inventory! if ((!watch || watch.failed || watch.reconcile) && !inventory.walking) { - await inventory.reconcile() + await inventory.reconcile(watch?.receipt('reconcile')) + if (watch) { + watch.reconcile = false + } } for (const event of watch?.events.values() ?? []) { - await inventory.invalidate(event.path) + await inventory.invalidate(event.path, true, true, watch?.eventReceipt(event.path)) + watch?.acknowledgeEvent(event.path) } await inventory.retryFailures() const delta = await inventory.drain( @@ -202,7 +244,9 @@ export class SessionSearchDependencyInventory { keys.add(key) } } + this.inventoryAck = delta.receipts yield* keys await inventory.acknowledge(delta.receipts) + this.inventoryAck = undefined } } diff --git a/src/main/ai-vault-search/session-search-dirty-transition.ts b/src/main/ai-vault-search/session-search-dirty-transition.ts index 5f80db218af..fae2ba2afaf 100644 --- a/src/main/ai-vault-search/session-search-dirty-transition.ts +++ b/src/main/ai-vault-search/session-search-dirty-transition.ts @@ -1,3 +1,4 @@ +import { SessionSearchAuthorityPage } from './session-search-authority-page' import { wakeSessionSearchPrimaryFailures } from './session-search-primary-retries' import { recordSessionSearchInput } from './session-search-observation-authority' import type { SessionSearchCatalog, SessionSearchCatalogRow } from './session-search-catalog' @@ -5,8 +6,11 @@ import type { SessionSearchObservationDebt } from './session-search-inventory-tr export type SessionSearchDirtyCommand = | { kind: 'dirty-page'; scope: string } + | { kind: 'dirty-ack'; scope: string; path: string; revision: number } | { kind: 'input-invalidate'; scope: string; path: string; force: boolean; subtree?: boolean } -export type SessionSearchDirtyResult = { dirtyInputs?: { path: string; force: boolean }[] } +export type SessionSearchDirtyResult = { + dirtyInputs?: { path: string; force: boolean; dirtyRevision: number }[] +} type Entry = { value: boolean; revision: number } function entry(row: SessionSearchCatalogRow | undefined): Entry | undefined { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the matching frontier owns its encoded value and revision. @@ -80,10 +84,17 @@ export function transitionSessionSearchDirty( ) return {} } + if (command.kind === 'dirty-ack') { + acknowledgeSessionSearchDirty(catalog, command.scope, command.path, command.revision) + const authority = new SessionSearchAuthorityPage(catalog, command.scope) + authority.settle(command.path) + authority.flush() + return {} + } const scope = `${command.scope}:dirty`, expansion = `${command.scope}:dirty-expansion` const page = catalog.execute({ kind: 'page', scope, group: '', orderRank: true }) - const dirtyInputs: { path: string; force: boolean }[] = [] + const dirtyInputs: { path: string; force: boolean; dirtyRevision: number }[] = [] let remaining = 256 for (const row of page.rows ?? []) { const fact = entry(row)! @@ -94,9 +105,17 @@ export function transitionSessionSearchDirty( continue } const stored = catalog.execute({ kind: 'get', scope: expansion, key: row.key }).rows?.[0] - const cursor: { revision: number; after?: string } | undefined = stored + const cursor: { revision: number; after?: string; complete?: boolean } | undefined = stored ? JSON.parse(stored.value) : undefined + if (cursor?.revision === fact.revision && cursor.complete) { + dirtyInputs.push({ path: row.key, force: fact.value, dirtyRevision: fact.revision }) + remaining-- + if (!remaining) { + break + } + continue + } const owners = catalog.execute({ kind: 'page', scope: `${command.scope}:files`, @@ -127,9 +146,13 @@ export function transitionSessionSearchDirty( entry(catalog.execute({ kind: 'get', scope, key: row.key }).rows?.[0])?.revision === fact.revision ) { - catalog.execute({ kind: 'delete', scope: expansion, key: row.key }) - catalog.execute({ kind: 'delete', scope, key: row.key }) - dirtyInputs.push({ path: row.key, force: fact.value }) + catalog.execute({ + kind: 'put', + scope: expansion, + rows: [{ ...row, value: JSON.stringify({ revision: fact.revision, complete: true }) }] + }) + dirtyInputs.push({ path: row.key, force: fact.value, dirtyRevision: fact.revision }) + remaining-- } if (!remaining) { break @@ -137,3 +160,18 @@ export function transitionSessionSearchDirty( } return { dirtyInputs } } + +export function acknowledgeSessionSearchDirty( + catalog: SessionSearchCatalog, + owner: string, + path: string, + revision: number +): void { + const scope = `${owner}:dirty` + const current = entry(catalog.execute({ kind: 'get', scope, key: path }).rows?.[0]) + if (current?.revision !== revision) { + return + } + catalog.execute({ kind: 'delete', scope, key: path }) + catalog.execute({ kind: 'delete', scope: `${owner}:dirty-expansion`, key: path }) +} diff --git a/src/main/ai-vault-search/session-search-discovery-inventory.ts b/src/main/ai-vault-search/session-search-discovery-inventory.ts index fa2b3bca73e..5dccb347543 100644 --- a/src/main/ai-vault-search/session-search-discovery-inventory.ts +++ b/src/main/ai-vault-search/session-search-discovery-inventory.ts @@ -11,7 +11,10 @@ export class SessionSearchDiscoveryInventory extends SessionSearchDiscoverySources implements SessionDiscoveryContext { - async files(args: DiscoverSessionFilesArgs): Promise { + files(args: DiscoverSessionFilesArgs): Promise { + return this.track(this.discoverFiles(args)) + } + private async discoverFiles(args: DiscoverSessionFilesArgs): Promise { const key = `${args.agent}:${args.rootDir}` this.seen.add(key) let root = this.roots.get(key) @@ -33,6 +36,18 @@ export class SessionSearchDiscoveryInventory await root.inventory.wakeFailures() root.wake = this.primaryWake } + const admitPrimary = async () => { + if ( + root.inventory.primarySeedWaiting && + root.inventory.seedDelivered && + (!this.admitSeed || (await this.admitSeed(args.rootDir, args.agent))) + ) { + await root.inventory.retryFailures() + return true + } + return false + } + await admitPrimary() let watching = false if (this.maintenance) { root.fallback = await this.watchers.requiresPolling(args.rootDir) @@ -42,7 +57,12 @@ export class SessionSearchDiscoveryInventory await root.inventory.reconcile() } } else { - const watch = await this.watchers.watch(args.rootDir, args.signal) + const watch = await this.watchers.watch( + args.rootDir, + args.signal, + 'recursive', + root.inventory.scope + ) if (!watch) { root.unwatched = true if (!root.inventory.walking) { @@ -54,7 +74,8 @@ export class SessionSearchDiscoveryInventory (this.force || watch.reconcile || root.identity !== watch.identity) && (root.identity !== undefined || !root.inventory.walking) ) { - await root.inventory.reconcile() + await root.inventory.reconcile(watch.receipt('reconcile')) + watch.reconcile = false } if ( root.identity !== watch.identity && @@ -72,13 +93,16 @@ export class SessionSearchDiscoveryInventory root.unwatched || (root.identity !== undefined && root.identity !== watch.identity)) ) { - await root.inventory.wakeFailures() + await root.inventory.wakeFailures(watch.receipt('recovery')) + watch.recovery = false } root.unwatched = watch.failed root.identity = watch.identity for (const event of watch.events.values()) { - await root.inventory.invalidate(event.path) + await root.inventory.invalidate(event.path, true, true, watch.eventReceipt(event.path)) + watch.acknowledgeEvent(event.path) } + watch.acknowledge() } } const auxiliaries = await Promise.all( @@ -101,7 +125,12 @@ export class SessionSearchDiscoveryInventory observer, watch: observer.polling ? undefined - : await this.watchers.watch(dependency.path, args.signal, dependency.mode) + : await this.watchers.watch( + dependency.path, + args.signal, + dependency.mode, + `dependency:${key}:${dependency.path}` + ) } }) ) @@ -111,9 +140,10 @@ export class SessionSearchDiscoveryInventory args ) await observer.stageChanges(auxiliary, args, this.overdue) + auxiliary?.acknowledge() await observer.settleDelivered(root.inventory.scope, this.overdue) } - await root.inventory.retryFailures() + await root.inventory.retryFailures(false) if (root.fallback || !watching) { await root.inventory.pollRecent(args, this.recentLimit) } @@ -173,7 +203,8 @@ export class SessionSearchDiscoveryInventory } } await root.inventory.acknowledge(delta.receipts) - if ((!delta.changes.length && !delta.removed.length) || this.overdue()) { + const admittedPrimary = await admitPrimary() + if ((!delta.changes.length && !delta.removed.length && !admittedPrimary) || this.overdue()) { break } } while (root.inventory.pending) @@ -185,10 +216,20 @@ export class SessionSearchDiscoveryInventory read: (issues: AiVaultScanIssue[]) => Promise, issues: AiVaultScanIssue[] ) { - return this.containers.containerPaths(root, read, issues) + return this.track(this.containers.containerPaths(root, read, issues)) } - async container( + container( + key: string, + paths: readonly string[], + read: (paths: readonly string[], limit: number) => Promise, + limit: number, + issues: readonly AiVaultScanIssue[], + enumerationComplete = true + ): Promise { + return this.track(this.discoverContainer(key, paths, read, limit, issues, enumerationComplete)) + } + private async discoverContainer( key: string, paths: readonly string[], read: (paths: readonly string[], limit: number) => Promise, diff --git a/src/main/ai-vault-search/session-search-discovery-sources.ts b/src/main/ai-vault-search/session-search-discovery-sources.ts index 6a4c9b06f65..eb8dc416903 100644 --- a/src/main/ai-vault-search/session-search-discovery-sources.ts +++ b/src/main/ai-vault-search/session-search-discovery-sources.ts @@ -24,6 +24,20 @@ type Root = { /** Search owns its notifications and raw membership; ordinary History discovery stays stateless. */ export class SessionSearchDiscoverySources { + private readonly active = new Set>() + protected track(work: Promise): Promise { + this.active.add(work) + void work.then( + () => this.active.delete(work), + () => this.active.delete(work) + ) + return work + } + async settled(): Promise { + while (this.active.size) { + await Promise.allSettled(this.active) + } + } protected readonly roots = new Map() readonly containers = new SessionSearchContainerInventory() protected readonly owner = new AbortController() @@ -39,6 +53,10 @@ export class SessionSearchDiscoverySources { onRemoved(accept: NonNullable): void { this.acceptRemoved = accept } + protected admitSeed: ((root: string, agent: string) => Promise) | undefined + onSeedAdmission(accept: NonNullable): void { + this.admitSeed = accept + } protected accept: | (( discovery: SessionFileDiscovery, @@ -91,7 +109,7 @@ export class SessionSearchDiscoverySources { } get pending(): boolean { - if (this.containers.pending) { + if (this.containers.pending || this.watchers.pending) { return true } for (const root of this.roots.values()) { diff --git a/src/main/ai-vault-search/session-search-input-handoff.test.ts b/src/main/ai-vault-search/session-search-input-handoff.test.ts new file mode 100644 index 00000000000..d9d0b5ee2cd --- /dev/null +++ b/src/main/ai-vault-search/session-search-input-handoff.test.ts @@ -0,0 +1,176 @@ +import './session-search-watcher.test-fixture' +import './session-search-worker.test-fixture' +import { join } from 'node:path' +import { mkdir } from 'node:fs/promises' +import { expect, it, vi } from 'vitest' +import { SessionSearchIndexer } from './session-search-indexer' +import { SessionSearchWorkerClient } from './session-search-worker-client' +import { + notifySessionSearchFileChange, + notifySessionSearchOverflow +} from './session-search-watcher.test-fixture' +import { + FakeSessionSearchClock, + openSessionSearchIndexerHarness, + writeClaudeTranscript +} from './session-search-indexer-test-fixture' + +vi.mock('./session-search-filesystem', () => ({ sessionSearchRequiresPolling: async () => false })) + +it.each([ + 'event-before', + 'event-after', + 'prepare-before', + 'prepare-after', + 'directory-schedule' +] as const)( + 'retains a delivered new conversation across %s rejection without a second notice', + async (failure) => { + const harness = await openSessionSearchIndexerHarness('ss-input-handoff') + const root = harness.roots.claudeProjectsDir! + const directory = join(root, 'new-project') + const first = join(harness.claudeProjectDir, 'first.jsonl') + const second = join( + failure === 'directory-schedule' ? directory : harness.claudeProjectDir, + 'second.jsonl' + ) + const clock = new FakeSessionSearchClock() + const errors: unknown[] = [] + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock, + onError: (error) => errors.push(error) + }) + const request = SessionSearchWorkerClient.prototype.request + let armed = false, + rejected = false + const gate = vi + .spyOn(SessionSearchWorkerClient.prototype, 'request') + .mockImplementation(async function (this: SessionSearchWorkerClient, command) { + const target = + command.op === 'catalog' && + ((failure.startsWith('event-') && + command.command.kind === 'input-invalidate' && + command.command.path === second) || + (failure.startsWith('prepare-') && + command.command.kind === 'observe-prepare' && + command.command.inputs.some((input) => input.path === second)) || + (failure === 'directory-schedule' && + command.command.kind === 'directory-schedule' && + command.command.path === directory)) + if (armed && !rejected && target) { + rejected = true + if (failure.endsWith('after')) { + await request.call(this, command) + } + throw new Error('fixture catalog handoff rejected') + } + return request.call(this, command) + }) + const hits = () => + harness.read((db) => + db + .prepare( + "SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'handoffneedle'" + ) + .get() + ) + try { + await mkdir(harness.claudeProjectDir, { recursive: true }) + await subject.start() + await writeClaudeTranscript(first, ['handoffneedle'], 'handoff-first') + await writeClaudeTranscript(second, ['handoffneedle'], 'handoff-second') + if (failure === 'directory-schedule') { + notifySessionSearchFileChange(directory) + } + armed = true + await subject.reconcile() + expect(rejected).toBe(true) + expect(errors).toHaveLength(1) + // Only resume the failed boundary; no new notification, full refresh or maintenance tick. + await subject.reconcile() + expect(hits()).toEqual({ n: 4 }) + expect(subject.status().phase).toBe('current') + } finally { + gate.mockRestore() + await subject.close() + await harness.cleanup() + } + } +) + +it.each(['before', 'after'] as const)( + 'retains native recovery across an accepted-result %s rejection', + async (completion) => { + const harness = await openSessionSearchIndexerHarness('ss-native-recovery') + const path = join(harness.claudeProjectDir, 'session.jsonl') + const clock = new FakeSessionSearchClock() + const subject = new SessionSearchIndexer({ + databasePath: harness.databasePath, + roots: harness.roots, + historyDays: null, + clock + }) + const request = SessionSearchWorkerClient.prototype.request + let armed = false, + rejected = false + const gate = vi + .spyOn(SessionSearchWorkerClient.prototype, 'request') + .mockImplementation(async function (this: SessionSearchWorkerClient, command) { + if ( + armed && + !rejected && + command.op === 'catalog' && + command.command.kind === 'retry-wake' && + command.command.all && + command.command.receipt?.consumer.endsWith(':recovery') + ) { + rejected = true + if (completion === 'after') { + await request.call(this, command) + } + throw new Error('fixture native recovery rejected') + } + return request.call(this, command) + }) + try { + await writeClaudeTranscript(path, ['nativeold'], 'native-recovery') + await subject.start() + harness.write((db) => + db.exec( + "CREATE TRIGGER refuse_native BEFORE UPDATE ON sessions BEGIN SELECT RAISE(ABORT,'fixture refused content'); END" + ) + ) + await writeClaudeTranscript(path, ['nativenew'], 'native-recovery') + for (let attempt = 0; attempt < 3; attempt++) { + clock.advance(20_000) + await subject.settled() + } + expect( + harness.read((db) => + db.prepare('SELECT state,fail_count FROM files WHERE path=?').get(path) + ) + ).toEqual({ state: 'failed', fail_count: 3 }) + harness.write((db) => db.exec('DROP TRIGGER refuse_native')) + armed = true + expect(notifySessionSearchOverflow(harness.roots.claudeProjectsDir!)).toBe(1) + await subject.reconcile() + expect(rejected).toBe(true) + await subject.reconcile() + expect( + harness.read((db) => + db + .prepare("SELECT count(*) AS n FROM messages_fts WHERE messages_fts MATCH 'nativenew'") + .get() + ) + ).toEqual({ n: 2 }) + expect(subject.status().phase).toBe('current') + } finally { + gate.mockRestore() + await subject.close() + await harness.cleanup() + } + } +) diff --git a/src/main/ai-vault-search/session-search-inventory-transition.ts b/src/main/ai-vault-search/session-search-inventory-transition.ts index d3cbed326f3..a5600ad7c59 100644 --- a/src/main/ai-vault-search/session-search-inventory-transition.ts +++ b/src/main/ai-vault-search/session-search-inventory-transition.ts @@ -1,3 +1,4 @@ +import { acknowledgeSessionSearchDirty } from './session-search-dirty-transition' import { SessionSearchAuthorityPage } from './session-search-authority-page' import { SESSION_INPUT_FAILURES_BEFORE_HELD_OUT } from '../ai-vault/session-observation-retries' import { recordSessionSearchRetry } from './session-search-retry-transition' @@ -137,6 +138,10 @@ export function transitionSessionSearchInventory( continue } remove(observing, outcome.path) + if (outcome.dirtyRevision !== undefined) { + acknowledgeSessionSearchDirty(catalog, command.scope, outcome.path, outcome.dirtyRevision) + authority.forget(`${command.scope}:dirty`, outcome.path) + } const accepted = decode(current) const previous = decode(get(failures, outcome.path)) if (previous?.attempted === command.generation) { diff --git a/src/main/ai-vault-search/session-search-observation-order.test.ts b/src/main/ai-vault-search/session-search-observation-order.test.ts index 7ecd98a06ab..8bf186ccd44 100644 --- a/src/main/ai-vault-search/session-search-observation-order.test.ts +++ b/src/main/ai-vault-search/session-search-observation-order.test.ts @@ -1,3 +1,4 @@ +import './session-search-watcher.test-fixture' import './session-search-worker.test-fixture' import { rm } from 'node:fs/promises' import { join } from 'node:path' @@ -120,7 +121,7 @@ it.each(['file', 'subtree'] as const)( await rm(child, { recursive: true }) } paused = false - notifySessionSearchOverflow(harness.roots.claudeProjectsDir!) + expect(notifySessionSearchOverflow(harness.roots.claudeProjectsDir!)).toBe(1) await subject.reconcile() expect(paused).toBe(true) refusing = variant === 'subtree' diff --git a/src/main/ai-vault-search/session-search-observation-preparation.ts b/src/main/ai-vault-search/session-search-observation-preparation.ts index 5743bc7f927..d060d03665a 100644 --- a/src/main/ai-vault-search/session-search-observation-preparation.ts +++ b/src/main/ai-vault-search/session-search-observation-preparation.ts @@ -1,3 +1,4 @@ +import { acknowledgeSessionSearchDirty } from './session-search-dirty-transition' import { dirname } from 'node:path' import { SessionSearchAuthorityPage } from './session-search-authority-page' import { boundedSessionSearchPage } from './session-search-catalog-pages' @@ -37,6 +38,7 @@ export function prepareSessionSearchObservations( failures, `${failures}:eligible`, `${command.scope}:dirty`, + `${command.scope}:dirty-expansion`, `${command.scope}:authority`, `${command.scope}:authority-root`, observing, @@ -52,8 +54,23 @@ export function prepareSessionSearchObservations( (input) => { const refusal = get(failures, input.path) const debt = authorityValue(refusal) - const revision = catalog.nextRevision() - const attempt = !debt || (debt.failures < 3 && debt.attempted !== command.generation) + const dirty = get(`${command.scope}:dirty`, input.path) + const stale = + input.dirtyRevision !== undefined && + authorityValue<{ revision: number }>(dirty)?.revision !== input.dirtyRevision + const previousToken = authorityValue(get(observing, input.path)) + const previous: [string, string, number] | undefined = previousToken + ? JSON.parse(previousToken) + : undefined + const reuse = + input.dirtyRevision !== undefined && + previous && + previous[0] === (get(files, input.path)?.value ?? '') && + previous[1] === (dirty?.value ?? '') && + authority.authority(input.path)?.self === previous[2] + const revision = reuse ? previous[2] : catalog.nextRevision() + const attempt = + !stale && (!debt || (debt.failures < 3 && debt.attempted !== command.generation)) const mergesDebt = !attempt && debt && @@ -99,6 +116,11 @@ export function prepareSessionSearchObservations( ) authority.forget(failures, input.path) } + if (!input.attempt && input.dirtyRevision !== undefined) { + acknowledgeSessionSearchDirty(catalog, command.scope, input.path, input.dirtyRevision) + authority.forget(`${command.scope}:dirty`, input.path) + authority.settle(input.path) + } if (input.attempt) { authority.record(input.path, input.authority, 0) put(observing, input.path, input.token) diff --git a/src/main/ai-vault-search/session-search-observation-types.ts b/src/main/ai-vault-search/session-search-observation-types.ts index 1e8880895cc..eaa46226279 100644 --- a/src/main/ai-vault-search/session-search-observation-types.ts +++ b/src/main/ai-vault-search/session-search-observation-types.ts @@ -17,6 +17,7 @@ export type SessionSearchObservationInput = { force: boolean readOwed?: boolean metadataOwed?: boolean + dirtyRevision?: number } export type SessionSearchPreparedObservation = SessionSearchObservationInput & { accepted?: SessionSearchObservedFacts diff --git a/src/main/ai-vault-search/session-search-pass.ts b/src/main/ai-vault-search/session-search-pass.ts index 174ece02d03..49ce7136627 100644 --- a/src/main/ai-vault-search/session-search-pass.ts +++ b/src/main/ai-vault-search/session-search-pass.ts @@ -116,11 +116,30 @@ export async function runSessionSearchPass( } while (state.candidates.pendingChanges) } let discoveryLane = Promise.resolve() - const serializeDiscovery = (consume: () => Promise): Promise => { + const serializeDiscovery = (consume: () => Promise): Promise => { const accepted = discoveryLane.then(consume) - discoveryLane = accepted + discoveryLane = accepted.then( + () => undefined, + () => undefined + ) return accepted } + state.discovery.onSeedAdmission((root, agent) => + serializeDiscovery(async () => { + const rows = await state.candidates.changedForRoot(root, agent) + if (rows.length) { + await state.parkedRetire.deletePage(rows.map((row) => row.candidate.file.path)) + state.queue.update( + await state.candidates.admitChanges( + rows.map(({ key, revision }) => ({ key, revision })), + !args.full, + state.recovering + ) + ) + } + return !(await state.candidates.changedForRoot(root, agent)).length + }) + ) state.discovery.onRemoved((paths, observations) => serializeDiscovery(async () => { if (!paths.length) { @@ -139,7 +158,7 @@ export async function runSessionSearchPass( ) const swept = await withSessionDiscovery(state.discovery, () => discoverSessionSearchCandidates(args.roots, { limitPerAgent: Infinity, signal, raw: true }) - ) + ).finally(() => state.discovery.settled()) await state.discovery.end() const issues: AiVaultScanIssue[] = [...swept.issues] const update = await state.candidates.update( diff --git a/src/main/ai-vault-search/session-search-retry-transition.ts b/src/main/ai-vault-search/session-search-retry-transition.ts index e7bf4058baf..447fabb95c6 100644 --- a/src/main/ai-vault-search/session-search-retry-transition.ts +++ b/src/main/ai-vault-search/session-search-retry-transition.ts @@ -33,8 +33,8 @@ export type SessionSearchRetryCommand = } | { kind: 'retry-remove'; scope: string; path: string } | { kind: 'retry-require'; scope: string; path: string } - | { kind: 'retry-wake'; scope: string; prefix?: string } - | { kind: 'retry-admit'; scope: string; root: string; generation: number } + | { kind: 'retry-wake'; scope: string; prefix?: string; all?: boolean } + | { kind: 'retry-admit'; scope: string; root: string; generation: number; primary?: boolean } export function recordSessionSearchRetry( catalog: SessionSearchCatalog, @@ -153,7 +153,9 @@ export function transitionSessionSearchRetry( command: SessionSearchRetryCommand ) { if (command.kind === 'retry-admit') { - admitSessionSearchPrimaryFailures(catalog, command.scope) + if (command.primary !== false) { + admitSessionSearchPrimaryFailures(catalog, command.scope) + } admit( catalog, command.scope, @@ -173,6 +175,16 @@ export function transitionSessionSearchRetry( return {} } if (command.kind === 'retry-wake') { + if (command.all && command.prefix) { + for (const suffix of ['primary-wakes', 'refusals', 'file-refusals']) { + catalog.execute({ + kind: 'retry-wake', + scope: `${command.scope}:${suffix}`, + prefix: command.prefix + }) + } + return {} + } if (command.scope.endsWith(':primary-wakes') && command.prefix) { wakeSessionSearchPrimaryFailures( catalog, diff --git a/src/main/ai-vault-search/session-search-root-watchers.ts b/src/main/ai-vault-search/session-search-root-watchers.ts index 942e94bbafa..553e0170634 100644 --- a/src/main/ai-vault-search/session-search-root-watchers.ts +++ b/src/main/ai-vault-search/session-search-root-watchers.ts @@ -5,7 +5,9 @@ import type { WatcherProcessSubscription } from '../ipc/filesystem/watcher-proce import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol' import { throwIfAiVaultScanCancelled } from '../ai-vault/ai-vault-scan-cancellation' import { RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS } from '../../shared/file-watch/runtime-file-watch-limits' -export type SessionSearchRootWatch = { +import { SessionSearchWatchDelivery } from './session-search-watch-delivery' +export type { SessionSearchRootWatch } from './session-search-watch-delivery' +type NativeWatch = { identity: string subscription?: WatcherProcessSubscription failed: boolean @@ -17,17 +19,36 @@ export type SessionSearchRootWatch = { } export class SessionSearchRootWatchers { - private readonly watches = new Map() + private readonly watches = new Map() private readonly owner = new AbortController() private readonly supervisor: WatcherProcessSupervisor - private probes = new Map>() + private sequence = 0 + private readonly consumers = new Map>() + private seen = new Set() + get pending(): boolean { + return [...this.consumers.values()].some((consumers) => + [...consumers.values()].some((consumer) => consumer.pending) + ) + } + private probes = new Map>() constructor(entryPath?: string) { this.supervisor = new WatcherProcessSupervisor({ entryPath }) } begin(): void { this.probes = new Map() + this.seen.clear() } async end(): Promise { + for (const [key, consumers] of this.consumers) { + for (const consumer of consumers.keys()) { + if (!this.seen.has(`${key}:${consumer}`)) { + consumers.delete(consumer) + } + } + if (!consumers.size) { + this.consumers.delete(key) + } + } for (const [path, watch] of this.watches) { if (!this.probes.has(path)) { this.watches.delete(path) @@ -38,29 +59,49 @@ export class SessionSearchRootWatchers { close(): Promise { this.owner.abort() this.watches.clear() + this.consumers.clear() return this.supervisor.disposeAndWait() } - watch( + async watch( path: string, signal?: AbortSignal, - mode: 'recursive' | 'shallow' = 'recursive' - ): Promise { + mode: 'recursive' | 'shallow' = 'recursive', + consumer = path + ) { const key = `${mode}:${path}` + this.seen.add(`${key}:${consumer}`) + let consumers = this.consumers.get(key) + if (!consumers) { + consumers = new Map() + this.consumers.set(key, consumers) + } + let delivery = consumers.get(consumer) + if (!delivery) { + delivery = new SessionSearchWatchDelivery(consumer, () => ++this.sequence) + consumers.set(consumer, delivery) + } let probe = this.probes.get(key) if (!probe) { - probe = this.probeWatch(path, key, mode, signal).then((watch) => { - if (!watch) { - return undefined - } - const delivery = { ...watch } - watch.events = new Map() - watch.reconcile = false - watch.recovery = false - return delivery - }) + probe = this.probeWatch(path, key, mode, signal) this.probes.set(key, probe) } - return probe + const watch = await probe + if (!watch) { + if (!delivery.pending) { + return undefined + } + const pending = delivery.capture() + pending.failed = true + pending.reconcile = true + return pending + } + for (const target of consumers.values()) { + target.stage(watch) + } + watch.events = new Map() + watch.reconcile = false + watch.recovery = false + return delivery.capture() } private async probeWatch( @@ -68,7 +109,7 @@ export class SessionSearchRootWatchers { key: string, mode: 'recursive' | 'shallow', signal?: AbortSignal - ): Promise { + ): Promise { const previous = this.watches.get(key) let identity: string try { @@ -81,6 +122,11 @@ export class SessionSearchRootWatchers { identity = `${stat.dev}:${stat.ino}:${stat.birthtimeMs}` } catch { throwIfAiVaultScanCancelled(signal) + if (previous) { + for (const target of this.consumers.get(key)?.values() ?? []) { + target.stage(previous) + } + } this.watches.delete(key) await previous?.subscription?.unsubscribe().catch(() => undefined) return undefined @@ -92,8 +138,13 @@ export class SessionSearchRootWatchers { ) { return previous } + if (previous) { + for (const target of this.consumers.get(key)?.values() ?? []) { + target.stage(previous) + } + } await previous?.subscription?.unsubscribe().catch(() => undefined) - const watch: SessionSearchRootWatch = { + const watch: NativeWatch = { identity, failed: false, events: new Map(), diff --git a/src/main/ai-vault-search/session-search-watch-delivery.ts b/src/main/ai-vault-search/session-search-watch-delivery.ts new file mode 100644 index 00000000000..bd5395b450a --- /dev/null +++ b/src/main/ai-vault-search/session-search-watch-delivery.ts @@ -0,0 +1,85 @@ +import type { SessionSearchInputReceipt } from './session-search-catalog-types' +import type { WatcherProcessEvent } from '../ipc/filesystem/watcher-process/parcel-watcher-process-protocol' + +type Notice = { + identity: string + failed: boolean + events: Map + reconcile: boolean + recovery: boolean +} +export type SessionSearchRootWatch = Notice & { + receipt: (phase: string) => SessionSearchInputReceipt + eventReceipt: (path: string) => SessionSearchInputReceipt + acknowledgeEvent: (path: string) => void + acknowledge: () => void +} + +/** Each consumer retains one unaccepted delivery beside coalesced later arrivals. */ +export class SessionSearchWatchDelivery { + private events = new Map() + private reconcile = false + private recovery = false + private active: SessionSearchRootWatch | undefined + private identity = '' + private failed = false + constructor( + private readonly consumer: string, + private readonly next: () => number + ) {} + get pending(): boolean { + return !!this.active || this.events.size > 0 || this.reconcile || this.recovery + } + stage(notice: Notice): void { + if (this.identity && this.identity !== notice.identity) { + this.reconcile = true + this.recovery = true + } + this.identity = notice.identity + this.failed = notice.failed + this.reconcile ||= notice.reconcile + this.recovery ||= notice.recovery + for (const [path, event] of notice.events) { + if (this.events.size >= 16_384 && !this.events.has(path)) { + this.reconcile = true + this.recovery = true + break + } + this.events.set(path, event) + } + } + capture(): SessionSearchRootWatch { + if (this.active) { + return this.active + } + const revision = this.next() + const events = this.events + const receipts = new Map([...events.keys()].map((path) => [path, this.next()])) + this.events = new Map() + const active: SessionSearchRootWatch = { + identity: this.identity, + failed: this.failed, + events, + reconcile: this.reconcile, + recovery: this.recovery, + receipt: (phase) => ({ consumer: `${this.consumer}:${phase}`, revision }), + eventReceipt: (path) => ({ + consumer: `${this.consumer}:events`, + revision: receipts.get(path)! + }), + acknowledgeEvent: (path) => { + events.delete(path) + receipts.delete(path) + }, + acknowledge: () => { + if (this.active === active) { + this.active = undefined + } + } + } + this.reconcile = false + this.recovery = false + this.active = active + return active + } +} diff --git a/src/main/ai-vault-search/session-search-watcher.test-fixture.ts b/src/main/ai-vault-search/session-search-watcher.test-fixture.ts index 0f90cbfc088..14334052506 100644 --- a/src/main/ai-vault-search/session-search-watcher.test-fixture.ts +++ b/src/main/ai-vault-search/session-search-watcher.test-fixture.ts @@ -64,10 +64,15 @@ export function notifySessionSearchFileChange(path: string): void { } } -export function notifySessionSearchOverflow(path: string): void { +export function notifySessionSearchOverflow(path: string): number { + let delivered = 0 for (const watch of watchers.values()) { if (watch.path === path) { - watch.hooks.onOverflow?.() + if (watch.hooks.onOverflow) { + watch.hooks.onOverflow() + delivered++ + } } } + return delivered } diff --git a/src/main/ai-vault/session-file-inventory-dirty.ts b/src/main/ai-vault/session-file-inventory-dirty.ts new file mode 100644 index 00000000000..2d2597490d0 --- /dev/null +++ b/src/main/ai-vault/session-file-inventory-dirty.ts @@ -0,0 +1,139 @@ +import { SessionInventoryDirectoryWalks } from './session-file-inventory-walk' +import { dirname } from 'node:path' +import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access' +import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation' +import { + isMissingTranscriptPathError, + type DiscoverSessionFilesArgs +} from './session-scanner-discovery' +import { isInsideSessionRoot } from './session-file-discovery' + +type DirtyInput = { path: string; force: boolean; dirtyRevision: number } +type Probe = { + input: DirtyInput + serial: number + revision?: number + result?: { directory: boolean } | { error: unknown } + phase: number + valid?: boolean +} + +/** A successful probe retains its next handoff across catalog failures. */ +export class SessionInventoryDirtyInputs extends SessionInventoryDirectoryWalks { + private probeSequence = 0 + private probe: Probe | undefined + protected get dirtyProbing(): boolean { + return this.probe !== undefined + } + protected async drainDirty(args: DiscoverSessionFilesArgs, limit: number): Promise { + if (this.probe) { + await this.finishProbe(args) + } + const page = await this.request({ kind: 'dirty-page', scope: this.scope }) + const inputs = page.dirtyInputs ?? [] + await this.observations.observePage( + inputs.filter(({ path }) => this.accepts(path, args)), + args, + limit + ) + for (const input of inputs) { + if (this.accepts(input.path, args)) { + continue + } + if (!isInsideSessionRoot(this.root, input.path)) { + await this.acknowledgeDirty(input) + continue + } + this.probe = { input, serial: ++this.probeSequence, phase: 0 } + await this.finishProbe(args) + } + } + private acknowledgeDirty(input: DirtyInput): Promise { + return this.request({ + kind: 'dirty-ack', + scope: this.scope, + path: input.path, + revision: input.dirtyRevision + }) + } + private async finishProbe(args: DiscoverSessionFilesArgs): Promise { + const probe = this.probe! + const { path } = probe.input + const receipt = (phase: string) => ({ + consumer: `dirty-probe:${phase}`, + revision: probe.serial + }) + if (probe.revision === undefined) { + probe.revision = ( + await this.request({ + kind: 'authority-probe', + scope: this.scope, + path, + receipt: receipt('begin') + }) + ).observationEpoch! + } + if (!probe.result) { + try { + const stat = await wslGatedStat(path, 'scan', args.signal) + probe.result = { directory: stat.isDirectory() } + } catch (error) { + throwIfAiVaultScanCancelled(args.signal) + probe.result = { error } + } + } + if (probe.valid === undefined) { + probe.valid = !( + await this.request({ + kind: 'authority-probe-end', + scope: this.scope, + path, + revision: probe.revision, + receipt: receipt('end') + }) + ).invalidProof + } + if (probe.valid) { + if ('error' in probe.result) { + if (isMissingTranscriptPathError(probe.result.error)) { + if (probe.phase === 0) { + await this.schedule(path, false, true, probe.revision) + probe.phase = 1 + } + await this.schedule(dirname(path)) + } else { + await this.issue(path, probe.result.error, args) + } + } else { + if (probe.phase === 0) { + await this.failures.delete(path) + probe.phase = 1 + } + if (probe.result.directory) { + await this.request({ + kind: 'directory-schedule', + scope: this.scope, + path, + depth: this.depth(path), + reconcile: false, + trailing: true, + receipt: receipt('schedule') + }) + } else if (await this.directories.get(path)) { + await this.schedule(dirname(path)) + } + } + } + await this.acknowledgeDirty(probe.input) + this.probe = undefined + } + private depth(path: string): number { + let depth = 0, + key = path + while (key !== this.root && dirname(key) !== key) { + depth++ + key = dirname(key) + } + return depth + } +} diff --git a/src/main/ai-vault/session-file-inventory-files.ts b/src/main/ai-vault/session-file-inventory-files.ts index 407b2fa2cba..8689e1f7099 100644 --- a/src/main/ai-vault/session-file-inventory-files.ts +++ b/src/main/ai-vault/session-file-inventory-files.ts @@ -64,7 +64,13 @@ export class SessionInventoryFiles { ) } async observePage( - inputs: { path: string; force: boolean; readOwed?: boolean; metadataOwed?: boolean }[], + inputs: { + path: string + force: boolean + readOwed?: boolean + metadataOwed?: boolean + dirtyRevision?: number + }[], args: DiscoverSessionFilesArgs, limit: number ): Promise> { diff --git a/src/main/ai-vault/session-file-inventory.ts b/src/main/ai-vault/session-file-inventory.ts index fbdfa6b10d7..eb4ca5b6bc6 100644 --- a/src/main/ai-vault/session-file-inventory.ts +++ b/src/main/ai-vault/session-file-inventory.ts @@ -1,18 +1,15 @@ +import type { SessionSearchInputReceipt } from '../ai-vault-search/session-search-catalog-types' import type { SessionSearchRemovalReceipt } from '../ai-vault-search/session-search-observation-authority' -import { SessionInventoryDirectoryWalks } from './session-file-inventory-walk' -import { dirname } from 'node:path' -import { wslGatedStat } from '../native-chat/wsl-transcript-fs-access' +import { SessionInventoryDirtyInputs } from './session-file-inventory-dirty' import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation' -import { - isMissingTranscriptPathError, - type DiscoverSessionFilesArgs -} from './session-scanner-discovery' -import { isInsideSessionRoot } from './session-file-discovery' +import type { DiscoverSessionFilesArgs } from './session-scanner-discovery' import type { SessionFileChange } from './session-file-inventory-files' export type { SessionFileChange } from './session-file-inventory-files' /** The writer retains queued directories, observations and membership across bounded passes. */ -export class SessionFileInventory extends SessionInventoryDirectoryWalks { +export class SessionFileInventory extends SessionInventoryDirtyInputs { + private wakeSequence = 0 + private waking: SessionSearchInputReceipt | undefined private reconcileAgain = false private reconciling = false private initialized = false @@ -48,6 +45,7 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks { } get pending(): boolean { return ( + this.dirtyProbing || this.observations.applying || this.authority.readySize > 0 || this.garbage.size > 0 || @@ -73,36 +71,75 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks { await this.reconcile() } } - async wakeFailures(): Promise { + async wakeFailures(receipt?: SessionSearchInputReceipt): Promise { + const acceptedReceipt = + receipt ?? (this.waking ??= { consumer: 'explicit-recovery', revision: ++this.wakeSequence }) await this.request({ kind: 'retry-wake', - scope: `${this.scope}:primary-wakes`, - prefix: this.root + scope: this.scope, + prefix: this.root, + all: true, + receipt: acceptedReceipt }) - await this.failures.wake() - await this.observations.failures.wake() + if (!receipt) { + this.waking = undefined + } } get refused(): boolean { return this.failures.size > 0 || this.observations.failures.size > 0 } - get walking(): boolean { - return this.cancellations.size > 0 || this.walks.size > 0 || this.reconcileAgain + get seedDelivered(): boolean { + return !this.observations.delivery.size && !this.observations.applying } - async reconcile(): Promise { + get primarySeedWaiting(): boolean { + return this.primaryWakes.size > 0 && !this.cancellations.size && !this.walks.size + } + get walking(): boolean { + return ( + this.cancellations.size > 0 || + this.walks.size > 0 || + this.reconcileAgain || + this.primarySeedWaiting + ) + } + async reconcile(receipt?: SessionSearchInputReceipt): Promise { + if (this.primarySeedWaiting) { + this.reconcileAgain = true + return + } + if (receipt) { + await this.request({ + kind: 'directory-schedule', + scope: this.scope, + path: this.root, + depth: 0, + reconcile: true, + trailing: true, + receipt + }) + this.reconciling = true + return + } if (this.reconciling) { this.reconcileAgain = true return } - this.reconciling = true await this.schedule(this.root, true) + this.reconciling = true } - async invalidate(path: string, force = true, wake = true): Promise { + async invalidate( + path: string, + force = true, + wake = true, + receipt?: SessionSearchInputReceipt + ): Promise { if (!wake) { await this.dirty.set(path, ((await this.dirty.get(path)) ?? false) || force) return } await this.request({ kind: 'input-invalidate', + receipt, scope: this.scope, path, force, @@ -197,7 +234,7 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks { await this.request({ kind: 'authority-cleanup', scope: this.scope }) } } - if (!this.walks.size && !this.cancellations.size) { + if (!this.walks.size && !this.cancellations.size && !this.primaryWakes.size) { this.reconciling = false if (this.reconcileAgain) { this.reconcileAgain = false @@ -238,57 +275,20 @@ export class SessionFileInventory extends SessionInventoryDirectoryWalks { acknowledge(receipts: { path: string; revision: number }[]): Promise { return this.observations.acknowledge(receipts) } - private async drainDirty(args: DiscoverSessionFilesArgs, limit: number): Promise { - const page = await this.request({ kind: 'dirty-page', scope: this.scope }) - const inputs = page.dirtyInputs ?? [] - await this.observations.observePage( - inputs.filter(({ path }) => this.accepts(path, args)), - args, - limit - ) - for (const { path } of inputs) { - if (this.accepts(path, args) || !isInsideSessionRoot(this.root, path)) { - continue - } - const revision = (await this.request({ kind: 'authority-probe', scope: this.scope, path })) - .observationEpoch! - try { - const stat = await wslGatedStat(path, 'scan', args.signal) - if ( - (await this.request({ kind: 'authority-probe-end', scope: this.scope, path, revision })) - .invalidProof - ) { - continue - } - await this.failures.delete(path) - if (stat.isDirectory()) { - await ((await this.walks.has(path)) ? this.trailing.put(path, true) : this.schedule(path)) - } else if (await this.directories.get(path)) { - await this.schedule(dirname(path)) - } - } catch (error) { - throwIfAiVaultScanCancelled(args.signal) - if ( - (await this.request({ kind: 'authority-probe-end', scope: this.scope, path, revision })) - .invalidProof - ) { - continue - } - if (isMissingTranscriptPathError(error)) { - await this.schedule(path, false, true, revision) - await this.schedule(dirname(path)) - } else { - await this.issue(path, error, args) - } - } - } - } - async retryFailures(): Promise { + async retryFailures(primary = true): Promise { await this.request({ kind: 'retry-admit', + primary, scope: this.scope, root: this.root, generation: this.failures.epoch }) + if (!this.primaryWakes.size && !this.walks.size && !this.cancellations.size) { + this.reconciling = false + if (this.reconcileAgain) { + this.reconcileAgain = false + await this.reconcile() + } + } } }