From 46b9761453d58f896bedc995e49ac2d3b5e2baaf Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:15:16 -0800 Subject: [PATCH] Accept read bookkeeping and delivery receipts together Return the accepted delivery snapshot with its observation and admit successor work with the matching receipt in one writer transaction. Preserve queued snapshots across selection changes and older acknowledgements. Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com> --- config/flaky-test-pattern-baseline.txt | 1 - .../indexing/session-search-pass.ts | 12 ++- ...session-search-candidate-admission-page.ts | 78 +++++++++++++++ .../session-search-candidate-inventory.ts | 7 +- .../session-search-candidate-reason.ts | 37 +++++++ .../session-search-catalog-batching.test.ts | 17 ++-- .../session-search-catalog-candidate-types.ts | 2 + .../session-search-discovery-sources.ts | 14 +-- .../session-search-inventory-transition.ts | 10 +- .../session-search-observation-types.ts | 1 + .../session-search-store-transitions.ts | 99 +------------------ .../session-search-work-catalog.ts | 21 ++-- .../storage/session-search-store.ts | 5 +- .../ai-vault/session-file-inventory-files.ts | 15 ++- src/main/ai-vault/session-file-inventory.ts | 6 +- 15 files changed, 184 insertions(+), 141 deletions(-) create mode 100644 src/main/ai-vault-search/session-search-candidate-admission-page.ts create mode 100644 src/main/ai-vault-search/session-search-candidate-reason.ts diff --git a/config/flaky-test-pattern-baseline.txt b/config/flaky-test-pattern-baseline.txt index 3c85e983054..25f0de2887d 100644 --- a/config/flaky-test-pattern-baseline.txt +++ b/config/flaky-test-pattern-baseline.txt @@ -449,7 +449,6 @@ 4 test-no-wall-clock:src/main/agent-hooks/server/testing/identity/server-launch-resume-identity.test.ts 5 test-no-wall-clock:src/main/agent-hooks/server/testing/replay/spool.test.ts 2 test-no-wall-clock:src/main/agent-hooks/installation/commands/windows-native-hook-exe-hosts.test.ts -1 test-no-wall-clock:src/main/ai-vault-search/indexing/session-search-indexer.test.ts 1 test-no-wall-clock:src/main/ai-vault/parse-cache/session-parse-cache-preview-upgrade.test.ts 2 test-no-wall-clock:src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.test.ts 2 test-no-wall-clock:src/main/ai-vault/scanners/cursor/session-scanner-cursor-chat-meta.test.ts diff --git a/src/main/ai-vault-search/indexing/session-search-pass.ts b/src/main/ai-vault-search/indexing/session-search-pass.ts index b469f626660..f4e3c75f394 100644 --- a/src/main/ai-vault-search/indexing/session-search-pass.ts +++ b/src/main/ai-vault-search/indexing/session-search-pass.ts @@ -234,11 +234,15 @@ export async function runSessionSearchPass( true, observation.metadata ? new Set([path]) : new Set() ) - state.queue.update( - await state.candidates.admitChanges(refreshed.receipts, !args.full, state.recovering) + const admitted = await state.candidates.admitChanges( + refreshed.receipts, + !args.full, + state.recovering, + { path, delivery: observation.receipt } ) - await state.discovery.acknowledgeRead(candidate, rootDir, args.roots, observation.receipt) - return state.queue.get(path) + state.queue.update(admitted) + state.discovery.acceptReadCounts(admitted.scopeCounts) + return admitted.work } }) await state.candidates.reconcilePublications( diff --git a/src/main/ai-vault-search/session-search-candidate-admission-page.ts b/src/main/ai-vault-search/session-search-candidate-admission-page.ts new file mode 100644 index 00000000000..a2db82f2ec3 --- /dev/null +++ b/src/main/ai-vault-search/session-search-candidate-admission-page.ts @@ -0,0 +1,78 @@ +import type { SessionSearchStore } from './storage/session-search-store' +import type { SessionSearchWorkCatalog } from './session-search-work-catalog' +import type { + SessionSearchCandidateCommand, + SessionSearchCandidateResult +} from './session-search-catalog-candidates' +import { requireSessionSearchCandidateReason } from './session-search-candidate-reason' + +/** Known successor work and its exact delivery receipt accept together, before the actual read. */ +export function admitSessionSearchCandidatePage( + store: SessionSearchStore, + queue: SessionSearchWorkCatalog, + command: Extract +): SessionSearchCandidateResult { + return store.catalogStorage.transaction('search_candidate_admit', () => { + const scopeCounts: [string, number, number][] = [] + for (const receipt of command.rows) { + let row = store.catalogCandidates.entry(receipt.key) + if (!row || row.revision !== receipt.revision) { + continue + } + if (row.selected) { + const debt = store.catalogStorage.execute({ + kind: 'get', + scope: 'discovery-metadata', + key: row.candidate.file.path + }).rows?.[0] + if (debt) { + requireSessionSearchCandidateReason(store, row.candidate.file.path, 'metadata') + row = store.catalogCandidates.entry(receipt.key)! + const accepted = store.catalogStorage.response({ + kind: 'ack-values', + scope: 'discovery-metadata', + rows: [{ key: debt.key, value: debt.value }] + }) + scopeCounts.push(...(accepted.scopeCounts ?? [])) + } + queue.enqueue( + row.candidate, + store, + command.recovering || row.forced, + command.recent, + row.metadata + ) + } else if ( + !store.catalogCandidates + .execute({ kind: 'path', path: row.candidate.file.path }) + .rows.some((member) => member.selected) + ) { + queue.removeAccepted(row.candidate.file.path) + } + store.catalogCandidates.execute({ + kind: 'consumed', + key: receipt.key, + revision: row.revision + }) + } + const delivery = command.read?.delivery + if (delivery) { + const accepted = store.catalogStorage.response({ + kind: 'observe-ack', + scope: delivery.scope, + receipts: [{ path: delivery.path, revision: delivery.revision }] + }) + scopeCounts.push(...(accepted.scopeCounts ?? [])) + } + return { + ...store.catalogCandidates.execute({ kind: 'batch', rows: [] }), + scopeCounts, + work: { + size: queue.size, + left: queue.left, + revision: queue.revision, + work: command.read ? queue.get(command.read.path) : undefined + } + } + }) +} diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index 9b9c14662ca..add7ee4f046 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -1,5 +1,6 @@ import type { SessionSearchContainerRemoval } from './session-search-container-transition' import type { SessionSearchRemovalReceipt } from './session-search-observation-authority' +import type { SessionSearchCandidateCommand } from './session-search-catalog-candidates' import type { SessionSearchCatalogCandidate } from './session-search-catalog-candidates' import type { SessionSearchWorkResult } from './session-search-work-queue' import { sessionSearchRecordPages } from './session-search-catalog-pages' @@ -75,15 +76,17 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli async admitChanges( receipts: { key: string; revision: number }[], recent: boolean, - recovering: boolean + recovering: boolean, + read?: Extract['read'] ) { const response = await this.request!({ kind: 'admit', + read, recent, recovering, rows: receipts }) - return response.work! + return { ...response.work!, scopeCounts: response.scopeCounts } } async acknowledgeChanges(receipts: { key: string; revision: number }[]): Promise { for (const row of receipts) { diff --git a/src/main/ai-vault-search/session-search-candidate-reason.ts b/src/main/ai-vault-search/session-search-candidate-reason.ts new file mode 100644 index 00000000000..60ec7c2d242 --- /dev/null +++ b/src/main/ai-vault-search/session-search-candidate-reason.ts @@ -0,0 +1,37 @@ +import type { SessionSearchStore } from './storage/session-search-store' +import { wakeSessionSearchMetadataWrite } from './session-search-store-write' + +function wakeFailedFile(store: SessionSearchStore, path: string): void { + const previous = store.fileRow(path) + if (previous?.state !== 'failed') { + return + } + store.setFileState(path, 'current') + const row = store.fileRow(path) + if (row) { + store.onSourceWake(previous, row) + } +} + +export function requireSessionSearchCandidateReason( + store: SessionSearchStore, + path: string, + reason: 'force' | 'metadata' +): boolean { + const selected = store.catalogCandidates + .execute({ kind: 'path', path }) + .rows.some((row) => row.selected) + if (selected) { + if ( + reason === 'metadata' && + wakeSessionSearchMetadataWrite(store.connection, path, store.fileRow(path)) + ) { + wakeFailedFile(store, path) + } + if (reason === 'force' && store.fileRow(path)?.state === 'failed') { + wakeFailedFile(store, path) + } + store.catalogCandidates.execute({ kind: 'require', path, reason, delivered: true }) + } + return selected +} diff --git a/src/main/ai-vault-search/session-search-catalog-batching.test.ts b/src/main/ai-vault-search/session-search-catalog-batching.test.ts index c521ae0ff97..29cbca38465 100644 --- a/src/main/ai-vault-search/session-search-catalog-batching.test.ts +++ b/src/main/ai-vault-search/session-search-catalog-batching.test.ts @@ -493,18 +493,15 @@ it.each([40, 400])( queue.beginPass() const first = await queue.next() expect(first?.candidate.file.path).toBe(candidates[0].file.path) + const successor = { + ...candidates[0], + file: { ...candidates[0].file, mtimeMs: 20000, modifiedAt: '1970-01-01T00:00:20.000Z' } + } await store.candidates({ kind: 'batch', rows: [ { - candidate: { - ...candidates[0], - file: { - ...candidates[0].file, - mtimeMs: 20000, - modifiedAt: '1970-01-01T00:00:20.000Z' - } - }, + candidate: successor, root: prefix, group: candidates[0].file.path, alias: candidates[0].file.path, @@ -548,10 +545,10 @@ it.each([40, 400])( queue.beginPass() const retry = await queue.next() expect(retry?.candidate.file.path).toBe(candidates[0].file.path) - await queue.enqueue(candidates[0], store, false, true, true) + await queue.enqueue(successor, store, false, true, true) await queue.acknowledge(retry!, false) const latest = await queue.next() - expect(latest).toMatchObject({ candidate: candidates[0], metadata: true }) + expect(latest).toMatchObject({ candidate: successor, metadata: true }) await queue.acknowledge(latest!, false) expect(await queue.next()).toBeUndefined() expect(largestResponse).toBeLessThanOrEqual(256 * 1024) diff --git a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts index 4394d467596..4d4d1f91fa7 100644 --- a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts @@ -26,6 +26,7 @@ export type SessionSearchCandidateCommand = rows: { key: string; revision: number }[] recent: boolean recovering: boolean + read?: { path: string; delivery?: { scope: string; path: string; revision: number } } } | { kind: 'put'; row: SessionSearchCatalogCandidate } | { @@ -71,4 +72,5 @@ export type SessionSearchCandidateResult = SessionSearchPublicationResult & { contested?: string[] work?: SessionSearchWorkResult unowned?: string[] + scopeCounts?: [string, number, number][] } diff --git a/src/main/ai-vault-search/session-search-discovery-sources.ts b/src/main/ai-vault-search/session-search-discovery-sources.ts index 8d576f3b0e3..d566b670b24 100644 --- a/src/main/ai-vault-search/session-search-discovery-sources.ts +++ b/src/main/ai-vault-search/session-search-discovery-sources.ts @@ -1,6 +1,7 @@ import type { SessionSearchRemovalReceipt } from './session-search-observation-authority' import { SessionSearchCatalogClient, + recordSessionSearchCatalogCounts, type SessionSearchCatalogRequest } from './session-search-catalog-client' import type { SessionSearchDependencyInventory } from './session-search-dependency-inventory' @@ -202,17 +203,10 @@ export class SessionSearchDiscoverySources { } } - async acknowledgeRead( - candidate: SessionFileCandidate, - logicalRoot: string, - roots: SessionSearchScanRoots, - receipt: { path: string; revision: number } | undefined - ): Promise { - if (!receipt) { - return + acceptReadCounts(scopeCounts: [string, number, number][] | undefined): void { + if (this.request) { + recordSessionSearchCatalogCounts(this.request, { scopeCounts }) } - const root = await this.fileOwner(candidate, logicalRoot, roots) - await root?.inventory.acknowledge([receipt]) } async end(): Promise { diff --git a/src/main/ai-vault-search/session-search-inventory-transition.ts b/src/main/ai-vault-search/session-search-inventory-transition.ts index 1c21422a148..7601b224166 100644 --- a/src/main/ai-vault-search/session-search-inventory-transition.ts +++ b/src/main/ai-vault-search/session-search-inventory-transition.ts @@ -213,13 +213,21 @@ export function transitionSessionSearchInventory( revision: catalog.nextRevision() }) } + const pendingDelivery = decode<{ revision: number; force: boolean; metadata?: boolean }>( + get(delivery, outcome.path) + ) observations.push({ path: outcome.path, file: { ...facts, modifiedAt: new Date(facts.mtimeMs).toISOString() }, changed: force || !same || dependencyChanged || !!metadata, metadata, force: dependencyChanged || (force && same), - readable: !unknown + readable: !unknown, + delivery: pendingDelivery && { + revision: pendingDelivery.revision, + force: pendingDelivery.force, + metadata: pendingDelivery.metadata + } }) } finally { authority.settle(outcome.path) diff --git a/src/main/ai-vault-search/session-search-observation-types.ts b/src/main/ai-vault-search/session-search-observation-types.ts index eaa46226279..95e7e463049 100644 --- a/src/main/ai-vault-search/session-search-observation-types.ts +++ b/src/main/ai-vault-search/session-search-observation-types.ts @@ -57,6 +57,7 @@ export type SessionSearchInventoryResult = { force: boolean readable: boolean metadata?: boolean + delivery?: { revision: number; force: boolean; metadata?: boolean } }[] filesDelta?: number failuresDelta?: number diff --git a/src/main/ai-vault-search/session-search-store-transitions.ts b/src/main/ai-vault-search/session-search-store-transitions.ts index d68baf9c6d9..8d41782e486 100644 --- a/src/main/ai-vault-search/session-search-store-transitions.ts +++ b/src/main/ai-vault-search/session-search-store-transitions.ts @@ -1,7 +1,7 @@ +export { requireSessionSearchCandidateReason } from './session-search-candidate-reason' +import { admitSessionSearchCandidatePage } from './session-search-candidate-admission-page' import { validSessionSearchContainerRemoval } from './session-search-container-transition' import { SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT } from './indexing/session-search-read-decision' -import { recordSessionSearchPrimaryFailure } from './session-search-primary-retries' -import { wakeSessionSearchMetadataWrite } from './session-search-store-write' import { stepSessionSearchPublication } from './session-search-publication-step' import type { SessionSearchWorkCatalog } from './session-search-work-catalog' import type { SessionSearchWorkCommand, SessionSearchWorkResult } from './session-search-work-queue' @@ -11,41 +11,6 @@ import type { } from './session-search-catalog-candidates' import type { SessionSearchStore } from './storage/session-search-store' -function wakeFailedFile(store: SessionSearchStore, path: string): void { - const previous = store.fileRow(path) - if (previous?.state !== 'failed') { - return - } - store.setFileState(path, 'current') - const row = store.fileRow(path) - if (row) { - store.onSourceWake(previous, row) - } -} - -export function requireSessionSearchCandidateReason( - store: SessionSearchStore, - path: string, - reason: 'force' | 'metadata' -): boolean { - const selected = store.catalogCandidates - .execute({ kind: 'path', path }) - .rows.some((row) => row.selected) - if (selected) { - if ( - reason === 'metadata' && - wakeSessionSearchMetadataWrite(store.connection, path, store.fileRow(path)) - ) { - wakeFailedFile(store, path) - } - if (reason === 'force' && store.fileRow(path)?.state === 'failed') { - wakeFailedFile(store, path) - } - store.catalogCandidates.execute({ kind: 'require', path, reason, delivered: true }) - } - return selected -} - export async function transitionSessionSearchCandidates( store: SessionSearchStore, command: SessionSearchCandidateCommand @@ -118,50 +83,7 @@ export async function transitionSessionSearchCandidates( } return result } - for (const receipt of command.rows) { - let row = store.catalogCandidates.entry(receipt.key) - if (!row || row.revision !== receipt.revision) { - continue - } - if (row.selected) { - const debt = ( - await store.catalog({ - kind: 'get', - scope: 'discovery-metadata', - key: row.candidate.file.path - }) - ).rows?.[0] - if (debt) { - requireSessionSearchCandidateReason(store, row.candidate.file.path, 'metadata') - row = store.catalogCandidates.entry(receipt.key)! - await store.catalog({ - kind: 'ack-values', - scope: 'discovery-metadata', - rows: [{ key: debt.key, value: debt.value }] - }) - } - await store.work({ - kind: 'enqueue', - candidate: row.candidate, - force: command.recovering || row.forced, - recent: command.recent, - metadata: row.metadata - }) - } else if ( - !store.catalogCandidates - .execute({ kind: 'path', path: row.candidate.file.path }) - .rows.some((member) => member.selected) - ) { - await store.work({ kind: 'remove', path: row.candidate.file.path }) - } - store.catalogCandidates.execute({ - kind: 'consumed', - key: receipt.key, - revision: row.revision - }) - } - const work = await store.work({ kind: 'stats' }) - return { ...store.catalogCandidates.execute({ kind: 'batch', rows: [] }), work } + return admitSessionSearchCandidatePage(store, store.catalogWork, command) } export async function transitionSessionSearchWork( @@ -174,21 +96,6 @@ export async function transitionSessionSearchWork( let eligibleThrough switch (command.kind) { case 'enqueue': - if (store.fileRow(command.candidate.file.path)?.state === 'failed') { - recordSessionSearchPrimaryFailure( - store.catalogStorage, - command.candidate.file.path, - true, - true - ) - } - if (command.force) { - store.catalogCandidates.execute({ - kind: 'require', - path: command.candidate.file.path, - reason: 'force' - }) - } await queue.enqueue(command.candidate, store, command.force, command.recent, command.metadata) break case 'get': diff --git a/src/main/ai-vault-search/session-search-work-catalog.ts b/src/main/ai-vault-search/session-search-work-catalog.ts index e5a9609992d..67b52ce825c 100644 --- a/src/main/ai-vault-search/session-search-work-catalog.ts +++ b/src/main/ai-vault-search/session-search-work-catalog.ts @@ -8,10 +8,9 @@ import { sessionSearchCandidateFacts, type SessionSearchFactReference } from './session-search-candidate-facts' -import type { - SessionSearchFileRow, - SessionSearchIndexStore -} from './storage/session-search-store-types' +import type { SessionSearchFileRow } from './storage/session-search-store-types' +import type { SessionSearchStore } from './storage/session-search-store' +import { recordSessionSearchPrimaryFailure } from './session-search-primary-retries' export type SessionSearchWork = { candidate: SessionFileCandidate @@ -60,14 +59,20 @@ export class SessionSearchWorkCatalog { metadata: row.metadata === 1 } } - async enqueue( + enqueue( candidate: SessionFileCandidate, - store: SessionSearchIndexStore, + store: SessionSearchStore, force = false, recent = true, metadata = false - ): Promise { + ): void { const path = candidate.file.path + if (store.fileRow(path)?.state === 'failed') { + recordSessionSearchPrimaryFailure(this.catalog, path, true, true) + } + if (force) { + store.catalogCandidates.execute({ kind: 'require', path, reason: 'force' }) + } const row = store.fileRow(path) const decision = sessionSearchReadDecision({ candidate, @@ -159,7 +164,7 @@ export class SessionSearchWorkCatalog { } return works } - async get(path: string): Promise { + get(path: string): SessionSearchWork | undefined { const row = this.entry(path) return row && this.project(row) } diff --git a/src/main/ai-vault-search/storage/session-search-store.ts b/src/main/ai-vault-search/storage/session-search-store.ts index 5deff0e02ba..58d3dcb5451 100644 --- a/src/main/ai-vault-search/storage/session-search-store.ts +++ b/src/main/ai-vault-search/storage/session-search-store.ts @@ -154,8 +154,11 @@ export class SessionSearchStore { return transitionSessionSearchCandidates(this, command) } work(command: SessionSearchWorkCommand): Promise { + return transitionSessionSearchWork(this, this.catalogWork, command) + } + get catalogWork(): SessionSearchWorkCatalog { this.workCatalog ??= new SessionSearchWorkCatalog(this.db, this.catalogStorage) - return transitionSessionSearchWork(this, this.workCatalog, command) + return this.workCatalog } onSourceWake(previous: SessionSearchFileRow, row: SessionSearchFileRow): void { this.effects.wake(previous, row) diff --git a/src/main/ai-vault/session-file-inventory-files.ts b/src/main/ai-vault/session-file-inventory-files.ts index 8689e1f7099..f9fbba9cbc4 100644 --- a/src/main/ai-vault/session-file-inventory-files.ts +++ b/src/main/ai-vault/session-file-inventory-files.ts @@ -15,6 +15,11 @@ import { recordSessionScanIssue } from './session-scan-issues' import { retainRecentSessionFile } from './session-file-observation' import { sidecarEnrichesWithoutReparse } from './parse-cache/session-scanner-sidecar-enrichment' +type ReadableObservation = { + file: FileWithMtime + delivery?: { revision: number; force: boolean; metadata?: boolean } +} + export type SessionFileChange = { file: FileWithMtime; force: boolean; metadata?: boolean } /** Accepted file facts and retry debt survive both directory epochs and delivered events. */ @@ -73,7 +78,7 @@ export class SessionInventoryFiles { }[], args: DiscoverSessionFilesArgs, limit: number - ): Promise> { + ): Promise> { this.handoffs.clear() const readable = await this.resume(limit) for (const page of sessionSearchRecordPages(inputs)) { @@ -101,7 +106,7 @@ export class SessionInventoryFiles { prepared: SessionSearchPreparedObservation[], args: DiscoverSessionFilesArgs, limit: number - ): Promise> { + ): Promise> { const outcomes: SessionSearchObservationOutcome[] = [] for (const input of prepared) { if (!input.attempt) { @@ -152,8 +157,8 @@ export class SessionInventoryFiles { get applying(): boolean { return !!this.pending } - async resume(limit: number): Promise> { - const readable = new Map() + async resume(limit: number): Promise> { + const readable = new Map() while (this.pending?.outcomes.length) { const page = sessionSearchRecordPages(this.pending.outcomes).next().value! const accepted = await this.files.transition({ @@ -180,7 +185,7 @@ export class SessionInventoryFiles { } retainRecentSessionFile(this.recent, file, limit) if (observation.readable) { - readable.set(observation.path, file) + readable.set(observation.path, { file, delivery: observation.delivery }) } } } diff --git a/src/main/ai-vault/session-file-inventory.ts b/src/main/ai-vault/session-file-inventory.ts index 09c53787011..59d6488b7d5 100644 --- a/src/main/ai-vault/session-file-inventory.ts +++ b/src/main/ai-vault/session-file-inventory.ts @@ -164,7 +164,7 @@ export class SessionFileInventory extends SessionInventoryDirtyInputs { metadataOnly = false, metadataOwed = false ) { - const file = await this.observations.observe( + const observed = await this.observations.observe( path, args, false, @@ -174,14 +174,14 @@ export class SessionFileInventory extends SessionInventoryDirtyInputs { ) const change = this.observations.changes.get(path) this.observations.changes.delete(path) - const delivery = await this.observations.delivery.get(path) + const { file, delivery } = observed ?? {} return file ? { file, force: change?.force || delivery?.force || false, metadata: change?.metadata || delivery?.metadata, changed: change !== undefined || !!delivery, - receipt: delivery ? { path, revision: delivery.revision } : undefined + receipt: delivery ? { scope: this.scope, path, revision: delivery.revision } : undefined } : { handoff: this.observations.handoffs.has(path)