diff --git a/config/flaky-test-pattern-baseline.txt b/config/flaky-test-pattern-baseline.txt index 3c85e983054..25f0de2887d 100644 --- a/config/flaky-test-pattern-baseline.txt +++ b/config/flaky-test-pattern-baseline.txt @@ -449,7 +449,6 @@ 4 test-no-wall-clock:src/main/agent-hooks/server/testing/identity/server-launch-resume-identity.test.ts 5 test-no-wall-clock:src/main/agent-hooks/server/testing/replay/spool.test.ts 2 test-no-wall-clock:src/main/agent-hooks/installation/commands/windows-native-hook-exe-hosts.test.ts -1 test-no-wall-clock:src/main/ai-vault-search/indexing/session-search-indexer.test.ts 1 test-no-wall-clock:src/main/ai-vault/parse-cache/session-parse-cache-preview-upgrade.test.ts 2 test-no-wall-clock:src/main/ai-vault/scanners/copilot/session-scanner-copilot-parser.test.ts 2 test-no-wall-clock:src/main/ai-vault/scanners/cursor/session-scanner-cursor-chat-meta.test.ts diff --git a/src/main/ai-vault-search/indexing/session-search-pass.ts b/src/main/ai-vault-search/indexing/session-search-pass.ts index b469f626660..f4e3c75f394 100644 --- a/src/main/ai-vault-search/indexing/session-search-pass.ts +++ b/src/main/ai-vault-search/indexing/session-search-pass.ts @@ -234,11 +234,15 @@ export async function runSessionSearchPass( true, observation.metadata ? new Set([path]) : new Set() ) - state.queue.update( - await state.candidates.admitChanges(refreshed.receipts, !args.full, state.recovering) + const admitted = await state.candidates.admitChanges( + refreshed.receipts, + !args.full, + state.recovering, + { path, delivery: observation.receipt } ) - await state.discovery.acknowledgeRead(candidate, rootDir, args.roots, observation.receipt) - return state.queue.get(path) + state.queue.update(admitted) + state.discovery.acceptReadCounts(admitted.scopeCounts) + return admitted.work } }) await state.candidates.reconcilePublications( diff --git a/src/main/ai-vault-search/session-search-candidate-admission-page.ts b/src/main/ai-vault-search/session-search-candidate-admission-page.ts new file mode 100644 index 00000000000..a2db82f2ec3 --- /dev/null +++ b/src/main/ai-vault-search/session-search-candidate-admission-page.ts @@ -0,0 +1,78 @@ +import type { SessionSearchStore } from './storage/session-search-store' +import type { SessionSearchWorkCatalog } from './session-search-work-catalog' +import type { + SessionSearchCandidateCommand, + SessionSearchCandidateResult +} from './session-search-catalog-candidates' +import { requireSessionSearchCandidateReason } from './session-search-candidate-reason' + +/** Known successor work and its exact delivery receipt accept together, before the actual read. */ +export function admitSessionSearchCandidatePage( + store: SessionSearchStore, + queue: SessionSearchWorkCatalog, + command: Extract +): SessionSearchCandidateResult { + return store.catalogStorage.transaction('search_candidate_admit', () => { + const scopeCounts: [string, number, number][] = [] + for (const receipt of command.rows) { + let row = store.catalogCandidates.entry(receipt.key) + if (!row || row.revision !== receipt.revision) { + continue + } + if (row.selected) { + const debt = store.catalogStorage.execute({ + kind: 'get', + scope: 'discovery-metadata', + key: row.candidate.file.path + }).rows?.[0] + if (debt) { + requireSessionSearchCandidateReason(store, row.candidate.file.path, 'metadata') + row = store.catalogCandidates.entry(receipt.key)! + const accepted = store.catalogStorage.response({ + kind: 'ack-values', + scope: 'discovery-metadata', + rows: [{ key: debt.key, value: debt.value }] + }) + scopeCounts.push(...(accepted.scopeCounts ?? [])) + } + queue.enqueue( + row.candidate, + store, + command.recovering || row.forced, + command.recent, + row.metadata + ) + } else if ( + !store.catalogCandidates + .execute({ kind: 'path', path: row.candidate.file.path }) + .rows.some((member) => member.selected) + ) { + queue.removeAccepted(row.candidate.file.path) + } + store.catalogCandidates.execute({ + kind: 'consumed', + key: receipt.key, + revision: row.revision + }) + } + const delivery = command.read?.delivery + if (delivery) { + const accepted = store.catalogStorage.response({ + kind: 'observe-ack', + scope: delivery.scope, + receipts: [{ path: delivery.path, revision: delivery.revision }] + }) + scopeCounts.push(...(accepted.scopeCounts ?? [])) + } + return { + ...store.catalogCandidates.execute({ kind: 'batch', rows: [] }), + scopeCounts, + work: { + size: queue.size, + left: queue.left, + revision: queue.revision, + work: command.read ? queue.get(command.read.path) : undefined + } + } + }) +} diff --git a/src/main/ai-vault-search/session-search-candidate-inventory.ts b/src/main/ai-vault-search/session-search-candidate-inventory.ts index 9b9c14662ca..add7ee4f046 100644 --- a/src/main/ai-vault-search/session-search-candidate-inventory.ts +++ b/src/main/ai-vault-search/session-search-candidate-inventory.ts @@ -1,5 +1,6 @@ import type { SessionSearchContainerRemoval } from './session-search-container-transition' import type { SessionSearchRemovalReceipt } from './session-search-observation-authority' +import type { SessionSearchCandidateCommand } from './session-search-catalog-candidates' import type { SessionSearchCatalogCandidate } from './session-search-catalog-candidates' import type { SessionSearchWorkResult } from './session-search-work-queue' import { sessionSearchRecordPages } from './session-search-catalog-pages' @@ -75,15 +76,17 @@ export class SessionSearchCandidateInventory extends SessionSearchCandidatePubli async admitChanges( receipts: { key: string; revision: number }[], recent: boolean, - recovering: boolean + recovering: boolean, + read?: Extract['read'] ) { const response = await this.request!({ kind: 'admit', + read, recent, recovering, rows: receipts }) - return response.work! + return { ...response.work!, scopeCounts: response.scopeCounts } } async acknowledgeChanges(receipts: { key: string; revision: number }[]): Promise { for (const row of receipts) { diff --git a/src/main/ai-vault-search/session-search-candidate-reason.ts b/src/main/ai-vault-search/session-search-candidate-reason.ts new file mode 100644 index 00000000000..60ec7c2d242 --- /dev/null +++ b/src/main/ai-vault-search/session-search-candidate-reason.ts @@ -0,0 +1,37 @@ +import type { SessionSearchStore } from './storage/session-search-store' +import { wakeSessionSearchMetadataWrite } from './session-search-store-write' + +function wakeFailedFile(store: SessionSearchStore, path: string): void { + const previous = store.fileRow(path) + if (previous?.state !== 'failed') { + return + } + store.setFileState(path, 'current') + const row = store.fileRow(path) + if (row) { + store.onSourceWake(previous, row) + } +} + +export function requireSessionSearchCandidateReason( + store: SessionSearchStore, + path: string, + reason: 'force' | 'metadata' +): boolean { + const selected = store.catalogCandidates + .execute({ kind: 'path', path }) + .rows.some((row) => row.selected) + if (selected) { + if ( + reason === 'metadata' && + wakeSessionSearchMetadataWrite(store.connection, path, store.fileRow(path)) + ) { + wakeFailedFile(store, path) + } + if (reason === 'force' && store.fileRow(path)?.state === 'failed') { + wakeFailedFile(store, path) + } + store.catalogCandidates.execute({ kind: 'require', path, reason, delivered: true }) + } + return selected +} diff --git a/src/main/ai-vault-search/session-search-catalog-batching.test.ts b/src/main/ai-vault-search/session-search-catalog-batching.test.ts index c521ae0ff97..29cbca38465 100644 --- a/src/main/ai-vault-search/session-search-catalog-batching.test.ts +++ b/src/main/ai-vault-search/session-search-catalog-batching.test.ts @@ -493,18 +493,15 @@ it.each([40, 400])( queue.beginPass() const first = await queue.next() expect(first?.candidate.file.path).toBe(candidates[0].file.path) + const successor = { + ...candidates[0], + file: { ...candidates[0].file, mtimeMs: 20000, modifiedAt: '1970-01-01T00:00:20.000Z' } + } await store.candidates({ kind: 'batch', rows: [ { - candidate: { - ...candidates[0], - file: { - ...candidates[0].file, - mtimeMs: 20000, - modifiedAt: '1970-01-01T00:00:20.000Z' - } - }, + candidate: successor, root: prefix, group: candidates[0].file.path, alias: candidates[0].file.path, @@ -548,10 +545,10 @@ it.each([40, 400])( queue.beginPass() const retry = await queue.next() expect(retry?.candidate.file.path).toBe(candidates[0].file.path) - await queue.enqueue(candidates[0], store, false, true, true) + await queue.enqueue(successor, store, false, true, true) await queue.acknowledge(retry!, false) const latest = await queue.next() - expect(latest).toMatchObject({ candidate: candidates[0], metadata: true }) + expect(latest).toMatchObject({ candidate: successor, metadata: true }) await queue.acknowledge(latest!, false) expect(await queue.next()).toBeUndefined() expect(largestResponse).toBeLessThanOrEqual(256 * 1024) diff --git a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts index 4394d467596..4d4d1f91fa7 100644 --- a/src/main/ai-vault-search/session-search-catalog-candidate-types.ts +++ b/src/main/ai-vault-search/session-search-catalog-candidate-types.ts @@ -26,6 +26,7 @@ export type SessionSearchCandidateCommand = rows: { key: string; revision: number }[] recent: boolean recovering: boolean + read?: { path: string; delivery?: { scope: string; path: string; revision: number } } } | { kind: 'put'; row: SessionSearchCatalogCandidate } | { @@ -71,4 +72,5 @@ export type SessionSearchCandidateResult = SessionSearchPublicationResult & { contested?: string[] work?: SessionSearchWorkResult unowned?: string[] + scopeCounts?: [string, number, number][] } diff --git a/src/main/ai-vault-search/session-search-discovery-sources.ts b/src/main/ai-vault-search/session-search-discovery-sources.ts index 8d576f3b0e3..d566b670b24 100644 --- a/src/main/ai-vault-search/session-search-discovery-sources.ts +++ b/src/main/ai-vault-search/session-search-discovery-sources.ts @@ -1,6 +1,7 @@ import type { SessionSearchRemovalReceipt } from './session-search-observation-authority' import { SessionSearchCatalogClient, + recordSessionSearchCatalogCounts, type SessionSearchCatalogRequest } from './session-search-catalog-client' import type { SessionSearchDependencyInventory } from './session-search-dependency-inventory' @@ -202,17 +203,10 @@ export class SessionSearchDiscoverySources { } } - async acknowledgeRead( - candidate: SessionFileCandidate, - logicalRoot: string, - roots: SessionSearchScanRoots, - receipt: { path: string; revision: number } | undefined - ): Promise { - if (!receipt) { - return + acceptReadCounts(scopeCounts: [string, number, number][] | undefined): void { + if (this.request) { + recordSessionSearchCatalogCounts(this.request, { scopeCounts }) } - const root = await this.fileOwner(candidate, logicalRoot, roots) - await root?.inventory.acknowledge([receipt]) } async end(): Promise { diff --git a/src/main/ai-vault-search/session-search-inventory-transition.ts b/src/main/ai-vault-search/session-search-inventory-transition.ts index 1c21422a148..7601b224166 100644 --- a/src/main/ai-vault-search/session-search-inventory-transition.ts +++ b/src/main/ai-vault-search/session-search-inventory-transition.ts @@ -213,13 +213,21 @@ export function transitionSessionSearchInventory( revision: catalog.nextRevision() }) } + const pendingDelivery = decode<{ revision: number; force: boolean; metadata?: boolean }>( + get(delivery, outcome.path) + ) observations.push({ path: outcome.path, file: { ...facts, modifiedAt: new Date(facts.mtimeMs).toISOString() }, changed: force || !same || dependencyChanged || !!metadata, metadata, force: dependencyChanged || (force && same), - readable: !unknown + readable: !unknown, + delivery: pendingDelivery && { + revision: pendingDelivery.revision, + force: pendingDelivery.force, + metadata: pendingDelivery.metadata + } }) } finally { authority.settle(outcome.path) diff --git a/src/main/ai-vault-search/session-search-observation-types.ts b/src/main/ai-vault-search/session-search-observation-types.ts index eaa46226279..95e7e463049 100644 --- a/src/main/ai-vault-search/session-search-observation-types.ts +++ b/src/main/ai-vault-search/session-search-observation-types.ts @@ -57,6 +57,7 @@ export type SessionSearchInventoryResult = { force: boolean readable: boolean metadata?: boolean + delivery?: { revision: number; force: boolean; metadata?: boolean } }[] filesDelta?: number failuresDelta?: number diff --git a/src/main/ai-vault-search/session-search-store-transitions.ts b/src/main/ai-vault-search/session-search-store-transitions.ts index d68baf9c6d9..8d41782e486 100644 --- a/src/main/ai-vault-search/session-search-store-transitions.ts +++ b/src/main/ai-vault-search/session-search-store-transitions.ts @@ -1,7 +1,7 @@ +export { requireSessionSearchCandidateReason } from './session-search-candidate-reason' +import { admitSessionSearchCandidatePage } from './session-search-candidate-admission-page' import { validSessionSearchContainerRemoval } from './session-search-container-transition' import { SESSION_SEARCH_FAILURES_BEFORE_HELD_OUT } from './indexing/session-search-read-decision' -import { recordSessionSearchPrimaryFailure } from './session-search-primary-retries' -import { wakeSessionSearchMetadataWrite } from './session-search-store-write' import { stepSessionSearchPublication } from './session-search-publication-step' import type { SessionSearchWorkCatalog } from './session-search-work-catalog' import type { SessionSearchWorkCommand, SessionSearchWorkResult } from './session-search-work-queue' @@ -11,41 +11,6 @@ import type { } from './session-search-catalog-candidates' import type { SessionSearchStore } from './storage/session-search-store' -function wakeFailedFile(store: SessionSearchStore, path: string): void { - const previous = store.fileRow(path) - if (previous?.state !== 'failed') { - return - } - store.setFileState(path, 'current') - const row = store.fileRow(path) - if (row) { - store.onSourceWake(previous, row) - } -} - -export function requireSessionSearchCandidateReason( - store: SessionSearchStore, - path: string, - reason: 'force' | 'metadata' -): boolean { - const selected = store.catalogCandidates - .execute({ kind: 'path', path }) - .rows.some((row) => row.selected) - if (selected) { - if ( - reason === 'metadata' && - wakeSessionSearchMetadataWrite(store.connection, path, store.fileRow(path)) - ) { - wakeFailedFile(store, path) - } - if (reason === 'force' && store.fileRow(path)?.state === 'failed') { - wakeFailedFile(store, path) - } - store.catalogCandidates.execute({ kind: 'require', path, reason, delivered: true }) - } - return selected -} - export async function transitionSessionSearchCandidates( store: SessionSearchStore, command: SessionSearchCandidateCommand @@ -118,50 +83,7 @@ export async function transitionSessionSearchCandidates( } return result } - for (const receipt of command.rows) { - let row = store.catalogCandidates.entry(receipt.key) - if (!row || row.revision !== receipt.revision) { - continue - } - if (row.selected) { - const debt = ( - await store.catalog({ - kind: 'get', - scope: 'discovery-metadata', - key: row.candidate.file.path - }) - ).rows?.[0] - if (debt) { - requireSessionSearchCandidateReason(store, row.candidate.file.path, 'metadata') - row = store.catalogCandidates.entry(receipt.key)! - await store.catalog({ - kind: 'ack-values', - scope: 'discovery-metadata', - rows: [{ key: debt.key, value: debt.value }] - }) - } - await store.work({ - kind: 'enqueue', - candidate: row.candidate, - force: command.recovering || row.forced, - recent: command.recent, - metadata: row.metadata - }) - } else if ( - !store.catalogCandidates - .execute({ kind: 'path', path: row.candidate.file.path }) - .rows.some((member) => member.selected) - ) { - await store.work({ kind: 'remove', path: row.candidate.file.path }) - } - store.catalogCandidates.execute({ - kind: 'consumed', - key: receipt.key, - revision: row.revision - }) - } - const work = await store.work({ kind: 'stats' }) - return { ...store.catalogCandidates.execute({ kind: 'batch', rows: [] }), work } + return admitSessionSearchCandidatePage(store, store.catalogWork, command) } export async function transitionSessionSearchWork( @@ -174,21 +96,6 @@ export async function transitionSessionSearchWork( let eligibleThrough switch (command.kind) { case 'enqueue': - if (store.fileRow(command.candidate.file.path)?.state === 'failed') { - recordSessionSearchPrimaryFailure( - store.catalogStorage, - command.candidate.file.path, - true, - true - ) - } - if (command.force) { - store.catalogCandidates.execute({ - kind: 'require', - path: command.candidate.file.path, - reason: 'force' - }) - } await queue.enqueue(command.candidate, store, command.force, command.recent, command.metadata) break case 'get': diff --git a/src/main/ai-vault-search/session-search-work-catalog.ts b/src/main/ai-vault-search/session-search-work-catalog.ts index e5a9609992d..67b52ce825c 100644 --- a/src/main/ai-vault-search/session-search-work-catalog.ts +++ b/src/main/ai-vault-search/session-search-work-catalog.ts @@ -8,10 +8,9 @@ import { sessionSearchCandidateFacts, type SessionSearchFactReference } from './session-search-candidate-facts' -import type { - SessionSearchFileRow, - SessionSearchIndexStore -} from './storage/session-search-store-types' +import type { SessionSearchFileRow } from './storage/session-search-store-types' +import type { SessionSearchStore } from './storage/session-search-store' +import { recordSessionSearchPrimaryFailure } from './session-search-primary-retries' export type SessionSearchWork = { candidate: SessionFileCandidate @@ -60,14 +59,20 @@ export class SessionSearchWorkCatalog { metadata: row.metadata === 1 } } - async enqueue( + enqueue( candidate: SessionFileCandidate, - store: SessionSearchIndexStore, + store: SessionSearchStore, force = false, recent = true, metadata = false - ): Promise { + ): void { const path = candidate.file.path + if (store.fileRow(path)?.state === 'failed') { + recordSessionSearchPrimaryFailure(this.catalog, path, true, true) + } + if (force) { + store.catalogCandidates.execute({ kind: 'require', path, reason: 'force' }) + } const row = store.fileRow(path) const decision = sessionSearchReadDecision({ candidate, @@ -159,7 +164,7 @@ export class SessionSearchWorkCatalog { } return works } - async get(path: string): Promise { + get(path: string): SessionSearchWork | undefined { const row = this.entry(path) return row && this.project(row) } diff --git a/src/main/ai-vault-search/storage/session-search-store.ts b/src/main/ai-vault-search/storage/session-search-store.ts index 5deff0e02ba..58d3dcb5451 100644 --- a/src/main/ai-vault-search/storage/session-search-store.ts +++ b/src/main/ai-vault-search/storage/session-search-store.ts @@ -154,8 +154,11 @@ export class SessionSearchStore { return transitionSessionSearchCandidates(this, command) } work(command: SessionSearchWorkCommand): Promise { + return transitionSessionSearchWork(this, this.catalogWork, command) + } + get catalogWork(): SessionSearchWorkCatalog { this.workCatalog ??= new SessionSearchWorkCatalog(this.db, this.catalogStorage) - return transitionSessionSearchWork(this, this.workCatalog, command) + return this.workCatalog } onSourceWake(previous: SessionSearchFileRow, row: SessionSearchFileRow): void { this.effects.wake(previous, row) diff --git a/src/main/ai-vault/session-file-inventory-files.ts b/src/main/ai-vault/session-file-inventory-files.ts index 8689e1f7099..f9fbba9cbc4 100644 --- a/src/main/ai-vault/session-file-inventory-files.ts +++ b/src/main/ai-vault/session-file-inventory-files.ts @@ -15,6 +15,11 @@ import { recordSessionScanIssue } from './session-scan-issues' import { retainRecentSessionFile } from './session-file-observation' import { sidecarEnrichesWithoutReparse } from './parse-cache/session-scanner-sidecar-enrichment' +type ReadableObservation = { + file: FileWithMtime + delivery?: { revision: number; force: boolean; metadata?: boolean } +} + export type SessionFileChange = { file: FileWithMtime; force: boolean; metadata?: boolean } /** Accepted file facts and retry debt survive both directory epochs and delivered events. */ @@ -73,7 +78,7 @@ export class SessionInventoryFiles { }[], args: DiscoverSessionFilesArgs, limit: number - ): Promise> { + ): Promise> { this.handoffs.clear() const readable = await this.resume(limit) for (const page of sessionSearchRecordPages(inputs)) { @@ -101,7 +106,7 @@ export class SessionInventoryFiles { prepared: SessionSearchPreparedObservation[], args: DiscoverSessionFilesArgs, limit: number - ): Promise> { + ): Promise> { const outcomes: SessionSearchObservationOutcome[] = [] for (const input of prepared) { if (!input.attempt) { @@ -152,8 +157,8 @@ export class SessionInventoryFiles { get applying(): boolean { return !!this.pending } - async resume(limit: number): Promise> { - const readable = new Map() + async resume(limit: number): Promise> { + const readable = new Map() while (this.pending?.outcomes.length) { const page = sessionSearchRecordPages(this.pending.outcomes).next().value! const accepted = await this.files.transition({ @@ -180,7 +185,7 @@ export class SessionInventoryFiles { } retainRecentSessionFile(this.recent, file, limit) if (observation.readable) { - readable.set(observation.path, file) + readable.set(observation.path, { file, delivery: observation.delivery }) } } } diff --git a/src/main/ai-vault/session-file-inventory.ts b/src/main/ai-vault/session-file-inventory.ts index 09c53787011..59d6488b7d5 100644 --- a/src/main/ai-vault/session-file-inventory.ts +++ b/src/main/ai-vault/session-file-inventory.ts @@ -164,7 +164,7 @@ export class SessionFileInventory extends SessionInventoryDirtyInputs { metadataOnly = false, metadataOwed = false ) { - const file = await this.observations.observe( + const observed = await this.observations.observe( path, args, false, @@ -174,14 +174,14 @@ export class SessionFileInventory extends SessionInventoryDirtyInputs { ) const change = this.observations.changes.get(path) this.observations.changes.delete(path) - const delivery = await this.observations.delivery.get(path) + const { file, delivery } = observed ?? {} return file ? { file, force: change?.force || delivery?.force || false, metadata: change?.metadata || delivery?.metadata, changed: change !== undefined || !!delivery, - receipt: delivery ? { path, revision: delivery.revision } : undefined + receipt: delivery ? { scope: this.scope, path, revision: delivery.revision } : undefined } : { handoff: this.observations.handoffs.has(path)