fix(search): include undated Cursor sessions in time filters (#31991)

* fix(search): date undated Cursor sessions and retain sidecar metadata * fix(search): truncate repaired Cursor dates to whole milliseconds SQLite rounds sub-millisecond file times while Date#toISOString truncates, so repaired rows could land 1ms after the date a re-read writes. Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
2026-10-30 16:47:34 +00:00 · 2026-10-22 20:30:54 -07:00
parent 8f2f620ae8
commit 0dc7b88358
6 changed files with 289 additions and 32 deletions
@@ -0,0 +1,205 @@
+import { appendFile, mkdir, stat, utimes, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { afterEach, beforeEach, expect, it } from 'vitest'
+import { isAiVaultScanCancelledError } from '../../shared/ai-vault-types'
+import {
+  parseAgentSessionFileCached,
+  resetSessionParseCacheForTests
+} from '../ai-vault/session-scanner-parse-cache'
+import { resetTranscriptConsumersForTests } from '../ai-vault/session-transcript-consumers'
+import { resetCursorChatMetaIndexCacheForTests } from '../ai-vault/session-scanner-cursor-chat-meta'
+import { SessionSearchEngine } from './session-search-engine'
+import { registerSessionSearchIndexConsumer } from './session-search-index-consumer'
+import {
+  openSessionSearchIndexFile,
+  type SessionSearchIndexFile
+} from './session-search-index-test-fixture'
+import { SessionSearchStore } from './session-search-store'
+import { sessionCandidate } from './session-search-transcript-fixtures'
+import { readWholeTranscript } from '../ai-vault/session-transcript-reader'
+
+let index: SessionSearchIndexFile
+let store: SessionSearchStore
+let engine: SessionSearchEngine
+let errors: unknown[]
+const EARLIER = '2026-10-01T10:00:00.125Z'
+const LATER = '2026-10-20T10:00:00.875Z'
+const SINCE = '2026-10-10T00:00:00.000Z'
+
+beforeEach(async () => {
+  resetSessionParseCacheForTests()
+  resetTranscriptConsumersForTests()
+  resetCursorChatMetaIndexCacheForTests()
+  index = await openSessionSearchIndexFile('ss-cursor-timestamps')
+  errors = []
+  store = new SessionSearchStore(index.path, (error) => errors.push(error))
+  registerSessionSearchIndexConsumer(store)
+  engine = new SessionSearchEngine(index.db)
+})
+
+afterEach(async () => {
+  resetTranscriptConsumersForTests()
+  resetSessionParseCacheForTests()
+  resetCursorChatMetaIndexCacheForTests()
+  store.close()
+  await index.close()
+})
+
+async function cursorTranscript(id: string, modifiedAt: string, timestamp?: string) {
+  const path = join(
+    dirname(index.path),
+    '.cursor',
+    'projects',
+    'app',
+    'agent-transcripts',
+    id,
+    `${id}.jsonl`
+  )
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(
+    path,
+    `${JSON.stringify({ role: 'user', message: { content: [{ type: 'text', text: `timestampneedle ${id}` }] }, ...(timestamp ? { timestamp } : {}) })}\n`
+  )
+  await utimes(path, new Date(modifiedAt), new Date(modifiedAt))
+  const candidate = await sessionCandidate('cursor', path)
+  const session = await parseAgentSessionFileCached(candidate, process.platform)
+  expect(errors).toEqual([])
+  return { path, candidate, session }
+}
+
+it('finds undated Cursor transcripts by since and newest using their file times', async () => {
+  const recent = await cursorTranscript('recent', LATER)
+  await cursorTranscript('older', EARLIER)
+  expect(recent.session).toMatchObject({ createdAt: null, updatedAt: null, modifiedAt: LATER })
+
+  const hits = engine.search({
+    query: 'timestampneedle',
+    filters: { agents: ['cursor'], since: SINCE, sort: 'newest' }
+  }).hits
+  expect(hits.map((hit) => [hit.sessionId, hit.updatedAt])).toEqual([['recent', LATER]])
+  expect(
+    engine
+      .search({ query: 'timestampneedle', filters: { sort: 'newest' } })
+      .hits.map((hit) => hit.sessionId)
+  ).toEqual(['recent', 'older'])
+  expect(
+    engine
+      .search({
+        query: 'path:agent-transcripts',
+        filters: { agents: ['cursor'], since: SINCE, sort: 'newest' }
+      })
+      .hits.map((hit) => hit.sessionId)
+  ).toEqual(['recent'])
+  expect(hits[0]?.evidence?.timestamp).toBeNull()
+})
+
+it('advances the fallback after an append without inventing message timestamps', async () => {
+  const { path } = await cursorTranscript('appended', EARLIER)
+  expect(engine.search({ query: 'timestampneedle', filters: { since: SINCE } }).hits).toEqual([])
+  await appendFile(
+    path,
+    `${JSON.stringify({ role: 'assistant', message: { content: [{ type: 'text', text: 'followupneedle' }] } })}\n`
+  )
+  await utimes(path, new Date(LATER), new Date(LATER))
+  await parseAgentSessionFileCached(await sessionCandidate('cursor', path), process.platform)
+  expect(errors).toEqual([])
+  expect(engine.search({ query: 'followupneedle', filters: { since: SINCE } }).hits).toMatchObject([
+    { sessionId: 'appended', updatedAt: LATER, evidence: { timestamp: null } }
+  ])
+})
+
+it('keeps a real transcript timestamp even when its file is newer', async () => {
+  await cursorTranscript('dated', LATER, EARLIER)
+  expect(engine.search({ query: 'timestampneedle', filters: { since: SINCE } }).hits).toEqual([])
+  expect(engine.search({ query: 'timestampneedle' }).hits).toMatchObject([{ updatedAt: EARLIER }])
+})
+
+it('prefers the Cursor metadata timestamp over the transcript file time', async () => {
+  const { path } = await cursorTranscript('sidecar', LATER)
+  const metaPath = join(dirname(index.path), '.cursor', 'chats', 'app', 'sidecar', 'meta.json')
+  await mkdir(dirname(metaPath), { recursive: true })
+  await writeFile(metaPath, JSON.stringify({ updatedAtMs: Date.parse(EARLIER), cwd: '/repo/app' }))
+  resetSessionParseCacheForTests()
+  resetCursorChatMetaIndexCacheForTests()
+  const candidate = await sessionCandidate('cursor', path)
+  const metaStat = await stat(metaPath)
+  candidate.file.sidecar = { path: metaPath, mtimeMs: metaStat.mtimeMs, sizeBytes: metaStat.size }
+  const session = await parseAgentSessionFileCached(candidate, process.platform)
+  expect(session?.updatedAt).toBe(EARLIER)
+  expect(errors).toEqual([])
+  expect(engine.search({ query: 'timestampneedle', filters: { since: SINCE } }).hits).toEqual([])
+  expect(engine.search({ query: 'timestampneedle' }).hits).toMatchObject([
+    { updatedAt: EARLIER, cwd: '/repo/app' }
+  ])
+  await appendFile(
+    path,
+    `${JSON.stringify({ role: 'assistant', message: { content: [{ type: 'text', text: 'sidecarappendneedle' }] } })}\n`
+  )
+  await parseAgentSessionFileCached(await sessionCandidate('cursor', path), process.platform)
+  expect(engine.search({ query: 'sidecarappendneedle' }).hits).toMatchObject([
+    { updatedAt: EARLIER, cwd: '/repo/app' }
+  ])
+})
+
+it('does not commit a read cancelled while completing consumer metadata', async () => {
+  const { path } = await cursorTranscript('cancelled', EARLIER)
+  const before = index.db.prepare('SELECT * FROM files').all()
+  const controller = new AbortController()
+  await expect(
+    readWholeTranscript({
+      candidate: await sessionCandidate('cursor', path),
+      platform: process.platform,
+      signal: controller.signal,
+      sessionForConsumers: async (session) => {
+        controller.abort()
+        return session
+      }
+    })
+  ).rejects.toSatisfy(isAiVaultScanCancelledError)
+  expect(errors).toEqual([])
+  expect(index.db.prepare('SELECT * FROM files').all()).toEqual(before)
+  expect(index.db.prepare('SELECT COUNT(*) AS n FROM messages').get()).toEqual({ n: 1 })
+})
+
+it('repairs existing null Cursor dates on reopen without rebuilding search content', async () => {
+  const { candidate } = await cursorTranscript('cached', LATER)
+  index.db.prepare('UPDATE sessions SET updated_at = NULL').run()
+  // APFS and NTFS record sub-millisecond mtimes; the repair must truncate like Date#toISOString.
+  index.db.prepare('UPDATE files SET mtime_ms = mtime_ms + 0.6').run()
+  const incarnation = index.db
+    .prepare("SELECT value FROM meta WHERE key = 'index_incarnation'")
+    .get()
+  const content = index.db.prepare('SELECT id, session_row_id FROM messages').all()
+  store.close()
+  store = new SessionSearchStore(index.path, (error) => errors.push(error))
+  expect(errors).toEqual([])
+  expect(index.db.prepare("SELECT value FROM meta WHERE key = 'index_incarnation'").get()).toEqual(
+    incarnation
+  )
+  expect(index.db.prepare('SELECT id, session_row_id FROM messages').all()).toEqual(content)
+  expect(index.db.prepare('SELECT byte_offset, mtime_ms FROM files').get()).toEqual({
+    byte_offset: candidate.file.sizeBytes,
+    mtime_ms: candidate.file.mtimeMs + 0.6
+  })
+  expect(engine.search({ query: 'timestampneedle', filters: { since: SINCE } }).hits).toMatchObject(
+    [{ sessionId: 'cached', updatedAt: LATER }]
+  )
+})
+
+it('does not repair an authoritative date or another provider from file metadata', async () => {
+  await cursorTranscript('dated', LATER, EARLIER)
+  await cursorTranscript('other-provider', LATER)
+  index.db
+    .prepare(
+      "UPDATE sessions SET agent = 'claude', updated_at = NULL WHERE session_id = 'other-provider'"
+    )
+    .run()
+  store.close()
+  store = new SessionSearchStore(index.path, (error) => errors.push(error))
+  expect(
+    index.db.prepare('SELECT session_id, updated_at FROM sessions ORDER BY session_id').all()
+  ).toEqual([
+    { session_id: 'dated', updated_at: EARLIER },
+    { session_id: 'other-provider', updated_at: null }
+  ])
+})
@@ -51,7 +51,11 @@ export class SessionSearchFileRecords {
   * does not chunk at all. The final commit overwrites all of it from the
   * decoded session; until then the title in particular is provisional.
   */
-  updateProvisionalSession(rowId: number, identity: TranscriptSessionIdentity): void {
+  updateProvisionalSession(
+    rowId: number,
+    identity: TranscriptSessionIdentity,
+    candidate: SessionFileCandidate
+  ): void {
    this.db
      .prepare(
        `UPDATE sessions SET session_id = ?, title = ?, cwd = ?, cwd_key = ?,
@@ -63,7 +67,7 @@ export class SessionSearchFileRecords {
        identity.cwd,
        cwdKey(identity.cwd),
        identity.createdAt,
-        identity.updatedAt,
+        identity.updatedAt ?? (candidate.agent === 'cursor' ? candidate.file.modifiedAt : null),
        rowId
      )
  }
@@ -86,7 +90,7 @@ export class SessionSearchFileRecords {
      cwdKey(session.cwd),
      session.branch,
      session.createdAt,
-      session.updatedAt,
+      session.updatedAt ?? (session.agent === 'cursor' ? session.modifiedAt : null),
      session.messageCount,
      session.resumeCommand,
      contentHash.hash,
@@ -315,7 +315,7 @@ export class SessionSearchIndexWriter {
            // session they hang off is written with whatever the parser has
            // decoded rather than left empty until a read that may never end.
            // `add` refuses to chunk without this, so it is never absent here.
-            this.records.updateProvisionalSession(session, named)
+            this.records.updateProvisionalSession(session, named, candidate)
          }
          this.records.upsertFile(
            candidate,
@@ -100,10 +100,22 @@ export function openSessionSearchDatabase(path: string): SyncDatabase {
  return openVersionedCacheDatabase(path, {
    version: SESSION_SEARCH_SCHEMA_VERSION,
    sql: SCHEMA_SQL,
-    afterSchema: (db) =>
-      db
-        .prepare('INSERT OR IGNORE INTO meta(key, value) VALUES (?, ?)')
-        .run('index_incarnation', randomUUID())
+    afterSchema: (db) => {
+      db.prepare('INSERT OR IGNORE INTO meta(key, value) VALUES (?, ?)').run(
+        'index_incarnation',
+        randomUUID()
+      )
+      // Older Cursor rows omitted the file-time fallback; repair metadata without rereading transcripts.
+      db.exec(`
+        UPDATE sessions SET updated_at = (
+          SELECT strftime('%Y-%m-%dT%H:%M:%fZ', CAST(files.mtime_ms AS INTEGER) / 1000.0, 'unixepoch')
+          FROM files WHERE files.path = sessions.file_path AND files.session_row_id = sessions.id
+        )
+        WHERE agent = 'cursor' AND updated_at IS NULL
+          AND EXISTS (SELECT 1 FROM files WHERE files.path = sessions.file_path
+            AND files.session_row_id = sessions.id)
+      `)
+    }
  })
 }

@@ -26,7 +26,8 @@ import { sidecarUnchanged } from './session-sidecar-stat'
 import {
  enrichSessionFromSidecar,
  reuseUnchangedSidecarSession,
-  sidecarEnrichesWithoutReparse
+  sidecarEnrichesWithoutReparse,
+  type SidecarEnrichment
 } from './session-scanner-sidecar-enrichment'
 import {
  readResumableTranscript,
@@ -227,6 +228,13 @@ async function parseCachedInLane(
    }
  }

+  let enriched: SidecarEnrichment = { session: null, refused: false }
+  const sessionForConsumers = async (
+    session: AiVaultSession | null
+  ): Promise<AiVaultSession | null> => {
+    enriched = await enrichSessionFromSidecar(candidate, session, platform, signal)
+    return enriched.session
+  }
  const stateFactory = resumableStateFactoryFor(candidate)
  if (stateFactory) {
    const read = await readResumableTranscript({
@@ -235,9 +243,9 @@ async function parseCachedInLane(
      resume: entry?.platform === platform ? entry.resume : null,
      stateFactory,
      stats,
-      signal
+      signal,
+      sessionForConsumers
    })
-    const enriched = await enrichSessionFromSidecar(candidate, read.session, platform, signal)
    storeSessionParseCacheEntry(key, {
      agent: candidate.agent,
      mtimeMs: file.mtimeMs,
@@ -254,11 +262,13 @@ async function parseCachedInLane(
    return enriched.session
  }

-  const session = await readWholeTranscript({ candidate, platform, stats, signal })
-  // Whole-file agents merge the sibling here just like the resumable branch
-  // does post-read; the raw fold stays in foldSession so a sibling-only change
-  // re-merges without re-reading the transcript.
-  const enriched = await enrichSessionFromSidecar(candidate, session, platform, signal)
+  const session = await readWholeTranscript({
+    candidate,
+    platform,
+    stats,
+    signal,
+    sessionForConsumers
+  })
  storeSessionParseCacheEntry(key, {
    agent: candidate.agent,
    mtimeMs: file.mtimeMs,
@@ -9,6 +9,12 @@ import {
 } from './session-parse-cache-store'
 import { TranscriptMessageChannel } from './session-transcript-channel'
 import { mergeSkippedTranscriptRecords } from './session-transcript-record-budget'
+import { throwIfAiVaultScanCancelled } from './ai-vault-scan-cancellation'
+
+type TranscriptSessionEnrichment = {
+  /** Complete consumer metadata before committing; the returned fold stays unmodified. */
+  sessionForConsumers?: (session: AiVaultSession | null) => Promise<AiVaultSession | null>
+}

 const NEWLINE_BYTE = 0x0a

@@ -56,14 +62,16 @@ export function requestWholeTranscriptRead(path: string): void {
 * grew and the recorded offset still sits on a line boundary. Anything else
 * (a rewrite, a truncation, a platform change) re-reads the whole file.
 */
-export async function readResumableTranscript(args: {
-  candidate: SessionFileCandidate
-  platform: NodeJS.Platform
-  resume: SessionParseResumePoint | null
-  stateFactory: (messages: TranscriptMessageChannel) => ResumableSessionParseState
-  stats?: TranscriptReadStats
-  signal?: AbortSignal
-}): Promise<ResumableTranscriptRead> {
+export async function readResumableTranscript(
+  args: TranscriptSessionEnrichment & {
+    candidate: SessionFileCandidate
+    platform: NodeJS.Platform
+    resume: SessionParseResumePoint | null
+    stateFactory: (messages: TranscriptMessageChannel) => ResumableSessionParseState
+    stats?: TranscriptReadStats
+    signal?: AbortSignal
+  }
+): Promise<ResumableTranscriptRead> {
  const { file } = args.candidate
  const resume = args.resume
  const canResume =
@@ -131,7 +139,15 @@ export async function readResumableTranscript(args: {
    }

    const session = await displayState.finalize(args.platform)
-    channel.finishRead({ session, byteOffset: readResult.consumedThrough, incomplete: false })
+    const consumerSession = args.sessionForConsumers
+      ? await args.sessionForConsumers(session)
+      : session
+    throwIfAiVaultScanCancelled(args.signal)
+    channel.finishRead({
+      session: consumerSession,
+      byteOffset: readResult.consumedThrough,
+      incomplete: false
+    })
    return {
      session,
      resume: {
@@ -164,12 +180,14 @@ export async function readResumableTranscript(args: {
 * (whole-JSON documents, Kimi's state doc, OpenCode). There is no cursor to
 * keep, so every read is a whole-file `replace`.
 */
-export async function readWholeTranscript(args: {
-  candidate: SessionFileCandidate
-  platform: NodeJS.Platform
-  stats?: TranscriptReadStats
-  signal?: AbortSignal
-}): Promise<AiVaultSession | null> {
+export async function readWholeTranscript(
+  args: TranscriptSessionEnrichment & {
+    candidate: SessionFileCandidate
+    platform: NodeJS.Platform
+    stats?: TranscriptReadStats
+    signal?: AbortSignal
+  }
+): Promise<AiVaultSession | null> {
  const { file } = args.candidate
  if (args.stats) {
    args.stats.fullParses++
@@ -179,7 +197,15 @@ export async function readWholeTranscript(args: {
  channel.beginRead({ candidate: args.candidate, mode: 'replace', previousByteOffset: 0 })
  try {
    const session = await parseAgentSessionFile(args.candidate, args.platform, channel, args.signal)
-    channel.finishRead({ session, byteOffset: file.sizeBytes ?? 0, incomplete: false })
+    const consumerSession = args.sessionForConsumers
+      ? await args.sessionForConsumers(session)
+      : session
+    throwIfAiVaultScanCancelled(args.signal)
+    channel.finishRead({
+      session: consumerSession,
+      byteOffset: file.sizeBytes ?? 0,
+      incomplete: false
+    })
    return session
  } catch (error) {
    channel.finishRead({