From aa9ad85713deb2abdff96c2c7a84cb0747c9a0bd Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:06:43 -0800 Subject: [PATCH] Read native chat history directly from domain rows by bounded keyset windows Domain schema v2 adds the history metadata the reader selects on (hidden, root, subagent row/group, named subagent IDs, turn marker), backfilled once from saved v1 payloads; every message upsert writes it from the merged current item. Tail/before/hydrate pick a bounded newest-first header window (the existing 2x limit reach, counted over every row so sparse-root chats stay bounded, completing the boundary sequence group) before decoding bodies. Forward catch-up selects an ordered event window, resets below the pruned cursor or across a gap, and returns current rows and current submissions for the touched identities. All reads share one reader snapshot. --- .../journal-domain-history-after.ts | 116 ++++++ .../journal-domain-history-metadata.ts | 39 ++ .../journal-domain-history-page.ts | 284 ++++++++++++++ .../journal-domain-history-window.ts | 107 ++++++ .../journal-domain-history.test.ts | 349 ++++++++++++++++++ .../journal-domain-schema.ts | 82 +++- .../journal-sql-message.test-fixture.ts | 40 +- .../journal-sql-message.ts | 21 +- 8 files changed, 1006 insertions(+), 32 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-domain-history-after.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-domain-history-page.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-domain-history-window.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-domain-history.test.ts diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts new file mode 100644 index 00000000000..73a8e467eb9 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts @@ -0,0 +1,116 @@ +import type Database from '../../sqlite/sync-database' +import type { AgentSessionHistoryResult } from '../../../shared/agent-session/agent-session-wire' +import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity' +import type { + AgentJournalRenderItem, + AgentJournalSnapshot +} from '../../../shared/agent-session/journal/agent-session-journal-types' +import { readAgentSessionForwardHistory } from '../agent-session-wire/publication/history/agent-session-history-forward-page' +import type { HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import type { JournalReadSince } from './journal-store-contracts' +import type { JournalOutcomeReadRow } from './replay/journal-read-row' +import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor' +import type { JournalDomainEventChanges } from './journal-domain-schema' +import type { JournalDomainHistoryHead } from './journal-domain-history-page' + +/** Select one ordered event window, then current rows for only its identities, in the caller's snapshot. */ +export function readJournalDomainForwardHistory( + db: Database.Database, + head: JournalDomainHistoryHead, + cursor: AgentJournalCursor | undefined, + limit: number, + reader: Pick, + latestTurn: AgentJournalRenderItem | null, + readSnapshot: (ids: Set) => AgentJournalSnapshot, + reset: ( + reason: Extract['reset'] + ) => AgentSessionHistoryResult +): AgentSessionHistoryResult { + return readAgentSessionForwardHistory( + { + readSince: (after, count) => readJournalDomainEvents(db, head, after, count), + canonicalItemId: (itemId) => itemId, + hasSubmission: () => false, + reset + }, + { + sessionId: head.sessionId, + cursor: head.cursor, + items: latestTurn ? [latestTurn] : [], + submissions: [] + }, + cursor, + limit, + reader, + readSnapshot + ) +} + +/** Retained events start after the pruned-through cursor; a missing sequence resets, never pages partially. */ +function readJournalDomainEvents( + db: Database.Database, + head: JournalDomainHistoryHead, + cursor: AgentJournalCursor, + limit = -1 +): JournalReadSince { + const resume = resolveJournalResume( + { + epoch: head.cursor.epoch, + lastSequence: head.cursor.sequence, + oldestSequence: head.retainedSequence + 1 + }, + cursor + ) + if (!resume.ok) { + return resume + } + const stored = db + .prepare(`SELECT sequence, changes_json FROM journal_events + WHERE session_id=? AND epoch=? AND sequence>? ORDER BY sequence LIMIT ?`) + .all(head.sessionId, head.epoch, resume.afterSequence, limit) + const complete = limit < 0 || stored.length < limit + const sequences = stored.map((row) => Number(row.sequence)) + if ( + findSequenceGap(sequences, resume.afterSequence + 1) || + (complete && (sequences.at(-1) ?? resume.afterSequence) !== head.cursor.sequence) + ) { + return { ok: false, reset: 'journal_gap' } + } + const rows: JournalOutcomeReadRow[] = [] + for (const row of stored) { + const changes = eventChanges(String(row.changes_json)) + if (!changes) { + return { ok: false, reset: 'schema_unreadable' } + } + rows.push({ + kind: 'outcome', + epoch: head.cursor.epoch, + seq: Number(row.sequence), + itemIds: [...changes.items.map((item) => item.itemId), ...(changes.removedItemIds ?? [])], + submissionIds: changes.submissions + }) + } + return { + ok: true, + rows, + through: complete ? head.cursor.sequence : sequences.at(-1)!, + cursor: head.cursor + } +} + +function eventChanges(json: string): JournalDomainEventChanges | null { + let changes: JournalDomainEventChanges + try { + changes = JSON.parse(json) + } catch { + return null + } + const ids = (value: unknown) => + Array.isArray(value) && value.every((id) => typeof id === 'string') + return Array.isArray(changes?.items) && + changes.items.every((item) => typeof item?.itemId === 'string') && + ids(changes.submissions) && + (changes.removedItemIds === undefined || ids(changes.removedItemIds)) + ? changes + : null +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts new file mode 100644 index 00000000000..755e87de2a0 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts @@ -0,0 +1,39 @@ +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { isRootAgentJournalItem } from '../../../shared/agent-session/journal/agent-session-journal-producer' +import { isAgentSessionContextClear } from '../../../shared/agent-session/agent-session-context-clear' +import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record' +import { + namedSubagents, + subagentFoldKey, + subagentItemBlock +} from '../../../shared/native-chat/transcript/native-chat-subagent-items' +import { isSubagentRow } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' + +/** Facts history selects before decoding a body; written from each current item revision. */ +export type JournalDomainHistoryMetadata = { + root: number + subagentRow: number + subagentGroupKey: string | null + namedSubagentIds: string | null + turnMarker: number +} + +export function journalDomainHistoryMetadata( + item: AgentJournalRenderItem +): JournalDomainHistoryMetadata { + const root = isRootAgentJournalItem(item) + const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null + const named = + root && item.body.kind === 'message' + ? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id) + : [] + return { + root: Number(root), + subagentRow: Number(isSubagentRow(item)), + subagentGroupKey: block ? subagentFoldKey(block) : null, + namedSubagentIds: named.length > 0 ? JSON.stringify(named) : null, + turnMarker: Number( + readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body) + ) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts new file mode 100644 index 00000000000..ecdea4f3cb9 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts @@ -0,0 +1,284 @@ +import type Database from '../../sqlite/sync-database' +import { + AGENT_SESSION_HISTORY_MAX_LIMIT, + type AgentSessionHistoryPage, + type AgentSessionHistoryRequest, + type AgentSessionHistoryResult +} from '../../../shared/agent-session/agent-session-wire' +import type { + AgentJournalRenderItem, + AgentJournalSnapshot, + AgentJournalSubmission +} from '../../../shared/agent-session/journal/agent-session-journal-types' +import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity' +import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer' +import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items' +import { + historyContentBytes, + oversizedHistoryItem, + resolveHistoryLimit, + submissionBytesByItemId, + type HistoryReader +} from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result' +import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster' +import { CLIENT_HISTORY_READER } from '../agent-session-wire/publication/history/agent-session-item-wire-form' +import { journalContextEpoch } from './context-epochs/journal-context-cursor' +import { readJournalDomainForwardHistory } from './journal-domain-history-after' +import { + journalDomainSequenceGroups, + selectJournalDomainHistoryWindow +} from './journal-domain-history-window' + +export type JournalDomainHistoryRequest = Omit & { + direction: AgentSessionHistoryRequest['direction'] | 'hydrate' + fence?: number + scope?: 'every-agent' | 'own-agent' + rowReplay?: boolean +} + +export type JournalDomainHistoryHead = { + sessionId: string + epoch: string + /** Public context identity of the physical epoch. */ + cursor: AgentJournalCursor + retainedSequence: number +} + +type HistoryReaderForm = Pick + +/** The caller owns a readonly connection; head, headers, bodies and any reset share one snapshot. */ +export function readJournalDomainHistoryPage( + db: Database.Database, + request: JournalDomainHistoryRequest, + reader: HistoryReaderForm = CLIENT_HISTORY_READER +): AgentSessionHistoryResult { + db.exec('BEGIN') + try { + const result = readHistory(db, readHead(db, request.sessionId), request, reader) + db.exec('COMMIT') + return result + } catch (error) { + db.exec('ROLLBACK') + throw error + } +} + +function readHead(db: Database.Database, sessionId: string): JournalDomainHistoryHead { + const row = db + .prepare(`SELECT epoch, last_sequence, retained_sequence, context_boundary FROM journal_chat_state + WHERE session_id=?`) + .get(sessionId) + if (!row) { + throw new Error('journal history targets a chat without domain state') + } + const epoch = String(row.epoch) + return { + sessionId, + epoch, + cursor: { + epoch: journalContextEpoch(epoch, Number(row.context_boundary)), + sequence: Number(row.last_sequence) + }, + retainedSequence: Number(row.retained_sequence) + } +} + +function readHistory( + db: Database.Database, + head: JournalDomainHistoryHead, + request: JournalDomainHistoryRequest, + reader: HistoryReaderForm +): AgentSessionHistoryResult { + const reset = ( + reason: Extract['reset'] + ): AgentSessionHistoryResult => ({ + ok: false, + reset: reason, + page: historyPage( + db, + head, + { ...request, direction: 'hydrate', fence: undefined, scope: 'every-agent' }, + reader + ) + }) + if (request.direction === 'after') { + return readJournalDomainForwardHistory( + db, + head, + request.cursor, + resolveHistoryLimit(request.limit), + reader, + latestTurnItem(db, head), + (ids) => currentRows(db, head, ids), + reset + ) + } + const cursor = request.direction === 'before' ? request.cursor : undefined + if (cursor && cursor.epoch !== head.cursor.epoch) { + return reset('epoch_changed') + } + if (cursor && cursor.sequence > head.cursor.sequence) { + return reset('cursor_ahead') + } + return { ok: true, page: historyPage(db, head, request, reader) } +} + +function historyPage( + db: Database.Database, + head: JournalDomainHistoryHead, + request: JournalDomainHistoryRequest, + reader: HistoryReaderForm +): AgentSessionHistoryPage { + const prefix = [head.sessionId, head.epoch] as const + const cursor = request.direction === 'before' ? request.cursor : undefined + const end = cursor?.sequence ?? Number.MAX_SAFE_INTEGER + const where = `session_id=? AND epoch=? AND hidden=0${request.scope === 'own-agent' ? ' AND root=1' : ''}` + const limit = + request.direction === 'hydrate' + ? AGENT_SESSION_HISTORY_MAX_LIMIT + : resolveHistoryLimit(request.limit) + const window = selectJournalDomainHistoryWindow(db, where, prefix, end, limit) + const submissions = itemSubmissions( + db, + head, + window.map((header) => header.itemId) + ) + const sizes = submissionBytesByItemId(submissions) + const body = db.prepare( + 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' + ) + const kept: AgentJournalRenderItem[][] = [] + let total = 0 + for (const group of journalDomainSequenceGroups(window)) { + const formed = group.map((header) => + reader.form(JSON.parse(String(body.get(...prefix, header.itemId)!.payload_json))) + ) + const bytes = historyContentBytes(formed, [], sizes) + if (kept.length > 0 && total + bytes > reader.budgetBytes) { + break + } + kept.push( + formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item)) + ) + total += bytes + if (bytes > reader.budgetBytes) { + break + } + } + const items = kept.toReversed().flat() + const oldest = items[0]?.sequence ?? end + const turn = latestTurnItem(db, head) + return buildAgentSessionHistoryPage({ + snapshot: { + sessionId: head.sessionId, + cursor: head.cursor, + items: turn ? [turn] : [], + submissions + }, + direction: request.direction === 'hydrate' ? 'tail' : request.direction, + items, + hasOlder: !!db + .prepare(`SELECT 1 FROM journal_items WHERE ${where} AND sequence=? LIMIT 1`) + .get(...prefix, end), + fallbackCursor: cursor ?? { epoch: head.cursor.epoch, sequence: 0 }, + nextCursor: items[0] ? { epoch: head.cursor.epoch, sequence: items[0].sequence } : undefined, + fence: request.fence, + subagentRoster: historyRoster(db, prefix, items) + }) +} + +function latestTurnItem( + db: Database.Database, + head: JournalDomainHistoryHead +): AgentJournalRenderItem | null { + const row = db + .prepare(`SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND hidden=0 + AND turn_marker=1 ORDER BY sequence DESC, sequence_index DESC, ordinal DESC LIMIT 1`) + .get(head.sessionId, head.epoch) + return row ? JSON.parse(String(row.payload_json)) : null +} + +/** Current attempts only, in submission order then stable admission ordinal. */ +function itemSubmissions( + db: Database.Database, + head: JournalDomainHistoryHead, + itemIds: readonly string[] +): AgentJournalSubmission[] { + return db + .prepare(`SELECT attempt.payload_json FROM json_each(?) wanted + CROSS JOIN journal_submission_attempts attempt + WHERE attempt.session_id=? AND attempt.epoch=? AND attempt.current=1 AND attempt.item_id=wanted.value + ORDER BY attempt.submitted_at, attempt.ordinal`) + .all(JSON.stringify(itemIds), head.sessionId, head.epoch) + .map((row) => JSON.parse(String(row.payload_json))) +} + +function currentRows( + db: Database.Database, + head: JournalDomainHistoryHead, + ids: ReadonlySet +): AgentJournalSnapshot { + const wanted = [...ids] + return { + sessionId: head.sessionId, + cursor: head.cursor, + items: db + .prepare(`SELECT item.payload_json FROM json_each(?) wanted + CROSS JOIN journal_items item + WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0 + ORDER BY item.sequence, item.sequence_index, item.ordinal`) + .all(JSON.stringify(wanted), head.sessionId, head.epoch) + .map((row) => JSON.parse(String(row.payload_json))), + submissions: itemSubmissions(db, head, wanted) + } +} + +/** Reads only root rows naming a subagent the page shows but does not name. */ +function historyRoster( + db: Database.Database, + prefix: readonly [string, string], + page: readonly AgentJournalRenderItem[] +) { + const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null)) + for (const item of page) { + if (item.agentId === undefined && item.body.kind === 'message') { + for (const entry of item.body.blocks.flatMap(namedSubagents)) { + missing.delete(entry.id) + } + } + } + if (missing.size === 0) { + return undefined + } + const body = db.prepare( + 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' + ) + const named: AgentJournalRenderItem[] = [] + for (const row of db + .prepare(`SELECT item_id, named_subagent_ids FROM journal_items + WHERE session_id=? AND epoch=? AND hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL + ORDER BY sequence, sequence_index, ordinal`) + .iterate(...prefix)) { + const ids: string[] = JSON.parse(String(row.named_subagent_ids)) + if (!ids.some((id) => missing.has(id))) { + continue + } + for (const id of ids) { + missing.delete(id) + } + named.push(JSON.parse(String(body.get(...prefix, String(row.item_id))!.payload_json))) + if (missing.size === 0 || named.length >= ROSTER_ROWS) { + break + } + } + return offPageSubagentRoster(named, page) +} + +// The roster keeps at most 64 entries; each selected row adds at least one. +const ROSTER_ROWS = 64 diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts new file mode 100644 index 00000000000..b9f9bf128fd --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts @@ -0,0 +1,107 @@ +import type Database from '../../sqlite/sync-database' +import type { SqliteRow } from '../../sqlite/sqlite-statement' + +export type JournalDomainHistoryHeader = { + itemId: string + sequence: number + own: boolean + subagentGroupKey: string | null +} + +/** Chooses the page's candidate identities before any body or submission is decoded. */ +export function selectJournalDomainHistoryWindow( + db: Database.Database, + where: string, + prefix: readonly [string, string], + end: number, + limit: number +): JournalDomainHistoryHeader[] { + return historyWindow(selectJournalDomainHistoryHeaders(db, where, prefix, end, limit), limit) +} + +/** Newest-first keyset over every scoped row: the existing 2×limit reach also caps sparse-root chats. + * An epoch+sequence cursor cannot split a sequence, so the boundary sequence is completed whole. */ +function selectJournalDomainHistoryHeaders( + db: Database.Database, + where: string, + prefix: readonly [string, string], + end: number, + limit: number +): JournalDomainHistoryHeader[] { + const columns = 'item_id, sequence, root, subagent_row, subagent_group_key' + const order = 'ORDER BY sequence DESC, sequence_index DESC, ordinal DESC' + const cap = 2 * limit + const rows = db + .prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence row.sequence === boundary) + rows.splice( + first, + rows.length - first, + ...db + .prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence=? ${order}`) + .all(...prefix, boundary) + ) + } + return rows.map((row: SqliteRow) => ({ + itemId: String(row.item_id), + sequence: Number(row.sequence), + own: row.root === 1 && row.subagent_row === 0, + subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key) + })) +} + +/** Window by own-agent rows, retaining interleaved subagent work and its whole groups within the candidates. */ +function historyWindow( + headers: readonly JournalDomainHistoryHeader[], + limit: number +): JournalDomainHistoryHeader[] { + let own = 0 + let start: number | undefined + for (const group of journalDomainSequenceGroups(headers)) { + const count = group.filter((header) => header.own).length + if (count === 0) { + continue + } + if (own > 0 && own + count > limit) { + const firstOf = new Map() + for (const header of headers) { + if (header.subagentGroupKey !== null) { + firstOf.set(header.subagentGroupKey, header.sequence) + } + } + let whole = start! + for (const header of headers) { + if (header.sequence < whole) { + break + } + if (header.subagentGroupKey !== null) { + whole = Math.min(whole, firstOf.get(header.subagentGroupKey)!) + } + } + return headers.filter((header) => header.sequence >= whole) + } + own += count + start = group[0]!.sequence + } + return [...headers] +} + +/** Newest group first; each group keeps render order. */ +export function* journalDomainSequenceGroups( + rows: readonly T[] +): Generator { + let group: T[] = [] + for (const row of rows) { + if (group.length > 0 && row.sequence !== group[0]!.sequence) { + yield group.toReversed() + group = [] + } + group.push(row) + } + if (group.length > 0) { + yield group.toReversed() + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts new file mode 100644 index 00000000000..286e0a8fd00 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts @@ -0,0 +1,349 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import type { AgentSessionHistoryResult } from '../../../shared/agent-session/agent-session-wire' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { measureHistoryItem } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import { message, messageFixture, target } from './journal-sql-message.test-fixture' +import { + readJournalDomainHistoryPage, + type JournalDomainHistoryRequest +} from './journal-domain-history-page' +import { migrateJournalDomainSchema } from './journal-domain-schema' + +const cleanup: (() => void)[] = [] +afterEach(() => { + for (const step of cleanup.splice(0)) { + step() + } +}) + +function history(db = new Database(':memory:')) { + const value = messageFixture(db) + cleanup.push(() => db.close()) + return { + ...value, + head: (sequence: number, retained = 0) => + db + .prepare('UPDATE journal_chat_state SET last_sequence=?, retained_sequence=?') + .run(sequence, retained), + event: (sequence: number, itemIds: string[]) => + db.prepare('INSERT INTO journal_events VALUES(?,?,?,0,?)').run( + target.sessionId, + target.epoch, + sequence, + JSON.stringify({ + items: itemIds.map((itemId) => ({ itemId, revision: 1, sequence, sequenceIndex: 0 })), + submissions: [], + cards: [] + }) + ), + read: ( + request: Omit, + reader?: Parameters[2] + ) => readJournalDomainHistoryPage(db, { sessionId: target.sessionId, ...request }, reader) + } +} + +function text( + sequence: number, + value: string, + extra: Partial = {} +): AgentJournalRenderItem { + return { + itemId: `orca:${sequence}:${extra.sequenceIndex ?? 0}`, + revision: 1, + sequence, + observedAt: sequence, + body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: value }] }, + turnScope: { kind: 'thread' }, + ...extra + } +} + +const sequences = (result: AgentSessionHistoryResult) => + result.page.items.map((item) => item.sequence) + +it('pages a chat of almost only subagent rows in bounded, gap-free pages', () => { + const { seedItem, head, read } = history() + seedItem({ + ...text(1, ''), + body: { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'subagent', id: 'child', label: 'Child', state: 'working', groupId: 'g' }] + } + }) + for (let sequence = 2; sequence <= 301; sequence++) { + seedItem(text(sequence, 'child work', { agentId: 'child' })) + } + head(301) + let result = read({ direction: 'tail', limit: 40 }) + expect(result.page.subagentRoster).toMatchObject([ + { itemId: 'orca:1:0', sequence: 1, entry: { id: 'child', label: 'Child' } } + ]) + const sizes: number[] = [] + const seen: number[] = [] + for (;;) { + expect(result.ok).toBe(true) + sizes.push(result.page.items.length) + seen.unshift(...sequences(result)) + if (!result.page.hasOlder) { + break + } + result = read({ direction: 'before', cursor: result.page.window.nextCursor, limit: 40 }) + } + expect(sizes).toEqual([80, 80, 80, 61]) + expect(seen).toEqual(Array.from({ length: 301 }, (_, index) => index + 1)) +}) + +it('never splits a same-sequence group, marking it oversized backward and paging it whole forward', () => { + const { seedItem, head, event, read } = history() + const group = Array.from({ length: 100 }, (_, index) => + text(50, 'x'.repeat(1000), { sequenceIndex: index }) + ) + seedItem(text(49, 'before')) + event(49, ['orca:49:0']) + for (const item of group) { + seedItem(item) + } + event( + 50, + group.map((item) => item.itemId) + ) + for (let sequence = 51; sequence <= 60; sequence++) { + seedItem(text(sequence, 'after')) + event(sequence, [`orca:${sequence}:0`]) + } + head(60, 48) + const small = { form: measureHistoryItem, budgetBytes: 50_000 } + const tail = read({ direction: 'tail', limit: 40 }, small) + expect(sequences(tail)).toEqual([51, 52, 53, 54, 55, 56, 57, 58, 59, 60]) + const wide = read({ direction: 'before', cursor: tail.page.window.nextCursor, limit: 40 }, small) + expect(wide.page.items.map((item) => item.itemId)).toEqual(group.map((item) => item.itemId)) + expect(new Set(wide.page.items.map((item) => item.body.kind))).toEqual(new Set(['status'])) + expect(wide.page).toMatchObject({ hasOlder: true, window: { nextCursor: { sequence: 50 } } }) + const rest = read({ direction: 'before', cursor: wide.page.window.nextCursor, limit: 40 }, small) + expect(sequences(rest)).toEqual([49]) + expect(rest.page.hasOlder).toBe(false) + + const forward: string[] = [] + let cursor = { epoch: 'epoch', sequence: 48 } + for (const expected of [true, true, false]) { + const page = read({ direction: 'after', cursor, limit: 4 }) + expect(page).toMatchObject({ ok: true, page: { hasNewer: expected } }) + forward.push(...page.page.items.map((item) => item.itemId)) + cursor = page.page.window.nextCursor + } + expect(cursor.sequence).toBe(60) + expect(forward).toEqual([ + 'orca:49:0', + ...group.map((item) => item.itemId), + ...Array.from({ length: 10 }, (_, index) => `orca:${51 + index}:0`) + ]) +}) + +it('catches up with the current revision, placement and submission, not the replayed metadata', async () => { + const { db, commit, seedSend, read } = history() + seedSend() + db.exec(`INSERT INTO journal_submission_attempts + SELECT session_id,epoch,message_id,0,0,0,item_id,0,0,NULL,fingerprint,'unknown',0,NULL,NULL,'{}' + FROM journal_submission_attempts WHERE current=1`) + await commit(message('first')) + await commit(message('second')) + const echo = message('original', 'unused', 'item') + echo.identity = { provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 } + echo.body.role = 'user' + await commit(echo) + const result = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 10 } }) + expect(result).toMatchObject({ + ok: true, + page: { + items: [ + { itemId: 'orca:send', sequence: 7, body: { blocks: [{ text: 'original' }] } }, + { + itemId: 'orca:assistant', + revision: 2, + sequence: 11, + body: { blocks: [{ text: 'second' }] } + } + ], + submissions: [{ clientMessageId: 'send', dispatchState: 'accepted', acceptedSequence: 1 }], + window: { nextCursor: { epoch: 'epoch', sequence: 13 } }, + hasNewer: false + } + }) + const child = message('child work', 'child', 'item') + child.options.agentId = 'child' + await commit(child) + expect( + read({ direction: 'tail', scope: 'own-agent' }).page.items.map((item) => item.itemId) + ).toEqual(['orca:send', 'orca:assistant']) + expect(read({ direction: 'tail' }).page.items.map((item) => item.itemId)).toEqual([ + 'orca:send', + 'orca:assistant', + 'orca:child' + ]) +}) + +it('resets a cursor below the pruned boundary or across a gap, and resumes at the boundary', () => { + const { db, seedItem, head, event, read } = history() + for (let sequence = 1; sequence <= 3; sequence++) { + seedItem(text(sequence, 'row')) + } + event(3, ['orca:3:0']) + head(3, 2) + expect(read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 1 } })).toMatchObject({ + ok: false, + reset: 'cursor_compacted', + page: { direction: 'tail', items: [{ sequence: 1 }, { sequence: 2 }, { sequence: 3 }] } + }) + const resumed = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 2 } }) + expect(resumed.ok).toBe(true) + expect(sequences(resumed)).toEqual([3]) + db.exec('DELETE FROM journal_events') + expect(read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 2 } })).toMatchObject({ + ok: false, + reset: 'journal_gap' + }) +}) + +it('answers every selected read from the snapshot its head came from', () => { + const dir = mkdtempSync(join(tmpdir(), 'journal-domain-history-')) + const path = join(dir, 'chat.db') + const writer = new Database(path) + writer.pragma('journal_mode=WAL') + const { seedItem, head } = history(writer) + seedItem(text(1, 'first')) + seedItem({ + itemId: 'orca:turn', + revision: 1, + sequence: 2, + observedAt: 2, + body: { kind: 'turn', turnId: 'turn', state: 'running', startedAt: 2 }, + turnScope: { kind: 'thread' } + }) + head(2) + const reader = new Database(path, { readonly: true }) + cleanup.push( + () => reader.close(), + () => rmSync(dir, { recursive: true }) + ) + let wrote = false + const result = readJournalDomainHistoryPage( + reader, + { sessionId: target.sessionId, direction: 'tail' }, + { + budgetBytes: 1024 * 1024, + form: (item) => { + if (!wrote) { + wrote = true + writer.exec('DELETE FROM journal_items; UPDATE journal_chat_state SET last_sequence=9') + } + return measureHistoryItem(item) + } + } + ) + expect(result).toMatchObject({ + ok: true, + page: { + items: [{ itemId: 'orca:1:0' }, { itemId: 'orca:turn' }], + liveCursor: { epoch: 'epoch', sequence: 2 }, + latestTurn: { itemId: 'orca:turn', turn: { state: 'running' } }, + hasOlder: false + } + }) + expect(writer.prepare('SELECT COUNT(*) AS count FROM journal_items').get()!.count).toBe(0) +}) + +it('backfills history metadata once from saved v1 rows and refuses a newer domain schema', () => { + const db = new Database(':memory:') + cleanup.push(() => db.close()) + migrateJournalDomainSchema(db) + db.exec(`DROP INDEX journal_items_root_position; + DROP INDEX journal_items_turn_marker; + DROP INDEX journal_items_named_subagents; + ALTER TABLE journal_items DROP COLUMN hidden; + ALTER TABLE journal_items DROP COLUMN root; + ALTER TABLE journal_items DROP COLUMN subagent_row; + ALTER TABLE journal_items DROP COLUMN subagent_group_key; + ALTER TABLE journal_items DROP COLUMN named_subagent_ids; + ALTER TABLE journal_items DROP COLUMN turn_marker; + DELETE FROM journal_domain_migrations WHERE version=2`) + const rows: AgentJournalRenderItem[] = [ + { + ...text(1, ''), + body: { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'subagent', id: 'child', label: 'Child', state: 'working', groupId: 'g' }] + } + }, + text(2, 'child work', { agentId: 'child' }), + { + itemId: 'orca:turn', + revision: 3, + sequence: 3, + observedAt: 3, + body: { kind: 'turn', turnId: 'turn', state: 'completed' }, + turnScope: { kind: 'thread' } + } + ] + for (const item of rows) { + db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json) + VALUES('chat','epoch',?,?,?,?,0,1,?)`).run( + item.itemId, + item.sequence, + item.revision, + item.sequence, + JSON.stringify(item) + ) + } + migrateJournalDomainSchema(db) + expect( + db + .prepare(`SELECT item_id, revision, hidden, root, subagent_row, subagent_group_key, + named_subagent_ids, turn_marker, payload_json FROM journal_items ORDER BY sequence`) + .all() + ).toEqual([ + { + item_id: 'orca:1:0', + revision: 1, + hidden: 0, + root: 1, + subagent_row: 1, + subagent_group_key: 'g', + named_subagent_ids: '["child"]', + turn_marker: 0, + payload_json: JSON.stringify(rows[0]) + }, + { + item_id: 'orca:2:0', + revision: 1, + hidden: 0, + root: 0, + subagent_row: 0, + subagent_group_key: null, + named_subagent_ids: null, + turn_marker: 0, + payload_json: JSON.stringify(rows[1]) + }, + { + item_id: 'orca:turn', + revision: 3, + hidden: 0, + root: 1, + subagent_row: 0, + subagent_group_key: null, + named_subagent_ids: null, + turn_marker: 1, + payload_json: JSON.stringify(rows[2]) + } + ]) + db.exec('INSERT INTO journal_domain_migrations(version) VALUES (3)') + expect(() => migrateJournalDomainSchema(db)).toThrow( + 'journal domain schema is newer than this writer' + ) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-domain-schema.ts b/src/main/native-chat/agent-session-journal/journal-domain-schema.ts index a2e5a4952a0..65be863546a 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-schema.ts @@ -1,4 +1,17 @@ import type Database from '../../sqlite/sync-database' +import type { SqliteBindings } from '../../sqlite/sqlite-statement' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { journalDomainHistoryMetadata } from './journal-domain-history-metadata' + +const JOURNAL_DOMAIN_VERSION = 2 + +/** journal_events.changes_json: identities only; readers load current rows for bodies. */ +export type JournalDomainEventChanges = { + items: { itemId: string; revision: number; sequence: number; sequenceIndex: number }[] + submissions: string[] + cards: string[] + removedItemIds?: string[] +} /** This ledger deliberately leaves the profile's global user_version unchanged. */ export function migrateJournalDomainSchema(db: Database.Database): void { @@ -8,15 +21,70 @@ export function migrateJournalDomainSchema(db: Database.Database): void { const version = Number( db.prepare('SELECT MAX(version) AS version FROM journal_domain_migrations').get()?.version ?? 0 ) - if (version === 1) { + if (version === JOURNAL_DOMAIN_VERSION) { return } - if (version > 1) { + if (version > JOURNAL_DOMAIN_VERSION) { throw new Error('journal domain schema is newer than this writer') } db.exec('BEGIN IMMEDIATE') try { - db.exec(` + if (version < 1) { + db.exec(JOURNAL_DOMAIN_V1) + } + migrateJournalDomainHistory(db) + db.exec('COMMIT') + } catch (error) { + db.exec('ROLLBACK') + throw error + } +} + +/** v2: history selects windows by these columns, so existing rows are backfilled once here. */ +function migrateJournalDomainHistory(db: Database.Database): void { + db.exec(` +ALTER TABLE journal_items ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0; +ALTER TABLE journal_items ADD COLUMN root INTEGER NOT NULL DEFAULT 1; +ALTER TABLE journal_items ADD COLUMN subagent_row INTEGER NOT NULL DEFAULT 0; +ALTER TABLE journal_items ADD COLUMN subagent_group_key TEXT; +ALTER TABLE journal_items ADD COLUMN named_subagent_ids TEXT; +ALTER TABLE journal_items ADD COLUMN turn_marker INTEGER NOT NULL DEFAULT 0; +CREATE INDEX journal_items_root_position ON journal_items(session_id, epoch, sequence, sequence_index, ordinal) + WHERE hidden=0 AND root=1; +CREATE INDEX journal_items_turn_marker ON journal_items(session_id, epoch, sequence, sequence_index, ordinal) + WHERE hidden=0 AND turn_marker=1; +CREATE INDEX journal_items_named_subagents ON journal_items(session_id, epoch, sequence, sequence_index, ordinal) + WHERE hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL; +`) + const page = db.prepare(`SELECT session_id, epoch, item_id, payload_json FROM journal_items + WHERE (session_id, epoch, item_id) > (?, ?, ?) ORDER BY session_id, epoch, item_id LIMIT 500`) + const update = db.prepare(`UPDATE journal_items SET root=?, subagent_row=?, subagent_group_key=?, + named_subagent_ids=?, turn_marker=? WHERE session_id=? AND epoch=? AND item_id=?`) + let after: SqliteBindings = ['', '', ''] + for (let rows = page.all(...after); rows.length > 0; rows = page.all(...after)) { + for (const row of rows) { + const item: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) + const metadata = journalDomainHistoryMetadata(item) + update.run( + metadata.root, + metadata.subagentRow, + metadata.subagentGroupKey, + metadata.namedSubagentIds, + metadata.turnMarker, + row.session_id, + row.epoch, + row.item_id + ) + } + const last = rows.at(-1)! + after = [last.session_id, last.epoch, last.item_id] + } + db.prepare('INSERT INTO journal_domain_migrations(version) VALUES (?)').run( + JOURNAL_DOMAIN_VERSION + ) +} + +const JOURNAL_DOMAIN_V1 = ` CREATE TABLE journal_chat_state ( session_id TEXT PRIMARY KEY, epoch TEXT NOT NULL, @@ -117,10 +185,4 @@ CREATE TABLE journal_events ( PRIMARY KEY(session_id, epoch, sequence) ) WITHOUT ROWID; INSERT INTO journal_domain_migrations(version) VALUES (1); -`) - db.exec('COMMIT') - } catch (error) { - db.exec('ROLLBACK') - throw error - } -} +` diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts index d3dc8bcd70c..a05dbabd1ac 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts @@ -7,6 +7,7 @@ import type { import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation' import { migrateJournalDomainSchema } from './journal-domain-schema' +import { journalDomainHistoryMetadata } from './journal-domain-history-metadata' import { JournalSqlBatch } from './journal-sql-batch' import { commitJournalSqlMessages, @@ -29,8 +30,7 @@ export function message( } } -export function messageFixture() { - const db = new Database(':memory:') +export function messageFixture(db = new Database(':memory:')) { db.pragma('user_version=4') migrateJournalDomainSchema(db) db.prepare( @@ -54,21 +54,27 @@ export function messageFixture() { const row = db.prepare('SELECT payload_json FROM journal_items WHERE item_id=?').get(itemId) return row ? JSON.parse(String(row.payload_json)) : undefined } - const seedItem = (item: AgentJournalRenderItem) => - db - .prepare(`INSERT INTO journal_items - (session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json) VALUES(?,?,?,?,?,?,?,?,?)`) - .run( - target.sessionId, - target.epoch, - item.itemId, - item.sequence, - item.revision, - item.sequence, - item.sequenceIndex ?? 0, - 1, - JSON.stringify(item) - ) + const seedItem = (item: AgentJournalRenderItem) => { + const metadata = journalDomainHistoryMetadata(item) + db.prepare(`INSERT INTO journal_items + (session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json, + root,subagent_row,subagent_group_key,named_subagent_ids,turn_marker) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`).run( + target.sessionId, + target.epoch, + item.itemId, + item.sequence, + item.revision, + item.sequence, + item.sequenceIndex ?? 0, + 1, + JSON.stringify(item), + metadata.root, + metadata.subagentRow, + metadata.subagentGroupKey, + metadata.namedSubagentIds, + metadata.turnMarker + ) + } const seedSend = () => { const body: AgentJournalMessageItem = { kind: 'message', diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.ts index 93368804be7..4a253e14bea 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.ts @@ -32,6 +32,8 @@ import { isJournalTextRevision } from './writing/journal-streamed-text-revision' import { assertJournalFence } from './writing/journal-write-guards' import type { JournalItemRow } from './durable-rows/journal-row-schema' import type { QueuedMessageRow } from './queued-messages/queued-message-table' +import { journalDomainHistoryMetadata } from './journal-domain-history-metadata' +import type { JournalDomainEventChanges } from './journal-domain-schema' import { readJournalSqlAcceptedInputs, type JournalSqlAcceptedInputs @@ -206,10 +208,14 @@ export function applyJournalSqlMessage( cards: [], inputChanged: false } - db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json) - VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET + const metadata = journalDomainHistoryMetadata(next) + db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json, + root,subagent_row,subagent_group_key,named_subagent_ids,turn_marker) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index, - execution_owner=excluded.execution_owner, payload_json=excluded.payload_json`).run( + execution_owner=excluded.execution_owner, payload_json=excluded.payload_json, + root=excluded.root, subagent_row=excluded.subagent_row, subagent_group_key=excluded.subagent_group_key, + named_subagent_ids=excluded.named_subagent_ids, turn_marker=excluded.turn_marker`).run( ...prefix, itemId, stored?.ordinal ?? sequence, @@ -218,7 +224,12 @@ export function applyJournalSqlMessage( next.sequenceIndex ?? 0, row.ownerFence!, row.executionOwner ?? stored?.execution_owner ?? null, - JSON.stringify(next) + JSON.stringify(next), + metadata.root, + metadata.subagentRow, + metadata.subagentGroupKey, + metadata.namedSubagentIds, + metadata.turnMarker ) db.prepare( 'DELETE FROM journal_item_tombstones WHERE session_id=? AND epoch=? AND item_id=?' @@ -267,7 +278,7 @@ export function applyJournalSqlMessage( ], submissions: result.receipts.map((receipt) => receipt.clientMessageId), cards: result.cards.map((card) => card.messageId) - }) + } satisfies JournalDomainEventChanges) ) return result }