Merge bounded native chat domain history

This commit is contained in:
Brennan Benson
2026-11-08 04:08:17 -08:00
8 changed files with 1006 additions and 32 deletions
@@ -0,0 +1,116 @@
import type Database from '../../sqlite/sync-database'
import type { AgentSessionHistoryResult } from '../../../shared/agent-session/agent-session-wire'
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
import type {
AgentJournalRenderItem,
AgentJournalSnapshot
} from '../../../shared/agent-session/journal/agent-session-journal-types'
import { readAgentSessionForwardHistory } from '../agent-session-wire/publication/history/agent-session-history-forward-page'
import type { HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
import type { JournalReadSince } from './journal-store-contracts'
import type { JournalOutcomeReadRow } from './replay/journal-read-row'
import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor'
import type { JournalDomainEventChanges } from './journal-domain-schema'
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
/** Select one ordered event window, then current rows for only its identities, in the caller's snapshot. */
export function readJournalDomainForwardHistory(
db: Database.Database,
head: JournalDomainHistoryHead,
cursor: AgentJournalCursor | undefined,
limit: number,
reader: Pick<HistoryReader, 'form' | 'budgetBytes'>,
latestTurn: AgentJournalRenderItem | null,
readSnapshot: (ids: Set<string>) => AgentJournalSnapshot,
reset: (
reason: Extract<AgentSessionHistoryResult, { ok: false }>['reset']
) => AgentSessionHistoryResult
): AgentSessionHistoryResult {
return readAgentSessionForwardHistory(
{
readSince: (after, count) => readJournalDomainEvents(db, head, after, count),
canonicalItemId: (itemId) => itemId,
hasSubmission: () => false,
reset
},
{
sessionId: head.sessionId,
cursor: head.cursor,
items: latestTurn ? [latestTurn] : [],
submissions: []
},
cursor,
limit,
reader,
readSnapshot
)
}
/** Retained events start after the pruned-through cursor; a missing sequence resets, never pages partially. */
function readJournalDomainEvents(
db: Database.Database,
head: JournalDomainHistoryHead,
cursor: AgentJournalCursor,
limit = -1
): JournalReadSince {
const resume = resolveJournalResume(
{
epoch: head.cursor.epoch,
lastSequence: head.cursor.sequence,
oldestSequence: head.retainedSequence + 1
},
cursor
)
if (!resume.ok) {
return resume
}
const stored = db
.prepare(`SELECT sequence, changes_json FROM journal_events
WHERE session_id=? AND epoch=? AND sequence>? ORDER BY sequence LIMIT ?`)
.all(head.sessionId, head.epoch, resume.afterSequence, limit)
const complete = limit < 0 || stored.length < limit
const sequences = stored.map((row) => Number(row.sequence))
if (
findSequenceGap(sequences, resume.afterSequence + 1) ||
(complete && (sequences.at(-1) ?? resume.afterSequence) !== head.cursor.sequence)
) {
return { ok: false, reset: 'journal_gap' }
}
const rows: JournalOutcomeReadRow[] = []
for (const row of stored) {
const changes = eventChanges(String(row.changes_json))
if (!changes) {
return { ok: false, reset: 'schema_unreadable' }
}
rows.push({
kind: 'outcome',
epoch: head.cursor.epoch,
seq: Number(row.sequence),
itemIds: [...changes.items.map((item) => item.itemId), ...(changes.removedItemIds ?? [])],
submissionIds: changes.submissions
})
}
return {
ok: true,
rows,
through: complete ? head.cursor.sequence : sequences.at(-1)!,
cursor: head.cursor
}
}
function eventChanges(json: string): JournalDomainEventChanges | null {
let changes: JournalDomainEventChanges
try {
changes = JSON.parse(json)
} catch {
return null
}
const ids = (value: unknown) =>
Array.isArray(value) && value.every((id) => typeof id === 'string')
return Array.isArray(changes?.items) &&
changes.items.every((item) => typeof item?.itemId === 'string') &&
ids(changes.submissions) &&
(changes.removedItemIds === undefined || ids(changes.removedItemIds))
? changes
: null
}
@@ -0,0 +1,39 @@
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { isRootAgentJournalItem } from '../../../shared/agent-session/journal/agent-session-journal-producer'
import { isAgentSessionContextClear } from '../../../shared/agent-session/agent-session-context-clear'
import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/agent-session-turn-record'
import {
namedSubagents,
subagentFoldKey,
subagentItemBlock
} from '../../../shared/native-chat/transcript/native-chat-subagent-items'
import { isSubagentRow } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
/** Facts history selects before decoding a body; written from each current item revision. */
export type JournalDomainHistoryMetadata = {
root: number
subagentRow: number
subagentGroupKey: string | null
namedSubagentIds: string | null
turnMarker: number
}
export function journalDomainHistoryMetadata(
item: AgentJournalRenderItem
): JournalDomainHistoryMetadata {
const root = isRootAgentJournalItem(item)
const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null
const named =
root && item.body.kind === 'message'
? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id)
: []
return {
root: Number(root),
subagentRow: Number(isSubagentRow(item)),
subagentGroupKey: block ? subagentFoldKey(block) : null,
namedSubagentIds: named.length > 0 ? JSON.stringify(named) : null,
turnMarker: Number(
readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body)
)
}
}
@@ -0,0 +1,284 @@
import type Database from '../../sqlite/sync-database'
import {
AGENT_SESSION_HISTORY_MAX_LIMIT,
type AgentSessionHistoryPage,
type AgentSessionHistoryRequest,
type AgentSessionHistoryResult
} from '../../../shared/agent-session/agent-session-wire'
import type {
AgentJournalRenderItem,
AgentJournalSnapshot,
AgentJournalSubmission
} from '../../../shared/agent-session/journal/agent-session-journal-types'
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items'
import {
historyContentBytes,
oversizedHistoryItem,
resolveHistoryLimit,
submissionBytesByItemId,
type HistoryReader
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result'
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
import { CLIENT_HISTORY_READER } from '../agent-session-wire/publication/history/agent-session-item-wire-form'
import { journalContextEpoch } from './context-epochs/journal-context-cursor'
import { readJournalDomainForwardHistory } from './journal-domain-history-after'
import {
journalDomainSequenceGroups,
selectJournalDomainHistoryWindow
} from './journal-domain-history-window'
export type JournalDomainHistoryRequest = Omit<AgentSessionHistoryRequest, 'direction'> & {
direction: AgentSessionHistoryRequest['direction'] | 'hydrate'
fence?: number
scope?: 'every-agent' | 'own-agent'
rowReplay?: boolean
}
export type JournalDomainHistoryHead = {
sessionId: string
epoch: string
/** Public context identity of the physical epoch. */
cursor: AgentJournalCursor
retainedSequence: number
}
type HistoryReaderForm = Pick<HistoryReader, 'form' | 'budgetBytes'>
/** The caller owns a readonly connection; head, headers, bodies and any reset share one snapshot. */
export function readJournalDomainHistoryPage(
db: Database.Database,
request: JournalDomainHistoryRequest,
reader: HistoryReaderForm = CLIENT_HISTORY_READER
): AgentSessionHistoryResult {
db.exec('BEGIN')
try {
const result = readHistory(db, readHead(db, request.sessionId), request, reader)
db.exec('COMMIT')
return result
} catch (error) {
db.exec('ROLLBACK')
throw error
}
}
function readHead(db: Database.Database, sessionId: string): JournalDomainHistoryHead {
const row = db
.prepare(`SELECT epoch, last_sequence, retained_sequence, context_boundary FROM journal_chat_state
WHERE session_id=?`)
.get(sessionId)
if (!row) {
throw new Error('journal history targets a chat without domain state')
}
const epoch = String(row.epoch)
return {
sessionId,
epoch,
cursor: {
epoch: journalContextEpoch(epoch, Number(row.context_boundary)),
sequence: Number(row.last_sequence)
},
retainedSequence: Number(row.retained_sequence)
}
}
function readHistory(
db: Database.Database,
head: JournalDomainHistoryHead,
request: JournalDomainHistoryRequest,
reader: HistoryReaderForm
): AgentSessionHistoryResult {
const reset = (
reason: Extract<AgentSessionHistoryResult, { ok: false }>['reset']
): AgentSessionHistoryResult => ({
ok: false,
reset: reason,
page: historyPage(
db,
head,
{ ...request, direction: 'hydrate', fence: undefined, scope: 'every-agent' },
reader
)
})
if (request.direction === 'after') {
return readJournalDomainForwardHistory(
db,
head,
request.cursor,
resolveHistoryLimit(request.limit),
reader,
latestTurnItem(db, head),
(ids) => currentRows(db, head, ids),
reset
)
}
const cursor = request.direction === 'before' ? request.cursor : undefined
if (cursor && cursor.epoch !== head.cursor.epoch) {
return reset('epoch_changed')
}
if (cursor && cursor.sequence > head.cursor.sequence) {
return reset('cursor_ahead')
}
return { ok: true, page: historyPage(db, head, request, reader) }
}
function historyPage(
db: Database.Database,
head: JournalDomainHistoryHead,
request: JournalDomainHistoryRequest,
reader: HistoryReaderForm
): AgentSessionHistoryPage {
const prefix = [head.sessionId, head.epoch] as const
const cursor = request.direction === 'before' ? request.cursor : undefined
const end = cursor?.sequence ?? Number.MAX_SAFE_INTEGER
const where = `session_id=? AND epoch=? AND hidden=0${request.scope === 'own-agent' ? ' AND root=1' : ''}`
const limit =
request.direction === 'hydrate'
? AGENT_SESSION_HISTORY_MAX_LIMIT
: resolveHistoryLimit(request.limit)
const window = selectJournalDomainHistoryWindow(db, where, prefix, end, limit)
const submissions = itemSubmissions(
db,
head,
window.map((header) => header.itemId)
)
const sizes = submissionBytesByItemId(submissions)
const body = db.prepare(
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
)
const kept: AgentJournalRenderItem[][] = []
let total = 0
for (const group of journalDomainSequenceGroups(window)) {
const formed = group.map((header) =>
reader.form(JSON.parse(String(body.get(...prefix, header.itemId)!.payload_json)))
)
const bytes = historyContentBytes(formed, [], sizes)
if (kept.length > 0 && total + bytes > reader.budgetBytes) {
break
}
kept.push(
formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item))
)
total += bytes
if (bytes > reader.budgetBytes) {
break
}
}
const items = kept.toReversed().flat()
const oldest = items[0]?.sequence ?? end
const turn = latestTurnItem(db, head)
return buildAgentSessionHistoryPage({
snapshot: {
sessionId: head.sessionId,
cursor: head.cursor,
items: turn ? [turn] : [],
submissions
},
direction: request.direction === 'hydrate' ? 'tail' : request.direction,
items,
hasOlder: !!db
.prepare(`SELECT 1 FROM journal_items WHERE ${where} AND sequence<? LIMIT 1`)
.get(...prefix, oldest),
hasNewer:
cursor !== undefined &&
!!db
.prepare(`SELECT 1 FROM journal_items WHERE ${where} AND sequence>=? LIMIT 1`)
.get(...prefix, end),
fallbackCursor: cursor ?? { epoch: head.cursor.epoch, sequence: 0 },
nextCursor: items[0] ? { epoch: head.cursor.epoch, sequence: items[0].sequence } : undefined,
fence: request.fence,
subagentRoster: historyRoster(db, prefix, items)
})
}
function latestTurnItem(
db: Database.Database,
head: JournalDomainHistoryHead
): AgentJournalRenderItem | null {
const row = db
.prepare(`SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND hidden=0
AND turn_marker=1 ORDER BY sequence DESC, sequence_index DESC, ordinal DESC LIMIT 1`)
.get(head.sessionId, head.epoch)
return row ? JSON.parse(String(row.payload_json)) : null
}
/** Current attempts only, in submission order then stable admission ordinal. */
function itemSubmissions(
db: Database.Database,
head: JournalDomainHistoryHead,
itemIds: readonly string[]
): AgentJournalSubmission[] {
return db
.prepare(`SELECT attempt.payload_json FROM json_each(?) wanted
CROSS JOIN journal_submission_attempts attempt
WHERE attempt.session_id=? AND attempt.epoch=? AND attempt.current=1 AND attempt.item_id=wanted.value
ORDER BY attempt.submitted_at, attempt.ordinal`)
.all(JSON.stringify(itemIds), head.sessionId, head.epoch)
.map((row) => JSON.parse(String(row.payload_json)))
}
function currentRows(
db: Database.Database,
head: JournalDomainHistoryHead,
ids: ReadonlySet<string>
): AgentJournalSnapshot {
const wanted = [...ids]
return {
sessionId: head.sessionId,
cursor: head.cursor,
items: db
.prepare(`SELECT item.payload_json FROM json_each(?) wanted
CROSS JOIN journal_items item
WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0
ORDER BY item.sequence, item.sequence_index, item.ordinal`)
.all(JSON.stringify(wanted), head.sessionId, head.epoch)
.map((row) => JSON.parse(String(row.payload_json))),
submissions: itemSubmissions(db, head, wanted)
}
}
/** Reads only root rows naming a subagent the page shows but does not name. */
function historyRoster(
db: Database.Database,
prefix: readonly [string, string],
page: readonly AgentJournalRenderItem[]
) {
const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null))
for (const item of page) {
if (item.agentId === undefined && item.body.kind === 'message') {
for (const entry of item.body.blocks.flatMap(namedSubagents)) {
missing.delete(entry.id)
}
}
}
if (missing.size === 0) {
return undefined
}
const body = db.prepare(
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
)
const named: AgentJournalRenderItem[] = []
for (const row of db
.prepare(`SELECT item_id, named_subagent_ids FROM journal_items
WHERE session_id=? AND epoch=? AND hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL
ORDER BY sequence, sequence_index, ordinal`)
.iterate(...prefix)) {
const ids: string[] = JSON.parse(String(row.named_subagent_ids))
if (!ids.some((id) => missing.has(id))) {
continue
}
for (const id of ids) {
missing.delete(id)
}
named.push(JSON.parse(String(body.get(...prefix, String(row.item_id))!.payload_json)))
if (missing.size === 0 || named.length >= ROSTER_ROWS) {
break
}
}
return offPageSubagentRoster(named, page)
}
// The roster keeps at most 64 entries; each selected row adds at least one.
const ROSTER_ROWS = 64
@@ -0,0 +1,107 @@
import type Database from '../../sqlite/sync-database'
import type { SqliteRow } from '../../sqlite/sqlite-statement'
export type JournalDomainHistoryHeader = {
itemId: string
sequence: number
own: boolean
subagentGroupKey: string | null
}
/** Chooses the page's candidate identities before any body or submission is decoded. */
export function selectJournalDomainHistoryWindow(
db: Database.Database,
where: string,
prefix: readonly [string, string],
end: number,
limit: number
): JournalDomainHistoryHeader[] {
return historyWindow(selectJournalDomainHistoryHeaders(db, where, prefix, end, limit), limit)
}
/** Newest-first keyset over every scoped row: the existing 2×limit reach also caps sparse-root chats.
* An epoch+sequence cursor cannot split a sequence, so the boundary sequence is completed whole. */
function selectJournalDomainHistoryHeaders(
db: Database.Database,
where: string,
prefix: readonly [string, string],
end: number,
limit: number
): JournalDomainHistoryHeader[] {
const columns = 'item_id, sequence, root, subagent_row, subagent_group_key'
const order = 'ORDER BY sequence DESC, sequence_index DESC, ordinal DESC'
const cap = 2 * limit
const rows = db
.prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence<? ${order} LIMIT ?`)
.all(...prefix, end, cap)
if (rows.length === cap) {
const boundary = rows.at(-1)!.sequence
const first = rows.findIndex((row) => row.sequence === boundary)
rows.splice(
first,
rows.length - first,
...db
.prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence=? ${order}`)
.all(...prefix, boundary)
)
}
return rows.map((row: SqliteRow) => ({
itemId: String(row.item_id),
sequence: Number(row.sequence),
own: row.root === 1 && row.subagent_row === 0,
subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key)
}))
}
/** Window by own-agent rows, retaining interleaved subagent work and its whole groups within the candidates. */
function historyWindow(
headers: readonly JournalDomainHistoryHeader[],
limit: number
): JournalDomainHistoryHeader[] {
let own = 0
let start: number | undefined
for (const group of journalDomainSequenceGroups(headers)) {
const count = group.filter((header) => header.own).length
if (count === 0) {
continue
}
if (own > 0 && own + count > limit) {
const firstOf = new Map<string, number>()
for (const header of headers) {
if (header.subagentGroupKey !== null) {
firstOf.set(header.subagentGroupKey, header.sequence)
}
}
let whole = start!
for (const header of headers) {
if (header.sequence < whole) {
break
}
if (header.subagentGroupKey !== null) {
whole = Math.min(whole, firstOf.get(header.subagentGroupKey)!)
}
}
return headers.filter((header) => header.sequence >= whole)
}
own += count
start = group[0]!.sequence
}
return [...headers]
}
/** Newest group first; each group keeps render order. */
export function* journalDomainSequenceGroups<T extends { sequence: number }>(
rows: readonly T[]
): Generator<T[]> {
let group: T[] = []
for (const row of rows) {
if (group.length > 0 && row.sequence !== group[0]!.sequence) {
yield group.toReversed()
group = []
}
group.push(row)
}
if (group.length > 0) {
yield group.toReversed()
}
}
@@ -0,0 +1,349 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, expect, it } from 'vitest'
import Database from '../../sqlite/sync-database'
import type { AgentSessionHistoryResult } from '../../../shared/agent-session/agent-session-wire'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { measureHistoryItem } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
import { message, messageFixture, target } from './journal-sql-message.test-fixture'
import {
readJournalDomainHistoryPage,
type JournalDomainHistoryRequest
} from './journal-domain-history-page'
import { migrateJournalDomainSchema } from './journal-domain-schema'
const cleanup: (() => void)[] = []
afterEach(() => {
for (const step of cleanup.splice(0)) {
step()
}
})
function history(db = new Database(':memory:')) {
const value = messageFixture(db)
cleanup.push(() => db.close())
return {
...value,
head: (sequence: number, retained = 0) =>
db
.prepare('UPDATE journal_chat_state SET last_sequence=?, retained_sequence=?')
.run(sequence, retained),
event: (sequence: number, itemIds: string[]) =>
db.prepare('INSERT INTO journal_events VALUES(?,?,?,0,?)').run(
target.sessionId,
target.epoch,
sequence,
JSON.stringify({
items: itemIds.map((itemId) => ({ itemId, revision: 1, sequence, sequenceIndex: 0 })),
submissions: [],
cards: []
})
),
read: (
request: Omit<JournalDomainHistoryRequest, 'sessionId'>,
reader?: Parameters<typeof readJournalDomainHistoryPage>[2]
) => readJournalDomainHistoryPage(db, { sessionId: target.sessionId, ...request }, reader)
}
}
function text(
sequence: number,
value: string,
extra: Partial<AgentJournalRenderItem> = {}
): AgentJournalRenderItem {
return {
itemId: `orca:${sequence}:${extra.sequenceIndex ?? 0}`,
revision: 1,
sequence,
observedAt: sequence,
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: value }] },
turnScope: { kind: 'thread' },
...extra
}
}
const sequences = (result: AgentSessionHistoryResult) =>
result.page.items.map((item) => item.sequence)
it('pages a chat of almost only subagent rows in bounded, gap-free pages', () => {
const { seedItem, head, read } = history()
seedItem({
...text(1, ''),
body: {
kind: 'message',
role: 'assistant',
blocks: [{ type: 'subagent', id: 'child', label: 'Child', state: 'working', groupId: 'g' }]
}
})
for (let sequence = 2; sequence <= 301; sequence++) {
seedItem(text(sequence, 'child work', { agentId: 'child' }))
}
head(301)
let result = read({ direction: 'tail', limit: 40 })
expect(result.page.subagentRoster).toMatchObject([
{ itemId: 'orca:1:0', sequence: 1, entry: { id: 'child', label: 'Child' } }
])
const sizes: number[] = []
const seen: number[] = []
for (;;) {
expect(result.ok).toBe(true)
sizes.push(result.page.items.length)
seen.unshift(...sequences(result))
if (!result.page.hasOlder) {
break
}
result = read({ direction: 'before', cursor: result.page.window.nextCursor, limit: 40 })
}
expect(sizes).toEqual([80, 80, 80, 61])
expect(seen).toEqual(Array.from({ length: 301 }, (_, index) => index + 1))
})
it('never splits a same-sequence group, marking it oversized backward and paging it whole forward', () => {
const { seedItem, head, event, read } = history()
const group = Array.from({ length: 100 }, (_, index) =>
text(50, 'x'.repeat(1000), { sequenceIndex: index })
)
seedItem(text(49, 'before'))
event(49, ['orca:49:0'])
for (const item of group) {
seedItem(item)
}
event(
50,
group.map((item) => item.itemId)
)
for (let sequence = 51; sequence <= 60; sequence++) {
seedItem(text(sequence, 'after'))
event(sequence, [`orca:${sequence}:0`])
}
head(60, 48)
const small = { form: measureHistoryItem, budgetBytes: 50_000 }
const tail = read({ direction: 'tail', limit: 40 }, small)
expect(sequences(tail)).toEqual([51, 52, 53, 54, 55, 56, 57, 58, 59, 60])
const wide = read({ direction: 'before', cursor: tail.page.window.nextCursor, limit: 40 }, small)
expect(wide.page.items.map((item) => item.itemId)).toEqual(group.map((item) => item.itemId))
expect(new Set(wide.page.items.map((item) => item.body.kind))).toEqual(new Set(['status']))
expect(wide.page).toMatchObject({ hasOlder: true, window: { nextCursor: { sequence: 50 } } })
const rest = read({ direction: 'before', cursor: wide.page.window.nextCursor, limit: 40 }, small)
expect(sequences(rest)).toEqual([49])
expect(rest.page.hasOlder).toBe(false)
const forward: string[] = []
let cursor = { epoch: 'epoch', sequence: 48 }
for (const expected of [true, true, false]) {
const page = read({ direction: 'after', cursor, limit: 4 })
expect(page).toMatchObject({ ok: true, page: { hasNewer: expected } })
forward.push(...page.page.items.map((item) => item.itemId))
cursor = page.page.window.nextCursor
}
expect(cursor.sequence).toBe(60)
expect(forward).toEqual([
'orca:49:0',
...group.map((item) => item.itemId),
...Array.from({ length: 10 }, (_, index) => `orca:${51 + index}:0`)
])
})
it('catches up with the current revision, placement and submission, not the replayed metadata', async () => {
const { db, commit, seedSend, read } = history()
seedSend()
db.exec(`INSERT INTO journal_submission_attempts
SELECT session_id,epoch,message_id,0,0,0,item_id,0,0,NULL,fingerprint,'unknown',0,NULL,NULL,'{}'
FROM journal_submission_attempts WHERE current=1`)
await commit(message('first'))
await commit(message('second'))
const echo = message('original', 'unused', 'item')
echo.identity = { provider: 'codex', threadId: 'thread', turnId: 'turn', ordinal: 0 }
echo.body.role = 'user'
await commit(echo)
const result = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 10 } })
expect(result).toMatchObject({
ok: true,
page: {
items: [
{ itemId: 'orca:send', sequence: 7, body: { blocks: [{ text: 'original' }] } },
{
itemId: 'orca:assistant',
revision: 2,
sequence: 11,
body: { blocks: [{ text: 'second' }] }
}
],
submissions: [{ clientMessageId: 'send', dispatchState: 'accepted', acceptedSequence: 1 }],
window: { nextCursor: { epoch: 'epoch', sequence: 13 } },
hasNewer: false
}
})
const child = message('child work', 'child', 'item')
child.options.agentId = 'child'
await commit(child)
expect(
read({ direction: 'tail', scope: 'own-agent' }).page.items.map((item) => item.itemId)
).toEqual(['orca:send', 'orca:assistant'])
expect(read({ direction: 'tail' }).page.items.map((item) => item.itemId)).toEqual([
'orca:send',
'orca:assistant',
'orca:child'
])
})
it('resets a cursor below the pruned boundary or across a gap, and resumes at the boundary', () => {
const { db, seedItem, head, event, read } = history()
for (let sequence = 1; sequence <= 3; sequence++) {
seedItem(text(sequence, 'row'))
}
event(3, ['orca:3:0'])
head(3, 2)
expect(read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 1 } })).toMatchObject({
ok: false,
reset: 'cursor_compacted',
page: { direction: 'tail', items: [{ sequence: 1 }, { sequence: 2 }, { sequence: 3 }] }
})
const resumed = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 2 } })
expect(resumed.ok).toBe(true)
expect(sequences(resumed)).toEqual([3])
db.exec('DELETE FROM journal_events')
expect(read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 2 } })).toMatchObject({
ok: false,
reset: 'journal_gap'
})
})
it('answers every selected read from the snapshot its head came from', () => {
const dir = mkdtempSync(join(tmpdir(), 'journal-domain-history-'))
const path = join(dir, 'chat.db')
const writer = new Database(path)
writer.pragma('journal_mode=WAL')
const { seedItem, head } = history(writer)
seedItem(text(1, 'first'))
seedItem({
itemId: 'orca:turn',
revision: 1,
sequence: 2,
observedAt: 2,
body: { kind: 'turn', turnId: 'turn', state: 'running', startedAt: 2 },
turnScope: { kind: 'thread' }
})
head(2)
const reader = new Database(path, { readonly: true })
cleanup.push(
() => reader.close(),
() => rmSync(dir, { recursive: true })
)
let wrote = false
const result = readJournalDomainHistoryPage(
reader,
{ sessionId: target.sessionId, direction: 'tail' },
{
budgetBytes: 1024 * 1024,
form: (item) => {
if (!wrote) {
wrote = true
writer.exec('DELETE FROM journal_items; UPDATE journal_chat_state SET last_sequence=9')
}
return measureHistoryItem(item)
}
}
)
expect(result).toMatchObject({
ok: true,
page: {
items: [{ itemId: 'orca:1:0' }, { itemId: 'orca:turn' }],
liveCursor: { epoch: 'epoch', sequence: 2 },
latestTurn: { itemId: 'orca:turn', turn: { state: 'running' } },
hasOlder: false
}
})
expect(writer.prepare('SELECT COUNT(*) AS count FROM journal_items').get()!.count).toBe(0)
})
it('backfills history metadata once from saved v1 rows and refuses a newer domain schema', () => {
const db = new Database(':memory:')
cleanup.push(() => db.close())
migrateJournalDomainSchema(db)
db.exec(`DROP INDEX journal_items_root_position;
DROP INDEX journal_items_turn_marker;
DROP INDEX journal_items_named_subagents;
ALTER TABLE journal_items DROP COLUMN hidden;
ALTER TABLE journal_items DROP COLUMN root;
ALTER TABLE journal_items DROP COLUMN subagent_row;
ALTER TABLE journal_items DROP COLUMN subagent_group_key;
ALTER TABLE journal_items DROP COLUMN named_subagent_ids;
ALTER TABLE journal_items DROP COLUMN turn_marker;
DELETE FROM journal_domain_migrations WHERE version=2`)
const rows: AgentJournalRenderItem[] = [
{
...text(1, ''),
body: {
kind: 'message',
role: 'assistant',
blocks: [{ type: 'subagent', id: 'child', label: 'Child', state: 'working', groupId: 'g' }]
}
},
text(2, 'child work', { agentId: 'child' }),
{
itemId: 'orca:turn',
revision: 3,
sequence: 3,
observedAt: 3,
body: { kind: 'turn', turnId: 'turn', state: 'completed' },
turnScope: { kind: 'thread' }
}
]
for (const item of rows) {
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json)
VALUES('chat','epoch',?,?,?,?,0,1,?)`).run(
item.itemId,
item.sequence,
item.revision,
item.sequence,
JSON.stringify(item)
)
}
migrateJournalDomainSchema(db)
expect(
db
.prepare(`SELECT item_id, revision, hidden, root, subagent_row, subagent_group_key,
named_subagent_ids, turn_marker, payload_json FROM journal_items ORDER BY sequence`)
.all()
).toEqual([
{
item_id: 'orca:1:0',
revision: 1,
hidden: 0,
root: 1,
subagent_row: 1,
subagent_group_key: 'g',
named_subagent_ids: '["child"]',
turn_marker: 0,
payload_json: JSON.stringify(rows[0])
},
{
item_id: 'orca:2:0',
revision: 1,
hidden: 0,
root: 0,
subagent_row: 0,
subagent_group_key: null,
named_subagent_ids: null,
turn_marker: 0,
payload_json: JSON.stringify(rows[1])
},
{
item_id: 'orca:turn',
revision: 3,
hidden: 0,
root: 1,
subagent_row: 0,
subagent_group_key: null,
named_subagent_ids: null,
turn_marker: 1,
payload_json: JSON.stringify(rows[2])
}
])
db.exec('INSERT INTO journal_domain_migrations(version) VALUES (3)')
expect(() => migrateJournalDomainSchema(db)).toThrow(
'journal domain schema is newer than this writer'
)
})
@@ -1,4 +1,17 @@
import type Database from '../../sqlite/sync-database'
import type { SqliteBindings } from '../../sqlite/sqlite-statement'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
const JOURNAL_DOMAIN_VERSION = 2
/** journal_events.changes_json: identities only; readers load current rows for bodies. */
export type JournalDomainEventChanges = {
items: { itemId: string; revision: number; sequence: number; sequenceIndex: number }[]
submissions: string[]
cards: string[]
removedItemIds?: string[]
}
/** This ledger deliberately leaves the profile's global user_version unchanged. */
export function migrateJournalDomainSchema(db: Database.Database): void {
@@ -8,15 +21,70 @@ export function migrateJournalDomainSchema(db: Database.Database): void {
const version = Number(
db.prepare('SELECT MAX(version) AS version FROM journal_domain_migrations').get()?.version ?? 0
)
if (version === 1) {
if (version === JOURNAL_DOMAIN_VERSION) {
return
}
if (version > 1) {
if (version > JOURNAL_DOMAIN_VERSION) {
throw new Error('journal domain schema is newer than this writer')
}
db.exec('BEGIN IMMEDIATE')
try {
db.exec(`
if (version < 1) {
db.exec(JOURNAL_DOMAIN_V1)
}
migrateJournalDomainHistory(db)
db.exec('COMMIT')
} catch (error) {
db.exec('ROLLBACK')
throw error
}
}
/** v2: history selects windows by these columns, so existing rows are backfilled once here. */
function migrateJournalDomainHistory(db: Database.Database): void {
db.exec(`
ALTER TABLE journal_items ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0;
ALTER TABLE journal_items ADD COLUMN root INTEGER NOT NULL DEFAULT 1;
ALTER TABLE journal_items ADD COLUMN subagent_row INTEGER NOT NULL DEFAULT 0;
ALTER TABLE journal_items ADD COLUMN subagent_group_key TEXT;
ALTER TABLE journal_items ADD COLUMN named_subagent_ids TEXT;
ALTER TABLE journal_items ADD COLUMN turn_marker INTEGER NOT NULL DEFAULT 0;
CREATE INDEX journal_items_root_position ON journal_items(session_id, epoch, sequence, sequence_index, ordinal)
WHERE hidden=0 AND root=1;
CREATE INDEX journal_items_turn_marker ON journal_items(session_id, epoch, sequence, sequence_index, ordinal)
WHERE hidden=0 AND turn_marker=1;
CREATE INDEX journal_items_named_subagents ON journal_items(session_id, epoch, sequence, sequence_index, ordinal)
WHERE hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL;
`)
const page = db.prepare(`SELECT session_id, epoch, item_id, payload_json FROM journal_items
WHERE (session_id, epoch, item_id) > (?, ?, ?) ORDER BY session_id, epoch, item_id LIMIT 500`)
const update = db.prepare(`UPDATE journal_items SET root=?, subagent_row=?, subagent_group_key=?,
named_subagent_ids=?, turn_marker=? WHERE session_id=? AND epoch=? AND item_id=?`)
let after: SqliteBindings = ['', '', '']
for (let rows = page.all(...after); rows.length > 0; rows = page.all(...after)) {
for (const row of rows) {
const item: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const metadata = journalDomainHistoryMetadata(item)
update.run(
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.namedSubagentIds,
metadata.turnMarker,
row.session_id,
row.epoch,
row.item_id
)
}
const last = rows.at(-1)!
after = [last.session_id, last.epoch, last.item_id]
}
db.prepare('INSERT INTO journal_domain_migrations(version) VALUES (?)').run(
JOURNAL_DOMAIN_VERSION
)
}
const JOURNAL_DOMAIN_V1 = `
CREATE TABLE journal_chat_state (
session_id TEXT PRIMARY KEY,
epoch TEXT NOT NULL,
@@ -117,10 +185,4 @@ CREATE TABLE journal_events (
PRIMARY KEY(session_id, epoch, sequence)
) WITHOUT ROWID;
INSERT INTO journal_domain_migrations(version) VALUES (1);
`)
db.exec('COMMIT')
} catch (error) {
db.exec('ROLLBACK')
throw error
}
}
`
@@ -7,6 +7,7 @@ import type {
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation'
import { migrateJournalDomainSchema } from './journal-domain-schema'
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
import { JournalSqlBatch } from './journal-sql-batch'
import {
commitJournalSqlMessages,
@@ -29,8 +30,7 @@ export function message(
}
}
export function messageFixture() {
const db = new Database(':memory:')
export function messageFixture(db = new Database(':memory:')) {
db.pragma('user_version=4')
migrateJournalDomainSchema(db)
db.prepare(
@@ -54,21 +54,27 @@ export function messageFixture() {
const row = db.prepare('SELECT payload_json FROM journal_items WHERE item_id=?').get(itemId)
return row ? JSON.parse(String(row.payload_json)) : undefined
}
const seedItem = (item: AgentJournalRenderItem) =>
db
.prepare(`INSERT INTO journal_items
(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json) VALUES(?,?,?,?,?,?,?,?,?)`)
.run(
target.sessionId,
target.epoch,
item.itemId,
item.sequence,
item.revision,
item.sequence,
item.sequenceIndex ?? 0,
1,
JSON.stringify(item)
)
const seedItem = (item: AgentJournalRenderItem) => {
const metadata = journalDomainHistoryMetadata(item)
db.prepare(`INSERT INTO journal_items
(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json,
root,subagent_row,subagent_group_key,named_subagent_ids,turn_marker) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(
target.sessionId,
target.epoch,
item.itemId,
item.sequence,
item.revision,
item.sequence,
item.sequenceIndex ?? 0,
1,
JSON.stringify(item),
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.namedSubagentIds,
metadata.turnMarker
)
}
const seedSend = () => {
const body: AgentJournalMessageItem = {
kind: 'message',
@@ -32,6 +32,8 @@ import { isJournalTextRevision } from './writing/journal-streamed-text-revision'
import { assertJournalFence } from './writing/journal-write-guards'
import type { JournalItemRow } from './durable-rows/journal-row-schema'
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
import type { JournalDomainEventChanges } from './journal-domain-schema'
import {
readJournalSqlAcceptedInputs,
type JournalSqlAcceptedInputs
@@ -206,10 +208,14 @@ export function applyJournalSqlMessage(
cards: [],
inputChanged: false
}
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json)
VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
const metadata = journalDomainHistoryMetadata(next)
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json,
root,subagent_row,subagent_group_key,named_subagent_ids,turn_marker)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index,
execution_owner=excluded.execution_owner, payload_json=excluded.payload_json`).run(
execution_owner=excluded.execution_owner, payload_json=excluded.payload_json,
root=excluded.root, subagent_row=excluded.subagent_row, subagent_group_key=excluded.subagent_group_key,
named_subagent_ids=excluded.named_subagent_ids, turn_marker=excluded.turn_marker`).run(
...prefix,
itemId,
stored?.ordinal ?? sequence,
@@ -218,7 +224,12 @@ export function applyJournalSqlMessage(
next.sequenceIndex ?? 0,
row.ownerFence!,
row.executionOwner ?? stored?.execution_owner ?? null,
JSON.stringify(next)
JSON.stringify(next),
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.namedSubagentIds,
metadata.turnMarker
)
db.prepare(
'DELETE FROM journal_item_tombstones WHERE session_id=? AND epoch=? AND item_id=?'
@@ -267,7 +278,7 @@ export function applyJournalSqlMessage(
],
submissions: result.receipts.map((receipt) => receipt.clientMessageId),
cards: result.cards.map((card) => card.messageId)
})
} satisfies JournalDomainEventChanges)
)
return result
}