diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts index a57fc879f75..00f3e6d6adb 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts @@ -8,7 +8,9 @@ import type { import { readAgentSessionForwardHistory } from '../agent-session-wire/publication/history/agent-session-history-forward-page' import type { HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' import type { JournalReadSince } from './journal-store-contracts' -import type { JournalOutcomeReadRow } from './replay/journal-read-row' +import type { JournalOutcomeReadRow, JournalReadRow } from './replay/journal-read-row' +import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { JOURNAL_DOMAIN_HISTORY_ITEM_BYTES } from './journal-domain-history-window' import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor' import type { JournalDomainEventChanges } from './journal-domain-schema' import type { JournalDomainHistoryHead } from './journal-domain-history-page' @@ -27,9 +29,17 @@ export function readJournalDomainForwardHistory( reason: Extract['reset'] ) => AgentSessionHistoryResult ): AgentSessionHistoryResult { + // One bounded identity read serves the size admission and the builder's lookahead. + const since = cursor && readJournalDomainEvents(db, head, cursor, limit + 1) + const admitted = since?.ok + ? fittingEventCount(db, head, since.rows.slice(0, limit), reader.budgetBytes) + : limit return readAgentSessionForwardHistory( { - readSince: (after, count) => readJournalDomainEvents(db, head, after, count), + readSince: (after, count = limit + 1) => + since?.ok + ? firstEvents(since, count) + : (since ?? readJournalDomainEvents(db, head, after, count)), canonicalItemId: (itemId) => itemId, hasSubmission: () => false, stopNoteIds: (ids) => journalDomainDependentStopNotes(db, head, ids), @@ -42,18 +52,65 @@ export function readJournalDomainForwardHistory( submissions: [] }, cursor, - limit, + admitted, reader, readSnapshot ) } +/** The longest event prefix whose touched rows' stored sizes fit, so excluded bodies are never read. */ +function fittingEventCount( + db: Database.Database, + head: JournalDomainHistoryHead, + rows: readonly JournalReadRow[], + budgetBytes: number +): number { + const touched = rows.map((row) => + row.kind === 'outcome' + ? [...row.itemIds, ...row.submissionIds.map(agentJournalSubmissionKey)] + : [] + ) + const sizes = new Map( + db + .prepare(`SELECT item.item_id, ${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes FROM json_each(?) wanted + CROSS JOIN journal_items item + WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0`) + .all(JSON.stringify([...new Set(touched.flat())]), head.sessionId, head.epoch) + .map((row) => [String(row.item_id), Number(row.bytes)]) + ) + const counted = new Set() + let total = 0 + let count = 0 + for (const ids of touched) { + for (const id of ids) { + if (!counted.has(id)) { + counted.add(id) + total += sizes.get(id) ?? 0 + } + } + if (count > 0 && total > budgetBytes) { + break + } + count += 1 + } + return Math.max(count, 1) +} + +function firstEvents( + since: Extract, + count: number +): JournalReadSince { + return since.rows.length <= count + ? since + : { ...since, rows: since.rows.slice(0, count), through: since.rows[count - 1]!.seq } +} + /** Retained events start after the pruned-through cursor; a missing sequence resets, never pages partially. */ function readJournalDomainEvents( db: Database.Database, head: JournalDomainHistoryHead, cursor: AgentJournalCursor, - limit = -1 + limit: number ): JournalReadSince { const resume = resolveJournalResume( { @@ -70,7 +127,7 @@ function readJournalDomainEvents( .prepare(`SELECT sequence, changes_json FROM journal_events WHERE session_id=? AND epoch=? AND sequence>? ORDER BY sequence LIMIT ?`) .all(head.sessionId, head.epoch, resume.afterSequence, limit) - const complete = limit < 0 || stored.length < limit + const complete = stored.length < limit const sequences = stored.map((row) => Number(row.sequence)) if ( findSequenceGap(sequences, resume.afterSequence + 1) || diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-bounds-schema.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-bounds-schema.ts new file mode 100644 index 00000000000..2863d7a82a7 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-bounds-schema.ts @@ -0,0 +1,65 @@ +import type Database from '../../sqlite/sync-database' +import type { SqliteBindings } from '../../sqlite/sqlite-statement' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { + journalDomainHistoryMetadata, + type JournalDomainHistoryMetadata +} from './journal-domain-history-metadata' + +type HistoryTarget = { sessionId: string; epoch: string } + +/** Next numbered migration body: sent sizes and subagent names history selects before any body. */ +export function migrateJournalDomainHistoryBounds(db: Database.Database): void { + db.exec(` +ALTER TABLE journal_items ADD COLUMN wire_bytes INTEGER; +CREATE TABLE journal_item_subagent_names ( + session_id TEXT NOT NULL, + epoch TEXT NOT NULL, + subagent_id TEXT NOT NULL, + item_id TEXT NOT NULL, + PRIMARY KEY(session_id, epoch, subagent_id, item_id) +) WITHOUT ROWID; +CREATE INDEX journal_item_subagent_names_item ON journal_item_subagent_names(session_id, epoch, item_id); +DROP INDEX journal_items_named_subagents; +`) + const page = db.prepare(`SELECT session_id, epoch, item_id, payload_json FROM journal_items + WHERE (session_id, epoch, item_id) > (?, ?, ?) ORDER BY session_id, epoch, item_id LIMIT 500`) + let after: SqliteBindings = ['', '', ''] + for (let rows = page.all(...after); rows.length > 0; rows = page.all(...after)) { + for (const row of rows) { + const payload = String(row.payload_json) + const item: AgentJournalRenderItem = JSON.parse(payload) + writeJournalDomainHistoryBounds( + db, + { sessionId: String(row.session_id), epoch: String(row.epoch) }, + String(row.item_id), + journalDomainHistoryMetadata(item, payload) + ) + } + const last = rows.at(-1)! + after = [last.session_id, last.epoch, last.item_id] + } +} + +/** Every journal_items upsert calls this after writing the row, in the same transaction. */ +export function writeJournalDomainHistoryBounds( + db: Database.Database, + target: HistoryTarget, + itemId: string, + metadata: Pick +): void { + const key = [target.sessionId, target.epoch, itemId] as const + db.prepare( + 'UPDATE journal_items SET wire_bytes=? WHERE session_id=? AND epoch=? AND item_id=?' + ).run(metadata.wireBytes, ...key) + db.prepare( + 'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?' + ).run(...key) + const insert = + db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(session_id,epoch,subagent_id,item_id) + VALUES(?,?,?,?)`) + const named: string[] = JSON.parse(metadata.namedSubagentIds ?? '[]') + for (const subagentId of named) { + insert.run(target.sessionId, target.epoch, subagentId, itemId) + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts index 755e87de2a0..c8ae5f39977 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-metadata.ts @@ -7,7 +7,11 @@ import { subagentFoldKey, subagentItemBlock } from '../../../shared/native-chat/transcript/native-chat-subagent-items' -import { isSubagentRow } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import { + HISTORY_PAGE_CONTENT_BUDGET_BYTES, + isSubagentRow +} from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import { agentJournalItemForWire } from '../agent-session-wire/publication/history/agent-session-item-wire-form' /** Facts history selects before decoding a body; written from each current item revision. */ export type JournalDomainHistoryMetadata = { @@ -16,10 +20,14 @@ export type JournalDomainHistoryMetadata = { subagentGroupKey: string | null namedSubagentIds: string | null turnMarker: number + /** Client-form bytes when they differ from the stored payload; null reads the stored length. */ + wireBytes: number | null } +/** Pass the stored payload text when the caller already serialized it. */ export function journalDomainHistoryMetadata( - item: AgentJournalRenderItem + item: AgentJournalRenderItem, + payloadJson?: string ): JournalDomainHistoryMetadata { const root = isRootAgentJournalItem(item) const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null @@ -34,6 +42,22 @@ export function journalDomainHistoryMetadata( namedSubagentIds: named.length > 0 ? JSON.stringify(named) : null, turnMarker: Number( readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body) - ) + ), + wireBytes: wireBytes(item, payloadJson) } } + +// The client form cuts tool/diff heads and whole items over the page budget; skipping the +// form elsewhere only overstates a size, which shortens a page but never drops content. +function wireBytes(item: AgentJournalRenderItem, payloadJson: string | undefined): number | null { + if ( + item.body.kind !== 'tool-call' && + item.body.kind !== 'diff' && + Buffer.byteLength(payloadJson ?? JSON.stringify(item), 'utf8') <= + HISTORY_PAGE_CONTENT_BUDGET_BYTES + ) { + return null + } + const sent = agentJournalItemForWire(item) + return sent.item === item ? null : sent.bytes +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts index 2b797b219d6..b8437ef876c 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts @@ -11,8 +11,6 @@ import type { AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types' import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity' -import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer' -import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items' import { historyContentBytes, oversizedHistoryItem, @@ -21,14 +19,15 @@ import { type HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result' -import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster' import { CLIENT_HISTORY_READER } from '../agent-session-wire/publication/history/agent-session-item-wire-form' import { journalContextEpoch } from './context-epochs/journal-context-cursor' import { readJournalDomainForwardHistory } from './journal-domain-history-after' +import { readJournalDomainHistoryRoster } from './journal-domain-history-roster' import { projectJournalDomainStopNote } from './journal-domain-stop-notes' import { - journalDomainSequenceGroups, - selectJournalDomainHistoryWindow + admitJournalDomainHistoryGroups, + selectJournalDomainHistoryWindow, + type JournalDomainHistoryHeader } from './journal-domain-history-window' export type JournalDomainHistoryRequest = Omit & { @@ -139,40 +138,18 @@ function historyPage( request.direction === 'hydrate' ? AGENT_SESSION_HISTORY_MAX_LIMIT : resolveHistoryLimit(request.limit) - const window = selectJournalDomainHistoryWindow(db, where, prefix, end, limit) + const admitted = admitJournalDomainHistoryGroups( + selectJournalDomainHistoryWindow(db, where, prefix, end, limit), + reader.budgetBytes + ) const submissions = itemSubmissions( db, head, - window.map((header) => header.itemId) + admitted.groups.flat().map((header) => header.itemId) ) - const sizes = submissionBytesByItemId(submissions) - const body = db.prepare( - 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' - ) - const kept: AgentJournalRenderItem[][] = [] - let total = 0 - for (const group of journalDomainSequenceGroups(window)) { - const formed = group.map((header) => - reader.form( - projectJournalDomainStopNote( - db, - head, - JSON.parse(String(body.get(...prefix, header.itemId)!.payload_json)) - ) - ) - ) - const bytes = historyContentBytes(formed, [], sizes) - if (kept.length > 0 && total + bytes > reader.budgetBytes) { - break - } - kept.push( - formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item)) - ) - total += bytes - if (bytes > reader.budgetBytes) { - break - } - } + const kept = admitted.oversized + ? [admitted.groups[0]!.map((header) => oversizedHistoryItem(itemEnvelope(db, prefix, header)))] + : formHistoryGroups(db, head, admitted.groups, submissions, reader) const items = kept.toReversed().flat() const oldest = items[0]?.sequence ?? end const turn = latestTurnItem(db, head) @@ -196,10 +173,63 @@ function historyPage( fallbackCursor: cursor ?? { epoch: head.cursor.epoch, sequence: 0 }, nextCursor: items[0] ? { epoch: head.cursor.epoch, sequence: items[0].sequence } : undefined, fence: request.fence, - subagentRoster: historyRoster(db, prefix, items) + subagentRoster: readJournalDomainHistoryRoster(db, prefix, items) }) } +/** The existing byte rule, authoritative over admitted groups only. */ +function formHistoryGroups( + db: Database.Database, + head: JournalDomainHistoryHead, + groups: readonly JournalDomainHistoryHeader[][], + submissions: readonly AgentJournalSubmission[], + reader: HistoryReaderForm +): AgentJournalRenderItem[][] { + const sizes = submissionBytesByItemId(submissions) + const body = db.prepare( + 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' + ) + const kept: AgentJournalRenderItem[][] = [] + let total = 0 + for (const group of groups) { + const formed = group.map((header) => + reader.form( + projectJournalDomainStopNote( + db, + head, + JSON.parse(String(body.get(head.sessionId, head.epoch, header.itemId)!.payload_json)) + ) + ) + ) + const bytes = historyContentBytes(formed, [], sizes) + if (kept.length > 0 && total + bytes > reader.budgetBytes) { + break + } + kept.push( + formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item)) + ) + total += bytes + if (bytes > reader.budgetBytes) { + break + } + } + return kept +} + +/** An oversized group's placeholders keep each row's identity and placement, never its body. */ +function itemEnvelope( + db: Database.Database, + prefix: readonly [string, string], + header: JournalDomainHistoryHeader +): AgentJournalRenderItem { + const row = db + .prepare(`SELECT json_remove(payload_json, '$.body') AS envelope FROM journal_items + WHERE session_id=? AND epoch=? AND item_id=?`) + .get(...prefix, header.itemId)! + // The placeholder replaces the omitted body. + return JSON.parse(String(row.envelope)) +} + function latestTurnItem( db: Database.Database, head: JournalDomainHistoryHead @@ -245,47 +275,3 @@ function currentRows( submissions: itemSubmissions(db, head, wanted) } } - -/** Reads only root rows naming a subagent the page shows but does not name. */ -function historyRoster( - db: Database.Database, - prefix: readonly [string, string], - page: readonly AgentJournalRenderItem[] -) { - const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null)) - for (const item of page) { - if (item.agentId === undefined && item.body.kind === 'message') { - for (const entry of item.body.blocks.flatMap(namedSubagents)) { - missing.delete(entry.id) - } - } - } - if (missing.size === 0) { - return undefined - } - const body = db.prepare( - 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' - ) - const named: AgentJournalRenderItem[] = [] - for (const row of db - .prepare(`SELECT item_id, named_subagent_ids FROM journal_items - WHERE session_id=? AND epoch=? AND hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL - ORDER BY sequence, sequence_index, ordinal`) - .iterate(...prefix)) { - const ids: string[] = JSON.parse(String(row.named_subagent_ids)) - if (!ids.some((id) => missing.has(id))) { - continue - } - for (const id of ids) { - missing.delete(id) - } - named.push(JSON.parse(String(body.get(...prefix, String(row.item_id))!.payload_json))) - if (missing.size === 0 || named.length >= ROSTER_ROWS) { - break - } - } - return offPageSubagentRoster(named, page) -} - -// The roster keeps at most 64 entries; each selected row adds at least one. -const ROSTER_ROWS = 64 diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts new file mode 100644 index 00000000000..bb197274f37 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts @@ -0,0 +1,51 @@ +import type Database from '../../sqlite/sync-database' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer' +import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items' +import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster' + +/** One indexed lookup per unnamed page subagent finds its earliest visible naming row. */ +export function readJournalDomainHistoryRoster( + db: Database.Database, + prefix: readonly [string, string], + page: readonly AgentJournalRenderItem[] +) { + const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null)) + for (const item of page) { + if (item.agentId === undefined && item.body.kind === 'message') { + for (const entry of item.body.blocks.flatMap(namedSubagents)) { + missing.delete(entry.id) + } + } + } + if (missing.size === 0) { + return undefined + } + const earliest = db.prepare(`SELECT item.item_id, item.sequence, item.sequence_index, item.ordinal + FROM journal_item_subagent_names name CROSS JOIN journal_items item + ON item.session_id=name.session_id AND item.epoch=name.epoch AND item.item_id=name.item_id + WHERE name.session_id=? AND name.epoch=? AND name.subagent_id=? AND item.hidden=0 AND item.root=1 + ORDER BY item.sequence, item.sequence_index, item.ordinal LIMIT 1`) + const naming = new Map() + for (const id of missing) { + const row = earliest.get(...prefix, id) + if (row) { + naming.set(String(row.item_id), [ + Number(row.sequence), + Number(row.sequence_index), + Number(row.ordinal) + ]) + } + } + const body = db.prepare( + 'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' + ) + const named = [...naming] + .sort(([, a], [, b]) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2]) + .slice(0, ROSTER_ROWS) + .map(([itemId]) => JSON.parse(String(body.get(...prefix, itemId)!.payload_json))) + return offPageSubagentRoster(named, page) +} + +// The roster keeps at most 64 entries; each selected row adds at least one. +const ROSTER_ROWS = 64 diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts index 468eb569dc1..7d1282238d8 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-window.ts @@ -7,8 +7,16 @@ export type JournalDomainHistoryHeader = { sequence: number own: boolean subagentGroupKey: string | null + /** Sent item plus current submission bytes, read without loading either body. */ + bytes: number } +/** Requires the item alias; octet_length reads a stored length, never the payload. */ +export const JOURNAL_DOMAIN_HISTORY_ITEM_BYTES = `COALESCE(item.wire_bytes, octet_length(item.payload_json)) + + COALESCE((SELECT octet_length(attempt.payload_json) FROM journal_submission_attempts attempt + WHERE attempt.session_id=item.session_id AND attempt.epoch=item.epoch + AND attempt.item_id=item.item_id AND attempt.current=1), 0)` + /** Chooses the page's candidate identities before any body or submission is decoded. */ export function selectJournalDomainHistoryWindow( db: Database.Database, @@ -29,11 +37,14 @@ function selectJournalDomainHistoryHeaders( end: number, limit: number ): JournalDomainHistoryHeader[] { - const columns = 'item_id, sequence, root, subagent_row, subagent_group_key' + const columns = `item_id, sequence, root, subagent_row, subagent_group_key, + ${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes` const order = 'ORDER BY sequence DESC, sequence_index DESC, ordinal DESC' const cap = 2 * limit const rows = db - .prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS) { @@ -53,7 +64,8 @@ function selectJournalDomainHistoryHeaders( itemId: String(row.item_id), sequence: Number(row.sequence), own: row.root === 1 && row.subagent_row === 0, - subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key) + subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key), + bytes: Number(row.bytes) })) } @@ -93,6 +105,27 @@ function historyWindow( return [...headers] } +/** Newest whole groups whose stored sizes fit; a lone first group over budget is oversized. */ +export function admitJournalDomainHistoryGroups( + window: readonly JournalDomainHistoryHeader[], + budgetBytes: number +): { groups: JournalDomainHistoryHeader[][]; oversized: boolean } { + const groups: JournalDomainHistoryHeader[][] = [] + let total = 0 + for (const group of journalDomainSequenceGroups(window)) { + const bytes = group.reduce((sum, header) => sum + header.bytes, 0) + if (groups.length > 0 && total + bytes > budgetBytes) { + break + } + groups.push(group) + total += bytes + if (bytes > budgetBytes) { + return { groups, oversized: true } + } + } + return { groups, oversized: false } +} + /** Newest group first; each group keeps render order. */ export function* journalDomainSequenceGroups( rows: readonly T[] diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts index e07f8f22dbe..f04e4997b14 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts @@ -14,6 +14,11 @@ import { type JournalDomainHistoryRequest } from './journal-domain-history-page' import { migrateJournalDomainSchema } from './journal-domain-schema' +import { journalDomainHistoryMetadata } from './journal-domain-history-metadata' +import { + migrateJournalDomainHistoryBounds, + writeJournalDomainHistoryBounds +} from './journal-domain-history-bounds-schema' import { JOURNAL_DOMAIN_SCHEMA_VERSION } from './database/journal-domain-migrations' import { migrateJournalDomainPredecessor } from './database/journal-domain-migrations.test-fixture' @@ -26,9 +31,15 @@ afterEach(() => { function history(db = new Database(':memory:')) { const value = messageFixture(db) + // The author installs this body as the next numbered migration and calls the bounds write at each item writer. + migrateJournalDomainHistoryBounds(db) cleanup.push(() => db.close()) return { ...value, + seedItem: (item: AgentJournalRenderItem) => { + value.seedItem(item) + writeJournalDomainHistoryBounds(db, target, item.itemId, journalDomainHistoryMetadata(item)) + }, head: (sequence: number, retained = 0) => db .prepare('UPDATE journal_chat_state SET last_sequence=?, retained_sequence=?') @@ -149,6 +160,29 @@ it('never splits a same-sequence group, marking it oversized backward and paging ]) }) +it('excludes a retained body and submission by stored size before reading either, in both directions', () => { + const { db, seedItem, head, event, read } = history() + for (let sequence = 1; sequence <= 4; sequence++) { + seedItem(text(sequence, 'row')) + event(sequence, [`orca:${sequence}:0`]) + } + head(4) + // JSON5 satisfies the expression index but fails the read if this body were decoded. + db.prepare("UPDATE journal_items SET payload_json=? WHERE item_id='orca:2:0'").run( + `{"pad":"${'x'.repeat(10_000)}",}` + ) + db.exec(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence, + attempt_index,current,item_id,ordinal,submitted_at,fingerprint,dispatch_state,payload_json) + VALUES('chat','epoch','send',2,0,1,'orca:2:0',2,2,'f','accepted','not json')`) + const small = { form: measureHistoryItem, budgetBytes: 4096 } + const tail = read({ direction: 'tail' }, small) + expect(sequences(tail)).toEqual([3, 4]) + expect(tail.page).toMatchObject({ hasOlder: true, window: { nextCursor: { sequence: 3 } } }) + const forward = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 0 } }, small) + expect(sequences(forward)).toEqual([1]) + expect(forward.page).toMatchObject({ hasNewer: true, window: { nextCursor: { sequence: 1 } } }) +}) + it('catches up with the current revision, placement and submission, not the replayed metadata', async () => { const { db, commit, seedSend, read } = history() seedSend()