diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts index 3d5a1f1118e..4b589d4f638 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-after.ts @@ -25,6 +25,11 @@ import { readJournalDomainClientItems } from './journal-domain-item-presentation import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor' import type { JournalDomainEventChanges } from './journal-domain-schema' import type { JournalDomainHistoryHead } from './journal-domain-history-page' +import { + admitJournalDomainImageEvents, + halvedJournalDomainEventLimit, + type JournalDomainHistorySnapshotReader +} from './journal-domain-history-image-admission' /** Select one ordered event window, then current rows for only its identities, in the caller's snapshot. */ export function readJournalDomainForwardHistory( @@ -34,18 +39,24 @@ export function readJournalDomainForwardHistory( limit: number, reader: Pick, latestTurn: AgentJournalRenderItem | null, - readSnapshot: (ids: Set, forms: Map) => AgentJournalSnapshot, + readSnapshot: JournalDomainHistorySnapshotReader, reset: ( reason: Extract['reset'] ) => AgentSessionHistoryResult, - queuedImages?: QueuedImageHistoryPolicy + queuedImages?: QueuedImageHistoryPolicy, + useStoredClientForms = queuedImages === undefined ): AgentSessionHistoryResult { // One bounded identity read serves the size admission and the builder's lookahead. const links = queuedImages && new Map(queuedImages.links) const since = cursor && readJournalDomainEvents(db, head, cursor, limit + 1, links) const forms = new Map() + const image = + since?.ok && queuedImages && !useStoredClientForms + ? admitJournalDomainImageEvents(db, head, since.rows.slice(0, limit), reader, readSnapshot) + : undefined const admitted = - since?.ok && queuedImages === undefined + image?.limit ?? + (since?.ok ? fittingJournalDomainEventCount( db, head, @@ -53,8 +64,12 @@ export function readJournalDomainForwardHistory( reader.budgetBytes, forms ) - : limit - let materialized: AgentJournalSnapshot | undefined + : limit) + const count = + since?.ok && queuedImages && useStoredClientForms + ? halvedJournalDomainEventLimit(Math.min(limit, since.rows.length), admitted) + : admitted + let materialized: AgentJournalSnapshot | undefined = image?.snapshot return readAgentSessionForwardHistory( { readSince: (after, count = limit + 1) => @@ -72,7 +87,7 @@ export function readJournalDomainForwardHistory( submissions: [] }, cursor, - admitted, + count, reader, (ids) => { materialized ??= readSnapshot(ids, forms) @@ -103,7 +118,14 @@ function readJournalDomainEvents( cursor: AgentJournalCursor, limit: number, queueLinks?: Map -): JournalReadSince { +): + | Exclude + | { + ok: true + rows: JournalOutcomeReadRow[] + through: number + cursor: AgentJournalCursor + } { const resume = resolveJournalResume( { epoch: head.cursor.epoch, @@ -227,7 +249,9 @@ export function readJournalDomainHistorySnapshot( ids: ReadonlySet, sourceIds: readonly string[] = [], present?: (send: AgentJournalSubmission) => AgentJournalSubmission, - forms?: Map + forms?: Map, + states?: ReadonlyMap, + ordinals?: Map ): AgentJournalSnapshot { const wanted = [...ids] return { @@ -243,7 +267,8 @@ export function readJournalDomainHistorySnapshot( ORDER BY item.sequence,item.sequence_index,item.ordinal`) .all(JSON.stringify(wanted), head.sessionId, head.epoch) .map((row) => String(row.item_id)), - forms + forms, + states ), ...readJournalDomainSourceHeaders( new JournalSqlScope(db, head.sessionId, head.epoch), @@ -254,7 +279,7 @@ export function readJournalDomainHistorySnapshot( readJournalDomainSourceBody(new JournalSqlScope(db, head.sessionId, head.epoch), header) ) ], - submissions: itemSubmissions(db, head, wanted, present) + submissions: itemSubmissions(db, head, wanted, present, ordinals) } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-groups.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-groups.ts index 142e3227517..6ea594efc7e 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-groups.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-groups.ts @@ -16,6 +16,7 @@ import { readJournalDomainSourceBody } from './journal-domain-source-items' import { readJournalDomainClientItems } from './journal-domain-item-presentation' +import { readJournalDomainStopStates } from './journal-domain-item-display' import { itemSubmissions } from './journal-domain-history-after' import type { JournalDomainHistoryHead } from './journal-domain-history-page' import type { JournalDomainHistoryHeader } from './journal-domain-history-window' @@ -37,9 +38,19 @@ export function formJournalDomainHistoryGroups( const ordinals = new Map() const byItem = new Map() const ids = groups.flatMap((group) => group.map((header) => header.itemId)) + const states = readJournalDomainStopStates( + db, + head, + ids.filter((id) => !forms.has(id)) + ) const bodies = reader.presentSubmission ? undefined - : new Map(readJournalDomainClientItems(db, head, ids, forms).map((item) => [item.itemId, item])) + : new Map( + readJournalDomainClientItems(db, head, ids, forms, states).map((item) => [ + item.itemId, + item + ]) + ) if (bodies) { itemSubmissions(db, head, ids, undefined, ordinals, byItem) } @@ -61,7 +72,8 @@ export function formJournalDomainHistoryGroups( db, head, group.map((header) => header.itemId), - forms + forms, + states ).map((item) => [item.itemId, item]) ) const formed = group.map((header) => diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-image-admission.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-image-admission.ts new file mode 100644 index 00000000000..8686f0ef6f4 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-image-admission.ts @@ -0,0 +1,93 @@ +import type Database from '../../sqlite/sync-database' +import type { AgentJournalSnapshot } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' +import { + historyContentBytes, + submissionBytesByItemId, + type HistoryItemForm, + type HistoryReader +} from '../agent-session-wire/publication/history/agent-session-history-page-bounds' +import type { JournalOutcomeReadRow } from './replay/journal-read-row' +import type { JournalDomainHistoryHead } from './journal-domain-history-page' +import { readJournalDomainStopStates } from './journal-domain-item-display' + +export type JournalDomainHistorySnapshotReader = ( + ids: Set, + forms: Map, + states?: ReadonlyMap, + ordinals?: Map +) => AgentJournalSnapshot + +const eventIds = (row: JournalOutcomeReadRow) => [ + ...row.itemIds, + ...row.submissionIds.map(agentJournalSubmissionKey) +] + +/** Exact prefix admission stops body reads before applying the shared reader's same halving rule. */ +export function admitJournalDomainImageEvents( + db: Database.Database, + head: JournalDomainHistoryHead, + rows: readonly JournalOutcomeReadRow[], + reader: Pick, + readSnapshot: JournalDomainHistorySnapshotReader +): { limit: number; snapshot: AgentJournalSnapshot } { + const states = readJournalDomainStopStates(db, head, [...new Set(rows.flatMap(eventIds))]) + const items = new Map() + const submissions = new Map() + const ordinals = new Map() + const visited = new Set() + let count = 0 + let total = 0 + for (const row of rows) { + const fresh = [...new Set(eventIds(row))].filter((id) => !visited.has(id)) + const group = readSnapshot(new Set(fresh), new Map(), states, ordinals) + const live = new Map(group.items.map((item) => [item.itemId, item])) + const bytes = historyContentBytes( + [...live.values()].map(reader.form), + fresh.filter((id) => !live.has(id)), + submissionBytesByItemId(group.submissions) + ) + if (count > 0 && total + bytes > reader.budgetBytes) { + break + } + for (const [id, item] of live) { + items.set(id, item) + } + for (const send of group.submissions) { + submissions.set(send.clientMessageId, send) + } + for (const id of fresh) { + visited.add(id) + } + count++ + total += bytes + if (total > reader.budgetBytes) { + break + } + } + // Touched identities only accumulate, so the first non-fit determines every halved prefix. + const limit = halvedJournalDomainEventLimit(rows.length, count) + const kept = new Set(rows.slice(0, limit).flatMap(eventIds)) + return { + limit: Math.max(1, limit), + snapshot: { + sessionId: head.sessionId, + cursor: head.cursor, + items: [...items.values()].filter((item) => kept.has(item.itemId)), + submissions: [...submissions.values()] + .filter((send) => kept.has(agentJournalSubmissionKey(send.clientMessageId))) + .sort( + (a, b) => + a.submittedAt - b.submittedAt || + ordinals.get(a.clientMessageId)! - ordinals.get(b.clientMessageId)! + ) + } + } +} + +export function halvedJournalDomainEventLimit(total: number, fitting: number): number { + while (total > 1 && total > fitting) { + total = Math.ceil(total / 2) + } + return Math.max(1, total) +} diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts index c5e4cc7aaa2..fa6ee6319bc 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts @@ -24,6 +24,7 @@ import { import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result' import { queuedImageSubmissionPresenter, + unchangedQueuedImageSubmission, type QueuedImageHistoryPolicy } from '../agent-session-wire/publication/history/agent-session-queued-image-reader' import { JournalSqlScope } from './journal-sql-scope' @@ -144,17 +145,22 @@ function readHistory( resolveHistoryLimit(request.limit), reader, latestTurnItem(db, head), - (ids, forms) => + (ids, forms, states, ordinals) => readJournalDomainHistorySnapshot( db, head, ids, queuedImageSourceIds(request), reader.presentSubmission, - forms + forms, + states, + ordinals ), reset, - request.reader?.queuedImages + request.reader?.queuedImages, + reader.form === CLIENT_HISTORY_READER.form && + (reader.presentSubmission === undefined || + reader.presentSubmission === unchangedQueuedImageSubmission) ) } const cursor = request.direction === 'before' ? request.cursor : undefined diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts index cbc8a33efc3..bfefdf43a47 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history.test.ts @@ -289,14 +289,21 @@ it('windows retained image sources before reading their bodies and admits their const forward = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 1 }, reader }) expect(sequences(forward)).toEqual([1, 2, 4, 5]) expect(forward.page).toMatchObject({ hasNewer: false, window: { nextCursor: { sequence: 5 } } }) + const deferred = db + .prepare("SELECT payload_json FROM journal_items WHERE item_id='orca:5:0'") + .get()!.payload_json + db.exec( + "UPDATE journal_items SET wire_version=NULL,payload_json='excluded stale source' WHERE item_id='orca:5:0'" + ) const excluded = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 1 }, - limit: 2, + limit: 4, reader: { ...reader, budgetBytes: 4096 } }) expect(sequences(excluded)).toEqual([2]) expect(excluded.page).toMatchObject({ hasNewer: true, window: { nextCursor: { sequence: 2 } } }) + db.prepare("UPDATE journal_items SET payload_json=? WHERE item_id='orca:5:0'").run(deferred) const bounded = { ...reader, form: 'bounded' as const, @@ -440,6 +447,32 @@ it('catches up with the current revision, placement and submission, not the repl expect(assertPlans().some((sql) => sql.includes('root=1'))).toBe(true) }) +it('orders queued-image submissions independently of touched-event order', () => { + const { db, seedItem, seedSend, head, event, read } = history() + seedSend() + seedItem({ ...text(8, 'early send'), itemId: 'orca:early' }) + db.exec(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence, + attempt_index,current,item_id,ordinal,submitted_at,fingerprint,dispatch_state,payload_json) + SELECT session_id,epoch,'early',1,0,1,'orca:early',2,0,fingerprint,dispatch_state, + json_set(payload_json,'$.clientMessageId','early','$.submittedAt',0) + FROM journal_submission_attempts WHERE message_id='send' AND current=1`) + event(9, ['orca:send']) + event(10, ['orca:early']) + head(10, 8) + for (const queuedImages of [ + { ids: [], imageIds: [] }, + { ids: [['send', true] as const], imageIds: ['card'] } + ]) { + const page = read({ + direction: 'after', + cursor: { epoch: 'epoch', sequence: 8 }, + reader: { form: 'client', queuedImages } + }) + expect(page.page.submissions.map((send) => send.clientMessageId)).toEqual(['early', 'send']) + expect(page.page.window.nextCursor).toEqual({ epoch: 'epoch', sequence: 10 }) + } +}) + it('resets a cursor below the pruned boundary or across a gap, and resumes at the boundary', () => { const { db, seedItem, head, event, read } = history() for (let sequence = 1; sequence <= 3; sequence++) { diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts index 2046be9f8c2..83f74e49147 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts @@ -34,7 +34,8 @@ export function resolveJournalDomainDisplay( export function readJournalDomainStopStates( db: Database.Database, target: JournalDomainItemTarget, - ids: readonly string[] + ids: readonly string[], + idsJson?: string ): Map { if (ids.length === 0) { return new Map() @@ -46,7 +47,7 @@ export function readJournalDomainStopStates( CROSS JOIN journal_items item WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.wire_version IS NOT ${AGENT_SESSION_ITEM_WIRE_VERSION} AND item.turn_item_id IS NOT NULL AND item.original_stop_note_json IS NOT NULL`) - .all(JSON.stringify(ids), target.sessionId, target.epoch) + .all(idsJson ?? JSON.stringify(ids), target.sessionId, target.epoch) .map((row) => String(row.turn_item_id)) ) ] @@ -82,12 +83,15 @@ export function readJournalDomainDisplayedRow( row: SqliteRow ): { item: AgentJournalRenderItem; payload: string } { const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) - const states = readJournalDomainStopStates(db, target, [String(row.item_id)]) + const states = + row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION + ? undefined + : readJournalDomainStopStates(db, target, [String(row.item_id)]) const item = resolveJournalDomainDisplay( source, row.wire_version, row.wire_json, - journalDomainDisplayTurnState(source, states) + states && journalDomainDisplayTurnState(source, states) ) return { item, payload: item === source ? String(row.payload_json) : JSON.stringify(item) } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts index a471b8a6318..7aa757eed8f 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts @@ -35,12 +35,13 @@ export function readJournalDomainClientForms( if (wanted.length === 0) { return forms } - states ??= readJournalDomainStopStates(db, target, wanted) + const wantedJson = JSON.stringify(wanted) + states ??= readJournalDomainStopStates(db, target, wanted, wantedJson) for (const row of db .prepare(`SELECT ${journalDomainClientFormColumns()} FROM json_each(?) wanted CROSS JOIN journal_items item WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0`) - .iterate(JSON.stringify(wanted), target.sessionId, target.epoch)) { + .iterate(wantedJson, target.sessionId, target.epoch)) { forms.set(String(row.item_id), readJournalDomainClientForm(row, states)) } return forms @@ -67,9 +68,10 @@ export function readJournalDomainClientItems( db: Database.Database, target: JournalDomainItemTarget, ids: readonly string[], - forms?: Map + forms?: Map, + states?: ReadonlyMap ): AgentJournalRenderItem[] { - const selected = readJournalDomainClientForms(db, target, ids, forms) + const selected = readJournalDomainClientForms(db, target, ids, forms, states) return ids.flatMap((id) => (selected.has(id) ? [selected.get(id)!.item] : [])) } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.test.ts index c79b87a3d85..b855d2a52b5 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.test.ts @@ -264,18 +264,20 @@ it('derives a stale Stop presentation from its owning turn before history admiss expect(display()).toMatchObject({ item: { body: { kind: 'status', text: 'Cancellation requested.' } } }) - expect( - readJournalDomainHistoryPage(db, { - sessionId: 'chat', - direction: 'tail', - reader: { form: 'client', budgetBytes: 1024 } - }).page.items - ).toContainEqual( - expect.objectContaining({ - itemId: noteId, - body: { kind: 'status', text: 'Cancellation requested.' } - }) - ) + for (const queuedImages of [undefined, { ids: [], imageIds: [] }]) { + expect( + readJournalDomainHistoryPage(db, { + sessionId: 'chat', + direction: 'tail', + reader: { form: 'client', budgetBytes: 1024, queuedImages } + }).page.items + ).toContainEqual( + expect.objectContaining({ + itemId: noteId, + body: { kind: 'status', text: 'Cancellation requested.' } + }) + ) + } const context = new JournalSqlStatusContext(new JournalSqlScope(db, 'chat', 'epoch')) expect(context.displayed(noteId)?.body).toEqual({ kind: 'status', @@ -301,7 +303,9 @@ it('derives a stale Stop presentation from its owning turn before history admiss }) expect(item(noteId)?.body).toEqual(original) // An ordinary owning-turn refresh repairs the stamp even when its displayed body is equal. - runJournalTransaction(db, () => refreshJournalDomainStopNotes(db, target, [turnId])) + expect( + runJournalTransaction(db, () => refreshJournalDomainStopNotes(db, target, [turnId])) + ).toEqual([]) expect(db.prepare('SELECT wire_version FROM journal_items WHERE item_id=?').get(noteId)).toEqual({ wire_version: 1 }) diff --git a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts index 6b14200943d..733fdaa9a7f 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts @@ -92,7 +92,7 @@ export function refreshJournalDomainStopNotes( ...prefix, next.itemId ) - if (!same || row.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION) { + if (!same) { changed.push(next) } } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-status-context.ts b/src/main/native-chat/agent-session-journal/journal-sql-status-context.ts index 83e7f6a215a..f6f9cf05186 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-status-context.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-status-context.ts @@ -6,6 +6,7 @@ import { runningStructuredAgentSessionTurnItemBySequence } from '../../../shared import type { JournalLatestStop } from './context-epochs/journal-stop-turn-end' import { journalContextEpoch } from './context-epochs/journal-context-epoch' import type { JournalSqlScope } from './journal-sql-scope' +import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract' import type { JournalStatusItem, JournalStatusItemSelection } from './journal-status-items' import { sqlJournalStatusItems } from './journal-sql-status-items' import { sqlFailedStatusItems, sqlHasStatusRequestItem } from './journal-sql-status-request-items' @@ -75,12 +76,15 @@ export class JournalSqlStatusContext { if (row) { this.scope.examined(row) const source = this.scope.decode(row.payload_json) - const states = readJournalDomainStopStates(this.scope.db, this.scope, [id]) + const states = + row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION + ? undefined + : readJournalDomainStopStates(this.scope.db, this.scope, [id]) const displayed = resolveJournalDomainDisplay( source, row.wire_version, row.wire_json, - journalDomainDisplayTurnState(source, states) + states && journalDomainDisplayTurnState(source, states) ) row.display_json = displayed === source ? row.payload_json : JSON.stringify(displayed) } diff --git a/src/main/native-chat/agent-session-wire/publication/history/agent-session-queued-image-reader.ts b/src/main/native-chat/agent-session-wire/publication/history/agent-session-queued-image-reader.ts index 4bbe2670bd8..01321de75d7 100644 --- a/src/main/native-chat/agent-session-wire/publication/history/agent-session-queued-image-reader.ts +++ b/src/main/native-chat/agent-session-wire/publication/history/agent-session-queued-image-reader.ts @@ -154,7 +154,14 @@ export async function heldQueuedImageHistoryPolicy( } } +export const unchangedQueuedImageSubmission = ( + send: AgentJournalSubmission +): AgentJournalSubmission => send + export function queuedImageSubmissionPresenter(policy: QueuedImageHistoryPolicy) { + if (policy.ids.length === 0 && policy.imageIds.length === 0) { + return unchangedQueuedImageSubmission + } const ids = new Map(policy.ids) const imageIds = new Set(policy.imageIds) return (send: AgentJournalSubmission): AgentJournalSubmission =>