fix(native-chat): bound queued-image history admission

This commit is contained in:
Brennan Benson
2026-11-11 06:37:27 -08:00
parent 225c2b27d7
commit b04aa45b33
11 changed files with 230 additions and 40 deletions
@@ -25,6 +25,11 @@ import { readJournalDomainClientItems } from './journal-domain-item-presentation
import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor'
import type { JournalDomainEventChanges } from './journal-domain-schema'
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
import {
admitJournalDomainImageEvents,
halvedJournalDomainEventLimit,
type JournalDomainHistorySnapshotReader
} from './journal-domain-history-image-admission'
/** Select one ordered event window, then current rows for only its identities, in the caller's snapshot. */
export function readJournalDomainForwardHistory(
@@ -34,18 +39,24 @@ export function readJournalDomainForwardHistory(
limit: number,
reader: Pick<HistoryReader, 'form' | 'budgetBytes'>,
latestTurn: AgentJournalRenderItem | null,
readSnapshot: (ids: Set<string>, forms: Map<string, HistoryItemForm>) => AgentJournalSnapshot,
readSnapshot: JournalDomainHistorySnapshotReader,
reset: (
reason: Extract<AgentSessionHistoryResult, { ok: false }>['reset']
) => AgentSessionHistoryResult,
queuedImages?: QueuedImageHistoryPolicy
queuedImages?: QueuedImageHistoryPolicy,
useStoredClientForms = queuedImages === undefined
): AgentSessionHistoryResult {
// One bounded identity read serves the size admission and the builder's lookahead.
const links = queuedImages && new Map(queuedImages.links)
const since = cursor && readJournalDomainEvents(db, head, cursor, limit + 1, links)
const forms = new Map<string, HistoryItemForm>()
const image =
since?.ok && queuedImages && !useStoredClientForms
? admitJournalDomainImageEvents(db, head, since.rows.slice(0, limit), reader, readSnapshot)
: undefined
const admitted =
since?.ok && queuedImages === undefined
image?.limit ??
(since?.ok
? fittingJournalDomainEventCount(
db,
head,
@@ -53,8 +64,12 @@ export function readJournalDomainForwardHistory(
reader.budgetBytes,
forms
)
: limit
let materialized: AgentJournalSnapshot | undefined
: limit)
const count =
since?.ok && queuedImages && useStoredClientForms
? halvedJournalDomainEventLimit(Math.min(limit, since.rows.length), admitted)
: admitted
let materialized: AgentJournalSnapshot | undefined = image?.snapshot
return readAgentSessionForwardHistory(
{
readSince: (after, count = limit + 1) =>
@@ -72,7 +87,7 @@ export function readJournalDomainForwardHistory(
submissions: []
},
cursor,
admitted,
count,
reader,
(ids) => {
materialized ??= readSnapshot(ids, forms)
@@ -103,7 +118,14 @@ function readJournalDomainEvents(
cursor: AgentJournalCursor,
limit: number,
queueLinks?: Map<string, readonly string[]>
): JournalReadSince {
):
| Exclude<JournalReadSince, { ok: true }>
| {
ok: true
rows: JournalOutcomeReadRow[]
through: number
cursor: AgentJournalCursor
} {
const resume = resolveJournalResume(
{
epoch: head.cursor.epoch,
@@ -227,7 +249,9 @@ export function readJournalDomainHistorySnapshot(
ids: ReadonlySet<string>,
sourceIds: readonly string[] = [],
present?: (send: AgentJournalSubmission) => AgentJournalSubmission,
forms?: Map<string, HistoryItemForm>
forms?: Map<string, HistoryItemForm>,
states?: ReadonlyMap<string, unknown>,
ordinals?: Map<string, number>
): AgentJournalSnapshot {
const wanted = [...ids]
return {
@@ -243,7 +267,8 @@ export function readJournalDomainHistorySnapshot(
ORDER BY item.sequence,item.sequence_index,item.ordinal`)
.all(JSON.stringify(wanted), head.sessionId, head.epoch)
.map((row) => String(row.item_id)),
forms
forms,
states
),
...readJournalDomainSourceHeaders(
new JournalSqlScope(db, head.sessionId, head.epoch),
@@ -254,7 +279,7 @@ export function readJournalDomainHistorySnapshot(
readJournalDomainSourceBody(new JournalSqlScope(db, head.sessionId, head.epoch), header)
)
],
submissions: itemSubmissions(db, head, wanted, present)
submissions: itemSubmissions(db, head, wanted, present, ordinals)
}
}
@@ -16,6 +16,7 @@ import {
readJournalDomainSourceBody
} from './journal-domain-source-items'
import { readJournalDomainClientItems } from './journal-domain-item-presentation'
import { readJournalDomainStopStates } from './journal-domain-item-display'
import { itemSubmissions } from './journal-domain-history-after'
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
import type { JournalDomainHistoryHeader } from './journal-domain-history-window'
@@ -37,9 +38,19 @@ export function formJournalDomainHistoryGroups(
const ordinals = new Map<string, number>()
const byItem = new Map<string, AgentJournalSubmission[]>()
const ids = groups.flatMap((group) => group.map((header) => header.itemId))
const states = readJournalDomainStopStates(
db,
head,
ids.filter((id) => !forms.has(id))
)
const bodies = reader.presentSubmission
? undefined
: new Map(readJournalDomainClientItems(db, head, ids, forms).map((item) => [item.itemId, item]))
: new Map(
readJournalDomainClientItems(db, head, ids, forms, states).map((item) => [
item.itemId,
item
])
)
if (bodies) {
itemSubmissions(db, head, ids, undefined, ordinals, byItem)
}
@@ -61,7 +72,8 @@ export function formJournalDomainHistoryGroups(
db,
head,
group.map((header) => header.itemId),
forms
forms,
states
).map((item) => [item.itemId, item])
)
const formed = group.map((header) =>
@@ -0,0 +1,93 @@
import type Database from '../../sqlite/sync-database'
import type { AgentJournalSnapshot } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
import {
historyContentBytes,
submissionBytesByItemId,
type HistoryItemForm,
type HistoryReader
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
import type { JournalOutcomeReadRow } from './replay/journal-read-row'
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
import { readJournalDomainStopStates } from './journal-domain-item-display'
export type JournalDomainHistorySnapshotReader = (
ids: Set<string>,
forms: Map<string, HistoryItemForm>,
states?: ReadonlyMap<string, unknown>,
ordinals?: Map<string, number>
) => AgentJournalSnapshot
const eventIds = (row: JournalOutcomeReadRow) => [
...row.itemIds,
...row.submissionIds.map(agentJournalSubmissionKey)
]
/** Exact prefix admission stops body reads before applying the shared reader's same halving rule. */
export function admitJournalDomainImageEvents(
db: Database.Database,
head: JournalDomainHistoryHead,
rows: readonly JournalOutcomeReadRow[],
reader: Pick<HistoryReader, 'form' | 'budgetBytes'>,
readSnapshot: JournalDomainHistorySnapshotReader
): { limit: number; snapshot: AgentJournalSnapshot } {
const states = readJournalDomainStopStates(db, head, [...new Set(rows.flatMap(eventIds))])
const items = new Map<string, AgentJournalSnapshot['items'][number]>()
const submissions = new Map<string, AgentJournalSnapshot['submissions'][number]>()
const ordinals = new Map<string, number>()
const visited = new Set<string>()
let count = 0
let total = 0
for (const row of rows) {
const fresh = [...new Set(eventIds(row))].filter((id) => !visited.has(id))
const group = readSnapshot(new Set(fresh), new Map(), states, ordinals)
const live = new Map(group.items.map((item) => [item.itemId, item]))
const bytes = historyContentBytes(
[...live.values()].map(reader.form),
fresh.filter((id) => !live.has(id)),
submissionBytesByItemId(group.submissions)
)
if (count > 0 && total + bytes > reader.budgetBytes) {
break
}
for (const [id, item] of live) {
items.set(id, item)
}
for (const send of group.submissions) {
submissions.set(send.clientMessageId, send)
}
for (const id of fresh) {
visited.add(id)
}
count++
total += bytes
if (total > reader.budgetBytes) {
break
}
}
// Touched identities only accumulate, so the first non-fit determines every halved prefix.
const limit = halvedJournalDomainEventLimit(rows.length, count)
const kept = new Set(rows.slice(0, limit).flatMap(eventIds))
return {
limit: Math.max(1, limit),
snapshot: {
sessionId: head.sessionId,
cursor: head.cursor,
items: [...items.values()].filter((item) => kept.has(item.itemId)),
submissions: [...submissions.values()]
.filter((send) => kept.has(agentJournalSubmissionKey(send.clientMessageId)))
.sort(
(a, b) =>
a.submittedAt - b.submittedAt ||
ordinals.get(a.clientMessageId)! - ordinals.get(b.clientMessageId)!
)
}
}
}
export function halvedJournalDomainEventLimit(total: number, fitting: number): number {
while (total > 1 && total > fitting) {
total = Math.ceil(total / 2)
}
return Math.max(1, total)
}
@@ -24,6 +24,7 @@ import {
import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result'
import {
queuedImageSubmissionPresenter,
unchangedQueuedImageSubmission,
type QueuedImageHistoryPolicy
} from '../agent-session-wire/publication/history/agent-session-queued-image-reader'
import { JournalSqlScope } from './journal-sql-scope'
@@ -144,17 +145,22 @@ function readHistory(
resolveHistoryLimit(request.limit),
reader,
latestTurnItem(db, head),
(ids, forms) =>
(ids, forms, states, ordinals) =>
readJournalDomainHistorySnapshot(
db,
head,
ids,
queuedImageSourceIds(request),
reader.presentSubmission,
forms
forms,
states,
ordinals
),
reset,
request.reader?.queuedImages
request.reader?.queuedImages,
reader.form === CLIENT_HISTORY_READER.form &&
(reader.presentSubmission === undefined ||
reader.presentSubmission === unchangedQueuedImageSubmission)
)
}
const cursor = request.direction === 'before' ? request.cursor : undefined
@@ -289,14 +289,21 @@ it('windows retained image sources before reading their bodies and admits their
const forward = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 1 }, reader })
expect(sequences(forward)).toEqual([1, 2, 4, 5])
expect(forward.page).toMatchObject({ hasNewer: false, window: { nextCursor: { sequence: 5 } } })
const deferred = db
.prepare("SELECT payload_json FROM journal_items WHERE item_id='orca:5:0'")
.get()!.payload_json
db.exec(
"UPDATE journal_items SET wire_version=NULL,payload_json='excluded stale source' WHERE item_id='orca:5:0'"
)
const excluded = read({
direction: 'after',
cursor: { epoch: 'epoch', sequence: 1 },
limit: 2,
limit: 4,
reader: { ...reader, budgetBytes: 4096 }
})
expect(sequences(excluded)).toEqual([2])
expect(excluded.page).toMatchObject({ hasNewer: true, window: { nextCursor: { sequence: 2 } } })
db.prepare("UPDATE journal_items SET payload_json=? WHERE item_id='orca:5:0'").run(deferred)
const bounded = {
...reader,
form: 'bounded' as const,
@@ -440,6 +447,32 @@ it('catches up with the current revision, placement and submission, not the repl
expect(assertPlans().some((sql) => sql.includes('root=1'))).toBe(true)
})
it('orders queued-image submissions independently of touched-event order', () => {
const { db, seedItem, seedSend, head, event, read } = history()
seedSend()
seedItem({ ...text(8, 'early send'), itemId: 'orca:early' })
db.exec(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence,
attempt_index,current,item_id,ordinal,submitted_at,fingerprint,dispatch_state,payload_json)
SELECT session_id,epoch,'early',1,0,1,'orca:early',2,0,fingerprint,dispatch_state,
json_set(payload_json,'$.clientMessageId','early','$.submittedAt',0)
FROM journal_submission_attempts WHERE message_id='send' AND current=1`)
event(9, ['orca:send'])
event(10, ['orca:early'])
head(10, 8)
for (const queuedImages of [
{ ids: [], imageIds: [] },
{ ids: [['send', true] as const], imageIds: ['card'] }
]) {
const page = read({
direction: 'after',
cursor: { epoch: 'epoch', sequence: 8 },
reader: { form: 'client', queuedImages }
})
expect(page.page.submissions.map((send) => send.clientMessageId)).toEqual(['early', 'send'])
expect(page.page.window.nextCursor).toEqual({ epoch: 'epoch', sequence: 10 })
}
})
it('resets a cursor below the pruned boundary or across a gap, and resumes at the boundary', () => {
const { db, seedItem, head, event, read } = history()
for (let sequence = 1; sequence <= 3; sequence++) {
@@ -34,7 +34,8 @@ export function resolveJournalDomainDisplay(
export function readJournalDomainStopStates(
db: Database.Database,
target: JournalDomainItemTarget,
ids: readonly string[]
ids: readonly string[],
idsJson?: string
): Map<string, unknown> {
if (ids.length === 0) {
return new Map()
@@ -46,7 +47,7 @@ export function readJournalDomainStopStates(
CROSS JOIN journal_items item WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value
AND item.wire_version IS NOT ${AGENT_SESSION_ITEM_WIRE_VERSION} AND item.turn_item_id IS NOT NULL
AND item.original_stop_note_json IS NOT NULL`)
.all(JSON.stringify(ids), target.sessionId, target.epoch)
.all(idsJson ?? JSON.stringify(ids), target.sessionId, target.epoch)
.map((row) => String(row.turn_item_id))
)
]
@@ -82,12 +83,15 @@ export function readJournalDomainDisplayedRow(
row: SqliteRow
): { item: AgentJournalRenderItem; payload: string } {
const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const states = readJournalDomainStopStates(db, target, [String(row.item_id)])
const states =
row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? undefined
: readJournalDomainStopStates(db, target, [String(row.item_id)])
const item = resolveJournalDomainDisplay(
source,
row.wire_version,
row.wire_json,
journalDomainDisplayTurnState(source, states)
states && journalDomainDisplayTurnState(source, states)
)
return { item, payload: item === source ? String(row.payload_json) : JSON.stringify(item) }
}
@@ -35,12 +35,13 @@ export function readJournalDomainClientForms(
if (wanted.length === 0) {
return forms
}
states ??= readJournalDomainStopStates(db, target, wanted)
const wantedJson = JSON.stringify(wanted)
states ??= readJournalDomainStopStates(db, target, wanted, wantedJson)
for (const row of db
.prepare(`SELECT ${journalDomainClientFormColumns()} FROM json_each(?) wanted
CROSS JOIN journal_items item
WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0`)
.iterate(JSON.stringify(wanted), target.sessionId, target.epoch)) {
.iterate(wantedJson, target.sessionId, target.epoch)) {
forms.set(String(row.item_id), readJournalDomainClientForm(row, states))
}
return forms
@@ -67,9 +68,10 @@ export function readJournalDomainClientItems(
db: Database.Database,
target: JournalDomainItemTarget,
ids: readonly string[],
forms?: Map<string, HistoryItemForm>
forms?: Map<string, HistoryItemForm>,
states?: ReadonlyMap<string, unknown>
): AgentJournalRenderItem[] {
const selected = readJournalDomainClientForms(db, target, ids, forms)
const selected = readJournalDomainClientForms(db, target, ids, forms, states)
return ids.flatMap((id) => (selected.has(id) ? [selected.get(id)!.item] : []))
}
@@ -264,18 +264,20 @@ it('derives a stale Stop presentation from its owning turn before history admiss
expect(display()).toMatchObject({
item: { body: { kind: 'status', text: 'Cancellation requested.' } }
})
expect(
readJournalDomainHistoryPage(db, {
sessionId: 'chat',
direction: 'tail',
reader: { form: 'client', budgetBytes: 1024 }
}).page.items
).toContainEqual(
expect.objectContaining({
itemId: noteId,
body: { kind: 'status', text: 'Cancellation requested.' }
})
)
for (const queuedImages of [undefined, { ids: [], imageIds: [] }]) {
expect(
readJournalDomainHistoryPage(db, {
sessionId: 'chat',
direction: 'tail',
reader: { form: 'client', budgetBytes: 1024, queuedImages }
}).page.items
).toContainEqual(
expect.objectContaining({
itemId: noteId,
body: { kind: 'status', text: 'Cancellation requested.' }
})
)
}
const context = new JournalSqlStatusContext(new JournalSqlScope(db, 'chat', 'epoch'))
expect(context.displayed(noteId)?.body).toEqual({
kind: 'status',
@@ -301,7 +303,9 @@ it('derives a stale Stop presentation from its owning turn before history admiss
})
expect(item(noteId)?.body).toEqual(original)
// An ordinary owning-turn refresh repairs the stamp even when its displayed body is equal.
runJournalTransaction(db, () => refreshJournalDomainStopNotes(db, target, [turnId]))
expect(
runJournalTransaction(db, () => refreshJournalDomainStopNotes(db, target, [turnId]))
).toEqual([])
expect(db.prepare('SELECT wire_version FROM journal_items WHERE item_id=?').get(noteId)).toEqual({
wire_version: 1
})
@@ -92,7 +92,7 @@ export function refreshJournalDomainStopNotes(
...prefix,
next.itemId
)
if (!same || row.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION) {
if (!same) {
changed.push(next)
}
}
@@ -6,6 +6,7 @@ import { runningStructuredAgentSessionTurnItemBySequence } from '../../../shared
import type { JournalLatestStop } from './context-epochs/journal-stop-turn-end'
import { journalContextEpoch } from './context-epochs/journal-context-epoch'
import type { JournalSqlScope } from './journal-sql-scope'
import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract'
import type { JournalStatusItem, JournalStatusItemSelection } from './journal-status-items'
import { sqlJournalStatusItems } from './journal-sql-status-items'
import { sqlFailedStatusItems, sqlHasStatusRequestItem } from './journal-sql-status-request-items'
@@ -75,12 +76,15 @@ export class JournalSqlStatusContext {
if (row) {
this.scope.examined(row)
const source = this.scope.decode<AgentJournalRenderItem>(row.payload_json)
const states = readJournalDomainStopStates(this.scope.db, this.scope, [id])
const states =
row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? undefined
: readJournalDomainStopStates(this.scope.db, this.scope, [id])
const displayed = resolveJournalDomainDisplay(
source,
row.wire_version,
row.wire_json,
journalDomainDisplayTurnState(source, states)
states && journalDomainDisplayTurnState(source, states)
)
row.display_json = displayed === source ? row.payload_json : JSON.stringify(displayed)
}
@@ -154,7 +154,14 @@ export async function heldQueuedImageHistoryPolicy(
}
}
export const unchangedQueuedImageSubmission = (
send: AgentJournalSubmission
): AgentJournalSubmission => send
export function queuedImageSubmissionPresenter(policy: QueuedImageHistoryPolicy) {
if (policy.ids.length === 0 && policy.imageIds.length === 0) {
return unchangedQueuedImageSubmission
}
const ids = new Map(policy.ids)
const imageIds = new Set(policy.imageIds)
return (send: AgentJournalSubmission): AgentJournalSubmission =>