mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
fix: select domain history by stored size before reading bodies
Backward, tail and hydrate pages now take each header's sent size and current submission size from the bounded header query, admit whole sequence groups by those sizes, and only then load the admitted bodies and submissions. An oversized lone group becomes placeholders without decoding its bodies. Forward pages pick the longest event prefix whose touched rows fit by the same sizes before the existing replay builder reads current rows. The roster finds each unnamed subagent's earliest visible naming row through one indexed lookup per id instead of scanning every naming row. The sent-size column and the subagent-name relation are a new migration body for the central migration list; item writers call the bounds write after each upsert.
This commit is contained in:
@@ -8,7 +8,9 @@ import type {
|
||||
import { readAgentSessionForwardHistory } from '../agent-session-wire/publication/history/agent-session-history-forward-page'
|
||||
import type { HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import type { JournalReadSince } from './journal-store-contracts'
|
||||
import type { JournalOutcomeReadRow } from './replay/journal-read-row'
|
||||
import type { JournalOutcomeReadRow, JournalReadRow } from './replay/journal-read-row'
|
||||
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
|
||||
import { JOURNAL_DOMAIN_HISTORY_ITEM_BYTES } from './journal-domain-history-window'
|
||||
import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor'
|
||||
import type { JournalDomainEventChanges } from './journal-domain-schema'
|
||||
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
|
||||
@@ -27,9 +29,17 @@ export function readJournalDomainForwardHistory(
|
||||
reason: Extract<AgentSessionHistoryResult, { ok: false }>['reset']
|
||||
) => AgentSessionHistoryResult
|
||||
): AgentSessionHistoryResult {
|
||||
// One bounded identity read serves the size admission and the builder's lookahead.
|
||||
const since = cursor && readJournalDomainEvents(db, head, cursor, limit + 1)
|
||||
const admitted = since?.ok
|
||||
? fittingEventCount(db, head, since.rows.slice(0, limit), reader.budgetBytes)
|
||||
: limit
|
||||
return readAgentSessionForwardHistory(
|
||||
{
|
||||
readSince: (after, count) => readJournalDomainEvents(db, head, after, count),
|
||||
readSince: (after, count = limit + 1) =>
|
||||
since?.ok
|
||||
? firstEvents(since, count)
|
||||
: (since ?? readJournalDomainEvents(db, head, after, count)),
|
||||
canonicalItemId: (itemId) => itemId,
|
||||
hasSubmission: () => false,
|
||||
stopNoteIds: (ids) => journalDomainDependentStopNotes(db, head, ids),
|
||||
@@ -42,18 +52,65 @@ export function readJournalDomainForwardHistory(
|
||||
submissions: []
|
||||
},
|
||||
cursor,
|
||||
limit,
|
||||
admitted,
|
||||
reader,
|
||||
readSnapshot
|
||||
)
|
||||
}
|
||||
|
||||
/** The longest event prefix whose touched rows' stored sizes fit, so excluded bodies are never read. */
|
||||
function fittingEventCount(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
rows: readonly JournalReadRow[],
|
||||
budgetBytes: number
|
||||
): number {
|
||||
const touched = rows.map((row) =>
|
||||
row.kind === 'outcome'
|
||||
? [...row.itemIds, ...row.submissionIds.map(agentJournalSubmissionKey)]
|
||||
: []
|
||||
)
|
||||
const sizes = new Map(
|
||||
db
|
||||
.prepare(`SELECT item.item_id, ${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes FROM json_each(?) wanted
|
||||
CROSS JOIN journal_items item
|
||||
WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0`)
|
||||
.all(JSON.stringify([...new Set(touched.flat())]), head.sessionId, head.epoch)
|
||||
.map((row) => [String(row.item_id), Number(row.bytes)])
|
||||
)
|
||||
const counted = new Set<string>()
|
||||
let total = 0
|
||||
let count = 0
|
||||
for (const ids of touched) {
|
||||
for (const id of ids) {
|
||||
if (!counted.has(id)) {
|
||||
counted.add(id)
|
||||
total += sizes.get(id) ?? 0
|
||||
}
|
||||
}
|
||||
if (count > 0 && total > budgetBytes) {
|
||||
break
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
return Math.max(count, 1)
|
||||
}
|
||||
|
||||
function firstEvents(
|
||||
since: Extract<JournalReadSince, { ok: true }>,
|
||||
count: number
|
||||
): JournalReadSince {
|
||||
return since.rows.length <= count
|
||||
? since
|
||||
: { ...since, rows: since.rows.slice(0, count), through: since.rows[count - 1]!.seq }
|
||||
}
|
||||
|
||||
/** Retained events start after the pruned-through cursor; a missing sequence resets, never pages partially. */
|
||||
function readJournalDomainEvents(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
cursor: AgentJournalCursor,
|
||||
limit = -1
|
||||
limit: number
|
||||
): JournalReadSince {
|
||||
const resume = resolveJournalResume(
|
||||
{
|
||||
@@ -70,7 +127,7 @@ function readJournalDomainEvents(
|
||||
.prepare(`SELECT sequence, changes_json FROM journal_events
|
||||
WHERE session_id=? AND epoch=? AND sequence>? ORDER BY sequence LIMIT ?`)
|
||||
.all(head.sessionId, head.epoch, resume.afterSequence, limit)
|
||||
const complete = limit < 0 || stored.length < limit
|
||||
const complete = stored.length < limit
|
||||
const sequences = stored.map((row) => Number(row.sequence))
|
||||
if (
|
||||
findSequenceGap(sequences, resume.afterSequence + 1) ||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { SqliteBindings } from '../../sqlite/sqlite-statement'
|
||||
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import {
|
||||
journalDomainHistoryMetadata,
|
||||
type JournalDomainHistoryMetadata
|
||||
} from './journal-domain-history-metadata'
|
||||
|
||||
type HistoryTarget = { sessionId: string; epoch: string }
|
||||
|
||||
/** Next numbered migration body: sent sizes and subagent names history selects before any body. */
|
||||
export function migrateJournalDomainHistoryBounds(db: Database.Database): void {
|
||||
db.exec(`
|
||||
ALTER TABLE journal_items ADD COLUMN wire_bytes INTEGER;
|
||||
CREATE TABLE journal_item_subagent_names (
|
||||
session_id TEXT NOT NULL,
|
||||
epoch TEXT NOT NULL,
|
||||
subagent_id TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, epoch, subagent_id, item_id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX journal_item_subagent_names_item ON journal_item_subagent_names(session_id, epoch, item_id);
|
||||
DROP INDEX journal_items_named_subagents;
|
||||
`)
|
||||
const page = db.prepare(`SELECT session_id, epoch, item_id, payload_json FROM journal_items
|
||||
WHERE (session_id, epoch, item_id) > (?, ?, ?) ORDER BY session_id, epoch, item_id LIMIT 500`)
|
||||
let after: SqliteBindings = ['', '', '']
|
||||
for (let rows = page.all(...after); rows.length > 0; rows = page.all(...after)) {
|
||||
for (const row of rows) {
|
||||
const payload = String(row.payload_json)
|
||||
const item: AgentJournalRenderItem = JSON.parse(payload)
|
||||
writeJournalDomainHistoryBounds(
|
||||
db,
|
||||
{ sessionId: String(row.session_id), epoch: String(row.epoch) },
|
||||
String(row.item_id),
|
||||
journalDomainHistoryMetadata(item, payload)
|
||||
)
|
||||
}
|
||||
const last = rows.at(-1)!
|
||||
after = [last.session_id, last.epoch, last.item_id]
|
||||
}
|
||||
}
|
||||
|
||||
/** Every journal_items upsert calls this after writing the row, in the same transaction. */
|
||||
export function writeJournalDomainHistoryBounds(
|
||||
db: Database.Database,
|
||||
target: HistoryTarget,
|
||||
itemId: string,
|
||||
metadata: Pick<JournalDomainHistoryMetadata, 'wireBytes' | 'namedSubagentIds'>
|
||||
): void {
|
||||
const key = [target.sessionId, target.epoch, itemId] as const
|
||||
db.prepare(
|
||||
'UPDATE journal_items SET wire_bytes=? WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(metadata.wireBytes, ...key)
|
||||
db.prepare(
|
||||
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(...key)
|
||||
const insert =
|
||||
db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(session_id,epoch,subagent_id,item_id)
|
||||
VALUES(?,?,?,?)`)
|
||||
const named: string[] = JSON.parse(metadata.namedSubagentIds ?? '[]')
|
||||
for (const subagentId of named) {
|
||||
insert.run(target.sessionId, target.epoch, subagentId, itemId)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,11 @@ import {
|
||||
subagentFoldKey,
|
||||
subagentItemBlock
|
||||
} from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import { isSubagentRow } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import {
|
||||
HISTORY_PAGE_CONTENT_BUDGET_BYTES,
|
||||
isSubagentRow
|
||||
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import { agentJournalItemForWire } from '../agent-session-wire/publication/history/agent-session-item-wire-form'
|
||||
|
||||
/** Facts history selects before decoding a body; written from each current item revision. */
|
||||
export type JournalDomainHistoryMetadata = {
|
||||
@@ -16,10 +20,14 @@ export type JournalDomainHistoryMetadata = {
|
||||
subagentGroupKey: string | null
|
||||
namedSubagentIds: string | null
|
||||
turnMarker: number
|
||||
/** Client-form bytes when they differ from the stored payload; null reads the stored length. */
|
||||
wireBytes: number | null
|
||||
}
|
||||
|
||||
/** Pass the stored payload text when the caller already serialized it. */
|
||||
export function journalDomainHistoryMetadata(
|
||||
item: AgentJournalRenderItem
|
||||
item: AgentJournalRenderItem,
|
||||
payloadJson?: string
|
||||
): JournalDomainHistoryMetadata {
|
||||
const root = isRootAgentJournalItem(item)
|
||||
const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null
|
||||
@@ -34,6 +42,22 @@ export function journalDomainHistoryMetadata(
|
||||
namedSubagentIds: named.length > 0 ? JSON.stringify(named) : null,
|
||||
turnMarker: Number(
|
||||
readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body)
|
||||
)
|
||||
),
|
||||
wireBytes: wireBytes(item, payloadJson)
|
||||
}
|
||||
}
|
||||
|
||||
// The client form cuts tool/diff heads and whole items over the page budget; skipping the
|
||||
// form elsewhere only overstates a size, which shortens a page but never drops content.
|
||||
function wireBytes(item: AgentJournalRenderItem, payloadJson: string | undefined): number | null {
|
||||
if (
|
||||
item.body.kind !== 'tool-call' &&
|
||||
item.body.kind !== 'diff' &&
|
||||
Buffer.byteLength(payloadJson ?? JSON.stringify(item), 'utf8') <=
|
||||
HISTORY_PAGE_CONTENT_BUDGET_BYTES
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const sent = agentJournalItemForWire(item)
|
||||
return sent.item === item ? null : sent.bytes
|
||||
}
|
||||
|
||||
@@ -11,8 +11,6 @@ import type {
|
||||
AgentJournalSubmission
|
||||
} from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
|
||||
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
|
||||
import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import {
|
||||
historyContentBytes,
|
||||
oversizedHistoryItem,
|
||||
@@ -21,14 +19,15 @@ import {
|
||||
type HistoryReader
|
||||
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result'
|
||||
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
|
||||
import { CLIENT_HISTORY_READER } from '../agent-session-wire/publication/history/agent-session-item-wire-form'
|
||||
import { journalContextEpoch } from './context-epochs/journal-context-cursor'
|
||||
import { readJournalDomainForwardHistory } from './journal-domain-history-after'
|
||||
import { readJournalDomainHistoryRoster } from './journal-domain-history-roster'
|
||||
import { projectJournalDomainStopNote } from './journal-domain-stop-notes'
|
||||
import {
|
||||
journalDomainSequenceGroups,
|
||||
selectJournalDomainHistoryWindow
|
||||
admitJournalDomainHistoryGroups,
|
||||
selectJournalDomainHistoryWindow,
|
||||
type JournalDomainHistoryHeader
|
||||
} from './journal-domain-history-window'
|
||||
|
||||
export type JournalDomainHistoryRequest = Omit<AgentSessionHistoryRequest, 'direction'> & {
|
||||
@@ -139,40 +138,18 @@ function historyPage(
|
||||
request.direction === 'hydrate'
|
||||
? AGENT_SESSION_HISTORY_MAX_LIMIT
|
||||
: resolveHistoryLimit(request.limit)
|
||||
const window = selectJournalDomainHistoryWindow(db, where, prefix, end, limit)
|
||||
const admitted = admitJournalDomainHistoryGroups(
|
||||
selectJournalDomainHistoryWindow(db, where, prefix, end, limit),
|
||||
reader.budgetBytes
|
||||
)
|
||||
const submissions = itemSubmissions(
|
||||
db,
|
||||
head,
|
||||
window.map((header) => header.itemId)
|
||||
admitted.groups.flat().map((header) => header.itemId)
|
||||
)
|
||||
const sizes = submissionBytesByItemId(submissions)
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const kept: AgentJournalRenderItem[][] = []
|
||||
let total = 0
|
||||
for (const group of journalDomainSequenceGroups(window)) {
|
||||
const formed = group.map((header) =>
|
||||
reader.form(
|
||||
projectJournalDomainStopNote(
|
||||
db,
|
||||
head,
|
||||
JSON.parse(String(body.get(...prefix, header.itemId)!.payload_json))
|
||||
)
|
||||
)
|
||||
)
|
||||
const bytes = historyContentBytes(formed, [], sizes)
|
||||
if (kept.length > 0 && total + bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
kept.push(
|
||||
formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item))
|
||||
)
|
||||
total += bytes
|
||||
if (bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
}
|
||||
const kept = admitted.oversized
|
||||
? [admitted.groups[0]!.map((header) => oversizedHistoryItem(itemEnvelope(db, prefix, header)))]
|
||||
: formHistoryGroups(db, head, admitted.groups, submissions, reader)
|
||||
const items = kept.toReversed().flat()
|
||||
const oldest = items[0]?.sequence ?? end
|
||||
const turn = latestTurnItem(db, head)
|
||||
@@ -196,10 +173,63 @@ function historyPage(
|
||||
fallbackCursor: cursor ?? { epoch: head.cursor.epoch, sequence: 0 },
|
||||
nextCursor: items[0] ? { epoch: head.cursor.epoch, sequence: items[0].sequence } : undefined,
|
||||
fence: request.fence,
|
||||
subagentRoster: historyRoster(db, prefix, items)
|
||||
subagentRoster: readJournalDomainHistoryRoster(db, prefix, items)
|
||||
})
|
||||
}
|
||||
|
||||
/** The existing byte rule, authoritative over admitted groups only. */
|
||||
function formHistoryGroups(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
groups: readonly JournalDomainHistoryHeader[][],
|
||||
submissions: readonly AgentJournalSubmission[],
|
||||
reader: HistoryReaderForm
|
||||
): AgentJournalRenderItem[][] {
|
||||
const sizes = submissionBytesByItemId(submissions)
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const kept: AgentJournalRenderItem[][] = []
|
||||
let total = 0
|
||||
for (const group of groups) {
|
||||
const formed = group.map((header) =>
|
||||
reader.form(
|
||||
projectJournalDomainStopNote(
|
||||
db,
|
||||
head,
|
||||
JSON.parse(String(body.get(head.sessionId, head.epoch, header.itemId)!.payload_json))
|
||||
)
|
||||
)
|
||||
)
|
||||
const bytes = historyContentBytes(formed, [], sizes)
|
||||
if (kept.length > 0 && total + bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
kept.push(
|
||||
formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item))
|
||||
)
|
||||
total += bytes
|
||||
if (bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
/** An oversized group's placeholders keep each row's identity and placement, never its body. */
|
||||
function itemEnvelope(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
header: JournalDomainHistoryHeader
|
||||
): AgentJournalRenderItem {
|
||||
const row = db
|
||||
.prepare(`SELECT json_remove(payload_json, '$.body') AS envelope FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND item_id=?`)
|
||||
.get(...prefix, header.itemId)!
|
||||
// The placeholder replaces the omitted body.
|
||||
return JSON.parse(String(row.envelope))
|
||||
}
|
||||
|
||||
function latestTurnItem(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead
|
||||
@@ -245,47 +275,3 @@ function currentRows(
|
||||
submissions: itemSubmissions(db, head, wanted)
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads only root rows naming a subagent the page shows but does not name. */
|
||||
function historyRoster(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
page: readonly AgentJournalRenderItem[]
|
||||
) {
|
||||
const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null))
|
||||
for (const item of page) {
|
||||
if (item.agentId === undefined && item.body.kind === 'message') {
|
||||
for (const entry of item.body.blocks.flatMap(namedSubagents)) {
|
||||
missing.delete(entry.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (missing.size === 0) {
|
||||
return undefined
|
||||
}
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const named: AgentJournalRenderItem[] = []
|
||||
for (const row of db
|
||||
.prepare(`SELECT item_id, named_subagent_ids FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL
|
||||
ORDER BY sequence, sequence_index, ordinal`)
|
||||
.iterate(...prefix)) {
|
||||
const ids: string[] = JSON.parse(String(row.named_subagent_ids))
|
||||
if (!ids.some((id) => missing.has(id))) {
|
||||
continue
|
||||
}
|
||||
for (const id of ids) {
|
||||
missing.delete(id)
|
||||
}
|
||||
named.push(JSON.parse(String(body.get(...prefix, String(row.item_id))!.payload_json)))
|
||||
if (missing.size === 0 || named.length >= ROSTER_ROWS) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return offPageSubagentRoster(named, page)
|
||||
}
|
||||
|
||||
// The roster keeps at most 64 entries; each selected row adds at least one.
|
||||
const ROSTER_ROWS = 64
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
|
||||
import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
|
||||
|
||||
/** One indexed lookup per unnamed page subagent finds its earliest visible naming row. */
|
||||
export function readJournalDomainHistoryRoster(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
page: readonly AgentJournalRenderItem[]
|
||||
) {
|
||||
const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null))
|
||||
for (const item of page) {
|
||||
if (item.agentId === undefined && item.body.kind === 'message') {
|
||||
for (const entry of item.body.blocks.flatMap(namedSubagents)) {
|
||||
missing.delete(entry.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (missing.size === 0) {
|
||||
return undefined
|
||||
}
|
||||
const earliest = db.prepare(`SELECT item.item_id, item.sequence, item.sequence_index, item.ordinal
|
||||
FROM journal_item_subagent_names name CROSS JOIN journal_items item
|
||||
ON item.session_id=name.session_id AND item.epoch=name.epoch AND item.item_id=name.item_id
|
||||
WHERE name.session_id=? AND name.epoch=? AND name.subagent_id=? AND item.hidden=0 AND item.root=1
|
||||
ORDER BY item.sequence, item.sequence_index, item.ordinal LIMIT 1`)
|
||||
const naming = new Map<string, [number, number, number]>()
|
||||
for (const id of missing) {
|
||||
const row = earliest.get(...prefix, id)
|
||||
if (row) {
|
||||
naming.set(String(row.item_id), [
|
||||
Number(row.sequence),
|
||||
Number(row.sequence_index),
|
||||
Number(row.ordinal)
|
||||
])
|
||||
}
|
||||
}
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const named = [...naming]
|
||||
.sort(([, a], [, b]) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2])
|
||||
.slice(0, ROSTER_ROWS)
|
||||
.map(([itemId]) => JSON.parse(String(body.get(...prefix, itemId)!.payload_json)))
|
||||
return offPageSubagentRoster(named, page)
|
||||
}
|
||||
|
||||
// The roster keeps at most 64 entries; each selected row adds at least one.
|
||||
const ROSTER_ROWS = 64
|
||||
@@ -7,8 +7,16 @@ export type JournalDomainHistoryHeader = {
|
||||
sequence: number
|
||||
own: boolean
|
||||
subagentGroupKey: string | null
|
||||
/** Sent item plus current submission bytes, read without loading either body. */
|
||||
bytes: number
|
||||
}
|
||||
|
||||
/** Requires the item alias; octet_length reads a stored length, never the payload. */
|
||||
export const JOURNAL_DOMAIN_HISTORY_ITEM_BYTES = `COALESCE(item.wire_bytes, octet_length(item.payload_json))
|
||||
+ COALESCE((SELECT octet_length(attempt.payload_json) FROM journal_submission_attempts attempt
|
||||
WHERE attempt.session_id=item.session_id AND attempt.epoch=item.epoch
|
||||
AND attempt.item_id=item.item_id AND attempt.current=1), 0)`
|
||||
|
||||
/** Chooses the page's candidate identities before any body or submission is decoded. */
|
||||
export function selectJournalDomainHistoryWindow(
|
||||
db: Database.Database,
|
||||
@@ -29,11 +37,14 @@ function selectJournalDomainHistoryHeaders(
|
||||
end: number,
|
||||
limit: number
|
||||
): JournalDomainHistoryHeader[] {
|
||||
const columns = 'item_id, sequence, root, subagent_row, subagent_group_key'
|
||||
const columns = `item_id, sequence, root, subagent_row, subagent_group_key,
|
||||
${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes`
|
||||
const order = 'ORDER BY sequence DESC, sequence_index DESC, ordinal DESC'
|
||||
const cap = 2 * limit
|
||||
const rows = db
|
||||
.prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence<? ${order} LIMIT ?`)
|
||||
.prepare(
|
||||
`SELECT ${columns} FROM journal_items item WHERE ${where} AND sequence<? ${order} LIMIT ?`
|
||||
)
|
||||
.all(...prefix, end, cap)
|
||||
if (rows.length === cap) {
|
||||
const boundary = rows.at(-1)!.sequence
|
||||
@@ -41,7 +52,7 @@ function selectJournalDomainHistoryHeaders(
|
||||
// A source lifecycle row creates at most 200 positions; ordinary domain writes create one.
|
||||
const group = db
|
||||
.prepare(
|
||||
`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence=? ${order} LIMIT ?`
|
||||
`SELECT ${columns} FROM journal_items item WHERE ${where} AND sequence=? ${order} LIMIT ?`
|
||||
)
|
||||
.all(...prefix, boundary, MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS + 1)
|
||||
if (group.length > MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS) {
|
||||
@@ -53,7 +64,8 @@ function selectJournalDomainHistoryHeaders(
|
||||
itemId: String(row.item_id),
|
||||
sequence: Number(row.sequence),
|
||||
own: row.root === 1 && row.subagent_row === 0,
|
||||
subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key)
|
||||
subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key),
|
||||
bytes: Number(row.bytes)
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -93,6 +105,27 @@ function historyWindow(
|
||||
return [...headers]
|
||||
}
|
||||
|
||||
/** Newest whole groups whose stored sizes fit; a lone first group over budget is oversized. */
|
||||
export function admitJournalDomainHistoryGroups(
|
||||
window: readonly JournalDomainHistoryHeader[],
|
||||
budgetBytes: number
|
||||
): { groups: JournalDomainHistoryHeader[][]; oversized: boolean } {
|
||||
const groups: JournalDomainHistoryHeader[][] = []
|
||||
let total = 0
|
||||
for (const group of journalDomainSequenceGroups(window)) {
|
||||
const bytes = group.reduce((sum, header) => sum + header.bytes, 0)
|
||||
if (groups.length > 0 && total + bytes > budgetBytes) {
|
||||
break
|
||||
}
|
||||
groups.push(group)
|
||||
total += bytes
|
||||
if (bytes > budgetBytes) {
|
||||
return { groups, oversized: true }
|
||||
}
|
||||
}
|
||||
return { groups, oversized: false }
|
||||
}
|
||||
|
||||
/** Newest group first; each group keeps render order. */
|
||||
export function* journalDomainSequenceGroups<T extends { sequence: number }>(
|
||||
rows: readonly T[]
|
||||
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
type JournalDomainHistoryRequest
|
||||
} from './journal-domain-history-page'
|
||||
import { migrateJournalDomainSchema } from './journal-domain-schema'
|
||||
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
import {
|
||||
migrateJournalDomainHistoryBounds,
|
||||
writeJournalDomainHistoryBounds
|
||||
} from './journal-domain-history-bounds-schema'
|
||||
import { JOURNAL_DOMAIN_SCHEMA_VERSION } from './database/journal-domain-migrations'
|
||||
import { migrateJournalDomainPredecessor } from './database/journal-domain-migrations.test-fixture'
|
||||
|
||||
@@ -26,9 +31,15 @@ afterEach(() => {
|
||||
|
||||
function history(db = new Database(':memory:')) {
|
||||
const value = messageFixture(db)
|
||||
// The author installs this body as the next numbered migration and calls the bounds write at each item writer.
|
||||
migrateJournalDomainHistoryBounds(db)
|
||||
cleanup.push(() => db.close())
|
||||
return {
|
||||
...value,
|
||||
seedItem: (item: AgentJournalRenderItem) => {
|
||||
value.seedItem(item)
|
||||
writeJournalDomainHistoryBounds(db, target, item.itemId, journalDomainHistoryMetadata(item))
|
||||
},
|
||||
head: (sequence: number, retained = 0) =>
|
||||
db
|
||||
.prepare('UPDATE journal_chat_state SET last_sequence=?, retained_sequence=?')
|
||||
@@ -149,6 +160,29 @@ it('never splits a same-sequence group, marking it oversized backward and paging
|
||||
])
|
||||
})
|
||||
|
||||
it('excludes a retained body and submission by stored size before reading either, in both directions', () => {
|
||||
const { db, seedItem, head, event, read } = history()
|
||||
for (let sequence = 1; sequence <= 4; sequence++) {
|
||||
seedItem(text(sequence, 'row'))
|
||||
event(sequence, [`orca:${sequence}:0`])
|
||||
}
|
||||
head(4)
|
||||
// JSON5 satisfies the expression index but fails the read if this body were decoded.
|
||||
db.prepare("UPDATE journal_items SET payload_json=? WHERE item_id='orca:2:0'").run(
|
||||
`{"pad":"${'x'.repeat(10_000)}",}`
|
||||
)
|
||||
db.exec(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence,
|
||||
attempt_index,current,item_id,ordinal,submitted_at,fingerprint,dispatch_state,payload_json)
|
||||
VALUES('chat','epoch','send',2,0,1,'orca:2:0',2,2,'f','accepted','not json')`)
|
||||
const small = { form: measureHistoryItem, budgetBytes: 4096 }
|
||||
const tail = read({ direction: 'tail' }, small)
|
||||
expect(sequences(tail)).toEqual([3, 4])
|
||||
expect(tail.page).toMatchObject({ hasOlder: true, window: { nextCursor: { sequence: 3 } } })
|
||||
const forward = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 0 } }, small)
|
||||
expect(sequences(forward)).toEqual([1])
|
||||
expect(forward.page).toMatchObject({ hasNewer: true, window: { nextCursor: { sequence: 1 } } })
|
||||
})
|
||||
|
||||
it('catches up with the current revision, placement and submission, not the replayed metadata', async () => {
|
||||
const { db, commit, seedSend, read } = history()
|
||||
seedSend()
|
||||
|
||||
Reference in New Issue
Block a user