mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
Merge bounded chat history selection
This commit is contained in:
@@ -8,7 +8,9 @@ import type {
|
||||
import { readAgentSessionForwardHistory } from '../agent-session-wire/publication/history/agent-session-history-forward-page'
|
||||
import type { HistoryReader } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import type { JournalReadSince } from './journal-store-contracts'
|
||||
import type { JournalOutcomeReadRow } from './replay/journal-read-row'
|
||||
import type { JournalOutcomeReadRow, JournalReadRow } from './replay/journal-read-row'
|
||||
import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key'
|
||||
import { JOURNAL_DOMAIN_HISTORY_ITEM_BYTES } from './journal-domain-history-window'
|
||||
import { findSequenceGap, resolveJournalResume } from './replay/journal-cursor'
|
||||
import type { JournalDomainEventChanges } from './journal-domain-schema'
|
||||
import type { JournalDomainHistoryHead } from './journal-domain-history-page'
|
||||
@@ -27,9 +29,17 @@ export function readJournalDomainForwardHistory(
|
||||
reason: Extract<AgentSessionHistoryResult, { ok: false }>['reset']
|
||||
) => AgentSessionHistoryResult
|
||||
): AgentSessionHistoryResult {
|
||||
// One bounded identity read serves the size admission and the builder's lookahead.
|
||||
const since = cursor && readJournalDomainEvents(db, head, cursor, limit + 1)
|
||||
const admitted = since?.ok
|
||||
? fittingEventCount(db, head, since.rows.slice(0, limit), reader.budgetBytes)
|
||||
: limit
|
||||
return readAgentSessionForwardHistory(
|
||||
{
|
||||
readSince: (after, count) => readJournalDomainEvents(db, head, after, count),
|
||||
readSince: (after, count = limit + 1) =>
|
||||
since?.ok
|
||||
? firstEvents(since, count)
|
||||
: (since ?? readJournalDomainEvents(db, head, after, count)),
|
||||
canonicalItemId: (itemId) => itemId,
|
||||
hasSubmission: () => false,
|
||||
stopNoteIds: (ids) => journalDomainDependentStopNotes(db, head, ids),
|
||||
@@ -42,18 +52,65 @@ export function readJournalDomainForwardHistory(
|
||||
submissions: []
|
||||
},
|
||||
cursor,
|
||||
limit,
|
||||
admitted,
|
||||
reader,
|
||||
readSnapshot
|
||||
)
|
||||
}
|
||||
|
||||
/** The longest event prefix whose touched rows' stored sizes fit, so excluded bodies are never read. */
|
||||
function fittingEventCount(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
rows: readonly JournalReadRow[],
|
||||
budgetBytes: number
|
||||
): number {
|
||||
const touched = rows.map((row) =>
|
||||
row.kind === 'outcome'
|
||||
? [...row.itemIds, ...row.submissionIds.map(agentJournalSubmissionKey)]
|
||||
: []
|
||||
)
|
||||
const sizes = new Map(
|
||||
db
|
||||
.prepare(`SELECT item.item_id, ${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes FROM json_each(?) wanted
|
||||
CROSS JOIN journal_items item
|
||||
WHERE item.session_id=? AND item.epoch=? AND item.item_id=wanted.value AND item.hidden=0`)
|
||||
.all(JSON.stringify([...new Set(touched.flat())]), head.sessionId, head.epoch)
|
||||
.map((row) => [String(row.item_id), Number(row.bytes)])
|
||||
)
|
||||
const counted = new Set<string>()
|
||||
let total = 0
|
||||
let count = 0
|
||||
for (const ids of touched) {
|
||||
for (const id of ids) {
|
||||
if (!counted.has(id)) {
|
||||
counted.add(id)
|
||||
total += sizes.get(id) ?? 0
|
||||
}
|
||||
}
|
||||
if (count > 0 && total > budgetBytes) {
|
||||
break
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
return Math.max(count, 1)
|
||||
}
|
||||
|
||||
function firstEvents(
|
||||
since: Extract<JournalReadSince, { ok: true }>,
|
||||
count: number
|
||||
): JournalReadSince {
|
||||
return since.rows.length <= count
|
||||
? since
|
||||
: { ...since, rows: since.rows.slice(0, count), through: since.rows[count - 1]!.seq }
|
||||
}
|
||||
|
||||
/** Retained events start after the pruned-through cursor; a missing sequence resets, never pages partially. */
|
||||
function readJournalDomainEvents(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
cursor: AgentJournalCursor,
|
||||
limit = -1
|
||||
limit: number
|
||||
): JournalReadSince {
|
||||
const resume = resolveJournalResume(
|
||||
{
|
||||
@@ -70,7 +127,7 @@ function readJournalDomainEvents(
|
||||
.prepare(`SELECT sequence, changes_json FROM journal_events
|
||||
WHERE session_id=? AND epoch=? AND sequence>? ORDER BY sequence LIMIT ?`)
|
||||
.all(head.sessionId, head.epoch, resume.afterSequence, limit)
|
||||
const complete = limit < 0 || stored.length < limit
|
||||
const complete = stored.length < limit
|
||||
const sequences = stored.map((row) => Number(row.sequence))
|
||||
if (
|
||||
findSequenceGap(sequences, resume.afterSequence + 1) ||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { SqliteBindings } from '../../sqlite/sqlite-statement'
|
||||
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import {
|
||||
journalDomainHistoryMetadata,
|
||||
type JournalDomainHistoryMetadata
|
||||
} from './journal-domain-history-metadata'
|
||||
|
||||
type HistoryTarget = { sessionId: string; epoch: string }
|
||||
|
||||
/** Next numbered migration body: sent sizes and subagent names history selects before any body. */
|
||||
export function migrateJournalDomainHistoryBounds(db: Database.Database): void {
|
||||
db.exec(`
|
||||
ALTER TABLE journal_items ADD COLUMN wire_bytes INTEGER;
|
||||
CREATE TABLE journal_item_subagent_names (
|
||||
session_id TEXT NOT NULL,
|
||||
epoch TEXT NOT NULL,
|
||||
subagent_id TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, epoch, subagent_id, item_id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX journal_item_subagent_names_item ON journal_item_subagent_names(session_id, epoch, item_id);
|
||||
DROP INDEX journal_items_named_subagents;
|
||||
`)
|
||||
const page = db.prepare(`SELECT session_id, epoch, item_id, payload_json FROM journal_items
|
||||
WHERE (session_id, epoch, item_id) > (?, ?, ?) ORDER BY session_id, epoch, item_id LIMIT 500`)
|
||||
let after: SqliteBindings = ['', '', '']
|
||||
for (let rows = page.all(...after); rows.length > 0; rows = page.all(...after)) {
|
||||
for (const row of rows) {
|
||||
const payload = String(row.payload_json)
|
||||
const item: AgentJournalRenderItem = JSON.parse(payload)
|
||||
writeJournalDomainHistoryBounds(
|
||||
db,
|
||||
{ sessionId: String(row.session_id), epoch: String(row.epoch) },
|
||||
String(row.item_id),
|
||||
journalDomainHistoryMetadata(item, payload)
|
||||
)
|
||||
}
|
||||
const last = rows.at(-1)!
|
||||
after = [last.session_id, last.epoch, last.item_id]
|
||||
}
|
||||
}
|
||||
|
||||
/** Every journal_items upsert calls this after writing the row, in the same transaction. */
|
||||
export function writeJournalDomainHistoryBounds(
|
||||
db: Database.Database,
|
||||
target: HistoryTarget,
|
||||
itemId: string,
|
||||
metadata: Pick<JournalDomainHistoryMetadata, 'wireBytes' | 'namedSubagentIds'>
|
||||
): void {
|
||||
const key = [target.sessionId, target.epoch, itemId] as const
|
||||
db.prepare(
|
||||
'UPDATE journal_items SET wire_bytes=? WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(metadata.wireBytes, ...key)
|
||||
db.prepare(
|
||||
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(...key)
|
||||
const insert =
|
||||
db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(session_id,epoch,subagent_id,item_id)
|
||||
VALUES(?,?,?,?)`)
|
||||
const named: string[] = JSON.parse(metadata.namedSubagentIds ?? '[]')
|
||||
for (const subagentId of named) {
|
||||
insert.run(target.sessionId, target.epoch, subagentId, itemId)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,11 @@ import {
|
||||
subagentFoldKey,
|
||||
subagentItemBlock
|
||||
} from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import { isSubagentRow } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import {
|
||||
HISTORY_PAGE_CONTENT_BUDGET_BYTES,
|
||||
isSubagentRow
|
||||
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import { agentJournalItemForWire } from '../agent-session-wire/publication/history/agent-session-item-wire-form'
|
||||
|
||||
/** Facts history selects before decoding a body; written from each current item revision. */
|
||||
export type JournalDomainHistoryMetadata = {
|
||||
@@ -16,10 +20,14 @@ export type JournalDomainHistoryMetadata = {
|
||||
subagentGroupKey: string | null
|
||||
namedSubagentIds: string | null
|
||||
turnMarker: number
|
||||
/** Client-form bytes when they differ from the stored payload; null reads the stored length. */
|
||||
wireBytes: number | null
|
||||
}
|
||||
|
||||
/** Pass the stored payload text when the caller already serialized it. */
|
||||
export function journalDomainHistoryMetadata(
|
||||
item: AgentJournalRenderItem
|
||||
item: AgentJournalRenderItem,
|
||||
payloadJson?: string
|
||||
): JournalDomainHistoryMetadata {
|
||||
const root = isRootAgentJournalItem(item)
|
||||
const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null
|
||||
@@ -34,6 +42,22 @@ export function journalDomainHistoryMetadata(
|
||||
namedSubagentIds: named.length > 0 ? JSON.stringify(named) : null,
|
||||
turnMarker: Number(
|
||||
readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body)
|
||||
)
|
||||
),
|
||||
wireBytes: wireBytes(item, payloadJson)
|
||||
}
|
||||
}
|
||||
|
||||
// The client form cuts tool/diff heads and whole items over the page budget; skipping the
|
||||
// form elsewhere only overstates a size, which shortens a page but never drops content.
|
||||
function wireBytes(item: AgentJournalRenderItem, payloadJson: string | undefined): number | null {
|
||||
if (
|
||||
item.body.kind !== 'tool-call' &&
|
||||
item.body.kind !== 'diff' &&
|
||||
Buffer.byteLength(payloadJson ?? JSON.stringify(item), 'utf8') <=
|
||||
HISTORY_PAGE_CONTENT_BUDGET_BYTES
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const sent = agentJournalItemForWire(item)
|
||||
return sent.item === item ? null : sent.bytes
|
||||
}
|
||||
|
||||
@@ -11,8 +11,6 @@ import type {
|
||||
AgentJournalSubmission
|
||||
} from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import type { AgentJournalCursor } from '../../../shared/agent-session/journal/agent-session-journal-identity'
|
||||
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
|
||||
import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import {
|
||||
historyContentBytes,
|
||||
oversizedHistoryItem,
|
||||
@@ -21,14 +19,15 @@ import {
|
||||
type HistoryReader
|
||||
} from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
|
||||
import { buildAgentSessionHistoryPage } from '../agent-session-wire/publication/history/agent-session-history-page-result'
|
||||
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
|
||||
import { CLIENT_HISTORY_READER } from '../agent-session-wire/publication/history/agent-session-item-wire-form'
|
||||
import { journalContextEpoch } from './context-epochs/journal-context-cursor'
|
||||
import { readJournalDomainForwardHistory } from './journal-domain-history-after'
|
||||
import { readJournalDomainHistoryRoster } from './journal-domain-history-roster'
|
||||
import { projectJournalDomainStopNote } from './journal-domain-stop-notes'
|
||||
import {
|
||||
journalDomainSequenceGroups,
|
||||
selectJournalDomainHistoryWindow
|
||||
admitJournalDomainHistoryGroups,
|
||||
selectJournalDomainHistoryWindow,
|
||||
type JournalDomainHistoryHeader
|
||||
} from './journal-domain-history-window'
|
||||
|
||||
export type JournalDomainHistoryRequest = Omit<AgentSessionHistoryRequest, 'direction'> & {
|
||||
@@ -139,40 +138,18 @@ function historyPage(
|
||||
request.direction === 'hydrate'
|
||||
? AGENT_SESSION_HISTORY_MAX_LIMIT
|
||||
: resolveHistoryLimit(request.limit)
|
||||
const window = selectJournalDomainHistoryWindow(db, where, prefix, end, limit)
|
||||
const admitted = admitJournalDomainHistoryGroups(
|
||||
selectJournalDomainHistoryWindow(db, where, prefix, end, limit),
|
||||
reader.budgetBytes
|
||||
)
|
||||
const submissions = itemSubmissions(
|
||||
db,
|
||||
head,
|
||||
window.map((header) => header.itemId)
|
||||
admitted.groups.flat().map((header) => header.itemId)
|
||||
)
|
||||
const sizes = submissionBytesByItemId(submissions)
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const kept: AgentJournalRenderItem[][] = []
|
||||
let total = 0
|
||||
for (const group of journalDomainSequenceGroups(window)) {
|
||||
const formed = group.map((header) =>
|
||||
reader.form(
|
||||
projectJournalDomainStopNote(
|
||||
db,
|
||||
head,
|
||||
JSON.parse(String(body.get(...prefix, header.itemId)!.payload_json))
|
||||
)
|
||||
)
|
||||
)
|
||||
const bytes = historyContentBytes(formed, [], sizes)
|
||||
if (kept.length > 0 && total + bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
kept.push(
|
||||
formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item))
|
||||
)
|
||||
total += bytes
|
||||
if (bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
}
|
||||
const kept = admitted.oversized
|
||||
? [admitted.groups[0]!.map((header) => oversizedHistoryItem(itemEnvelope(db, prefix, header)))]
|
||||
: formHistoryGroups(db, head, admitted.groups, submissions, reader)
|
||||
const items = kept.toReversed().flat()
|
||||
const oldest = items[0]?.sequence ?? end
|
||||
const turn = latestTurnItem(db, head)
|
||||
@@ -196,10 +173,63 @@ function historyPage(
|
||||
fallbackCursor: cursor ?? { epoch: head.cursor.epoch, sequence: 0 },
|
||||
nextCursor: items[0] ? { epoch: head.cursor.epoch, sequence: items[0].sequence } : undefined,
|
||||
fence: request.fence,
|
||||
subagentRoster: historyRoster(db, prefix, items)
|
||||
subagentRoster: readJournalDomainHistoryRoster(db, prefix, items)
|
||||
})
|
||||
}
|
||||
|
||||
/** The existing byte rule, authoritative over admitted groups only. */
|
||||
function formHistoryGroups(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead,
|
||||
groups: readonly JournalDomainHistoryHeader[][],
|
||||
submissions: readonly AgentJournalSubmission[],
|
||||
reader: HistoryReaderForm
|
||||
): AgentJournalRenderItem[][] {
|
||||
const sizes = submissionBytesByItemId(submissions)
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const kept: AgentJournalRenderItem[][] = []
|
||||
let total = 0
|
||||
for (const group of groups) {
|
||||
const formed = group.map((header) =>
|
||||
reader.form(
|
||||
projectJournalDomainStopNote(
|
||||
db,
|
||||
head,
|
||||
JSON.parse(String(body.get(head.sessionId, head.epoch, header.itemId)!.payload_json))
|
||||
)
|
||||
)
|
||||
)
|
||||
const bytes = historyContentBytes(formed, [], sizes)
|
||||
if (kept.length > 0 && total + bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
kept.push(
|
||||
formed.map(({ item }) => (bytes > reader.budgetBytes ? oversizedHistoryItem(item) : item))
|
||||
)
|
||||
total += bytes
|
||||
if (bytes > reader.budgetBytes) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
/** An oversized group's placeholders keep each row's identity and placement, never its body. */
|
||||
function itemEnvelope(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
header: JournalDomainHistoryHeader
|
||||
): AgentJournalRenderItem {
|
||||
const row = db
|
||||
.prepare(`SELECT json_remove(payload_json, '$.body') AS envelope FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND item_id=?`)
|
||||
.get(...prefix, header.itemId)!
|
||||
// The placeholder replaces the omitted body.
|
||||
return JSON.parse(String(row.envelope))
|
||||
}
|
||||
|
||||
function latestTurnItem(
|
||||
db: Database.Database,
|
||||
head: JournalDomainHistoryHead
|
||||
@@ -245,47 +275,3 @@ function currentRows(
|
||||
submissions: itemSubmissions(db, head, wanted)
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads only root rows naming a subagent the page shows but does not name. */
|
||||
function historyRoster(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
page: readonly AgentJournalRenderItem[]
|
||||
) {
|
||||
const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null))
|
||||
for (const item of page) {
|
||||
if (item.agentId === undefined && item.body.kind === 'message') {
|
||||
for (const entry of item.body.blocks.flatMap(namedSubagents)) {
|
||||
missing.delete(entry.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (missing.size === 0) {
|
||||
return undefined
|
||||
}
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const named: AgentJournalRenderItem[] = []
|
||||
for (const row of db
|
||||
.prepare(`SELECT item_id, named_subagent_ids FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND hidden=0 AND root=1 AND named_subagent_ids IS NOT NULL
|
||||
ORDER BY sequence, sequence_index, ordinal`)
|
||||
.iterate(...prefix)) {
|
||||
const ids: string[] = JSON.parse(String(row.named_subagent_ids))
|
||||
if (!ids.some((id) => missing.has(id))) {
|
||||
continue
|
||||
}
|
||||
for (const id of ids) {
|
||||
missing.delete(id)
|
||||
}
|
||||
named.push(JSON.parse(String(body.get(...prefix, String(row.item_id))!.payload_json)))
|
||||
if (missing.size === 0 || named.length >= ROSTER_ROWS) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return offPageSubagentRoster(named, page)
|
||||
}
|
||||
|
||||
// The roster keeps at most 64 entries; each selected row adds at least one.
|
||||
const ROSTER_ROWS = 64
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
|
||||
import { namedSubagents } from '../../../shared/native-chat/transcript/native-chat-subagent-items'
|
||||
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
|
||||
|
||||
/** One indexed lookup per unnamed page subagent finds its earliest visible naming row. */
|
||||
export function readJournalDomainHistoryRoster(
|
||||
db: Database.Database,
|
||||
prefix: readonly [string, string],
|
||||
page: readonly AgentJournalRenderItem[]
|
||||
) {
|
||||
const missing = new Set(page.map(agentJournalItemSubagentId).filter((id) => id !== null))
|
||||
for (const item of page) {
|
||||
if (item.agentId === undefined && item.body.kind === 'message') {
|
||||
for (const entry of item.body.blocks.flatMap(namedSubagents)) {
|
||||
missing.delete(entry.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (missing.size === 0) {
|
||||
return undefined
|
||||
}
|
||||
const earliest = db.prepare(`SELECT item.item_id, item.sequence, item.sequence_index, item.ordinal
|
||||
FROM journal_item_subagent_names name CROSS JOIN journal_items item
|
||||
ON item.session_id=name.session_id AND item.epoch=name.epoch AND item.item_id=name.item_id
|
||||
WHERE name.session_id=? AND name.epoch=? AND name.subagent_id=? AND item.hidden=0 AND item.root=1
|
||||
ORDER BY item.sequence, item.sequence_index, item.ordinal LIMIT 1`)
|
||||
const naming = new Map<string, [number, number, number]>()
|
||||
for (const id of missing) {
|
||||
const row = earliest.get(...prefix, id)
|
||||
if (row) {
|
||||
naming.set(String(row.item_id), [
|
||||
Number(row.sequence),
|
||||
Number(row.sequence_index),
|
||||
Number(row.ordinal)
|
||||
])
|
||||
}
|
||||
}
|
||||
const body = db.prepare(
|
||||
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
)
|
||||
const named = [...naming]
|
||||
.sort(([, a], [, b]) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2])
|
||||
.slice(0, ROSTER_ROWS)
|
||||
.map(([itemId]) => JSON.parse(String(body.get(...prefix, itemId)!.payload_json)))
|
||||
return offPageSubagentRoster(named, page)
|
||||
}
|
||||
|
||||
// The roster keeps at most 64 entries; each selected row adds at least one.
|
||||
const ROSTER_ROWS = 64
|
||||
@@ -7,8 +7,16 @@ export type JournalDomainHistoryHeader = {
|
||||
sequence: number
|
||||
own: boolean
|
||||
subagentGroupKey: string | null
|
||||
/** Sent item plus current submission bytes, read without loading either body. */
|
||||
bytes: number
|
||||
}
|
||||
|
||||
/** Requires the item alias; octet_length reads a stored length, never the payload. */
|
||||
export const JOURNAL_DOMAIN_HISTORY_ITEM_BYTES = `COALESCE(item.wire_bytes, octet_length(item.payload_json))
|
||||
+ COALESCE((SELECT octet_length(attempt.payload_json) FROM journal_submission_attempts attempt
|
||||
WHERE attempt.session_id=item.session_id AND attempt.epoch=item.epoch
|
||||
AND attempt.item_id=item.item_id AND attempt.current=1), 0)`
|
||||
|
||||
/** Chooses the page's candidate identities before any body or submission is decoded. */
|
||||
export function selectJournalDomainHistoryWindow(
|
||||
db: Database.Database,
|
||||
@@ -29,11 +37,14 @@ function selectJournalDomainHistoryHeaders(
|
||||
end: number,
|
||||
limit: number
|
||||
): JournalDomainHistoryHeader[] {
|
||||
const columns = 'item_id, sequence, root, subagent_row, subagent_group_key'
|
||||
const columns = `item_id, sequence, root, subagent_row, subagent_group_key,
|
||||
${JOURNAL_DOMAIN_HISTORY_ITEM_BYTES} AS bytes`
|
||||
const order = 'ORDER BY sequence DESC, sequence_index DESC, ordinal DESC'
|
||||
const cap = 2 * limit
|
||||
const rows = db
|
||||
.prepare(`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence<? ${order} LIMIT ?`)
|
||||
.prepare(
|
||||
`SELECT ${columns} FROM journal_items item WHERE ${where} AND sequence<? ${order} LIMIT ?`
|
||||
)
|
||||
.all(...prefix, end, cap)
|
||||
if (rows.length === cap) {
|
||||
const boundary = rows.at(-1)!.sequence
|
||||
@@ -41,7 +52,7 @@ function selectJournalDomainHistoryHeaders(
|
||||
// A source lifecycle row creates at most 200 positions; ordinary domain writes create one.
|
||||
const group = db
|
||||
.prepare(
|
||||
`SELECT ${columns} FROM journal_items WHERE ${where} AND sequence=? ${order} LIMIT ?`
|
||||
`SELECT ${columns} FROM journal_items item WHERE ${where} AND sequence=? ${order} LIMIT ?`
|
||||
)
|
||||
.all(...prefix, boundary, MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS + 1)
|
||||
if (group.length > MAX_JOURNAL_LIFECYCLE_BATCH_MUTATIONS) {
|
||||
@@ -53,7 +64,8 @@ function selectJournalDomainHistoryHeaders(
|
||||
itemId: String(row.item_id),
|
||||
sequence: Number(row.sequence),
|
||||
own: row.root === 1 && row.subagent_row === 0,
|
||||
subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key)
|
||||
subagentGroupKey: row.subagent_group_key === null ? null : String(row.subagent_group_key),
|
||||
bytes: Number(row.bytes)
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -93,6 +105,27 @@ function historyWindow(
|
||||
return [...headers]
|
||||
}
|
||||
|
||||
/** Newest whole groups whose stored sizes fit; a lone first group over budget is oversized. */
|
||||
export function admitJournalDomainHistoryGroups(
|
||||
window: readonly JournalDomainHistoryHeader[],
|
||||
budgetBytes: number
|
||||
): { groups: JournalDomainHistoryHeader[][]; oversized: boolean } {
|
||||
const groups: JournalDomainHistoryHeader[][] = []
|
||||
let total = 0
|
||||
for (const group of journalDomainSequenceGroups(window)) {
|
||||
const bytes = group.reduce((sum, header) => sum + header.bytes, 0)
|
||||
if (groups.length > 0 && total + bytes > budgetBytes) {
|
||||
break
|
||||
}
|
||||
groups.push(group)
|
||||
total += bytes
|
||||
if (bytes > budgetBytes) {
|
||||
return { groups, oversized: true }
|
||||
}
|
||||
}
|
||||
return { groups, oversized: false }
|
||||
}
|
||||
|
||||
/** Newest group first; each group keeps render order. */
|
||||
export function* journalDomainSequenceGroups<T extends { sequence: number }>(
|
||||
rows: readonly T[]
|
||||
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
type JournalDomainHistoryRequest
|
||||
} from './journal-domain-history-page'
|
||||
import { migrateJournalDomainSchema } from './journal-domain-schema'
|
||||
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
import {
|
||||
migrateJournalDomainHistoryBounds,
|
||||
writeJournalDomainHistoryBounds
|
||||
} from './journal-domain-history-bounds-schema'
|
||||
import { JOURNAL_DOMAIN_SCHEMA_VERSION } from './database/journal-domain-migrations'
|
||||
import { migrateJournalDomainPredecessor } from './database/journal-domain-migrations.test-fixture'
|
||||
|
||||
@@ -26,9 +31,15 @@ afterEach(() => {
|
||||
|
||||
function history(db = new Database(':memory:')) {
|
||||
const value = messageFixture(db)
|
||||
// The author installs this body as the next numbered migration and calls the bounds write at each item writer.
|
||||
migrateJournalDomainHistoryBounds(db)
|
||||
cleanup.push(() => db.close())
|
||||
return {
|
||||
...value,
|
||||
seedItem: (item: AgentJournalRenderItem) => {
|
||||
value.seedItem(item)
|
||||
writeJournalDomainHistoryBounds(db, target, item.itemId, journalDomainHistoryMetadata(item))
|
||||
},
|
||||
head: (sequence: number, retained = 0) =>
|
||||
db
|
||||
.prepare('UPDATE journal_chat_state SET last_sequence=?, retained_sequence=?')
|
||||
@@ -149,6 +160,29 @@ it('never splits a same-sequence group, marking it oversized backward and paging
|
||||
])
|
||||
})
|
||||
|
||||
it('excludes a retained body and submission by stored size before reading either, in both directions', () => {
|
||||
const { db, seedItem, head, event, read } = history()
|
||||
for (let sequence = 1; sequence <= 4; sequence++) {
|
||||
seedItem(text(sequence, 'row'))
|
||||
event(sequence, [`orca:${sequence}:0`])
|
||||
}
|
||||
head(4)
|
||||
// JSON5 satisfies the expression index but fails the read if this body were decoded.
|
||||
db.prepare("UPDATE journal_items SET payload_json=? WHERE item_id='orca:2:0'").run(
|
||||
`{"pad":"${'x'.repeat(10_000)}",}`
|
||||
)
|
||||
db.exec(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence,
|
||||
attempt_index,current,item_id,ordinal,submitted_at,fingerprint,dispatch_state,payload_json)
|
||||
VALUES('chat','epoch','send',2,0,1,'orca:2:0',2,2,'f','accepted','not json')`)
|
||||
const small = { form: measureHistoryItem, budgetBytes: 4096 }
|
||||
const tail = read({ direction: 'tail' }, small)
|
||||
expect(sequences(tail)).toEqual([3, 4])
|
||||
expect(tail.page).toMatchObject({ hasOlder: true, window: { nextCursor: { sequence: 3 } } })
|
||||
const forward = read({ direction: 'after', cursor: { epoch: 'epoch', sequence: 0 } }, small)
|
||||
expect(sequences(forward)).toEqual([1])
|
||||
expect(forward.page).toMatchObject({ hasNewer: true, window: { nextCursor: { sequence: 1 } } })
|
||||
})
|
||||
|
||||
it('catches up with the current revision, placement and submission, not the replayed metadata', async () => {
|
||||
const { db, commit, seedSend, read } = history()
|
||||
seedSend()
|
||||
|
||||
Reference in New Issue
Block a user