Store saved tool outputs compressed (writes off until older builds can read them) (#43312)

* wip: stored tool output format, hydration and writer

* wip: compaction page bounds, fork payload fix, type fixes

* wip: merge stored-output imports

* fix: plain client heads, writer-entry hydration, compaction passes and row isolation

* fix: forward-only compaction cursor, length guard, no casts, worker baselines

* perf: cap stored text at 2 MiB and read selected blobs in kind scans

* feat: store finished tool outputs compressed with an inline preview for older builds

* fix: ship stored-output reads with writes off; fork capture reads selected blobs

Older builds rewrite and fork rows without knowing about stored outputs, so writes
stay off until those builds leave the downgrade window.

* fix: a stored output copied without its blob keeps its preview instead of failing the write

Forks of an output whose blob could not be read no longer abort; covers the writes-off rewrite.

* chore: move the items-by-kind page's sqlite uses to its own baseline entry

* perf: compaction reads row headers lazily so a page cut by its budget never refetches the rest

Up to 511 headers per page were fetched again by the next page when large rows ended it early.
This commit is contained in:
Brennan Benson
2026-11-16 22:45:17 -08:00
committed by GitHub
parent 5771447213
commit a902a7babb
37 changed files with 797 additions and 83 deletions
@@ -1,6 +1,7 @@
# Legacy synchronous compression uses per file; counts may only go DOWN.
# Only explicit worker/utility entry files are exempt; their imports remain guarded.
1 src/main/browser/extensions/actions/browser-extension-action-icon.ts
2 src/main/native-chat/agent-session-journal/journal-stored-output.ts
1 src/main/runtime/rpc/mobile/e2ee/mobile-e2ee-v2-compressed-frame.ts
1 src/main/ssh/remote-exec/ssh-posix-command-wrapper.ts
2 src/main/ssh/remote-exec/ssh-remote-powershell.ts
+5 -2
View File
@@ -38,7 +38,7 @@
3 src/main/native-chat/agent-session-journal/database/journal-checkpoint-owner.ts
13 src/main/native-chat/agent-session-journal/database/journal-database.ts
7 src/main/native-chat/agent-session-journal/database/journal-domain-database.ts
2 src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts
5 src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts
20 src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts
1 src/main/native-chat/agent-session-journal/database/journal-host-database.ts
32 src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts
@@ -103,6 +103,7 @@
10 src/main/native-chat/agent-session-journal/journal-fork-stage-page.ts
12 src/main/native-chat/agent-session-journal/journal-fork-stages.ts
4 src/main/native-chat/agent-session-journal/journal-fork-workspace.ts
2 src/main/native-chat/agent-session-journal/journal-items-by-kind-page.ts
2 src/main/native-chat/agent-session-journal/journal-open-queue-withdrawal.ts
4 src/main/native-chat/agent-session-journal/journal-sql-accepted-inputs.ts
6 src/main/native-chat/agent-session-journal/journal-sql-automatic-submission-command.ts
@@ -114,7 +115,7 @@
2 src/main/native-chat/agent-session-journal/journal-sql-edit-lease-command.ts
12 src/main/native-chat/agent-session-journal/journal-sql-event-tail.ts
4 src/main/native-chat/agent-session-journal/journal-sql-history-items.ts
24 src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts
22 src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts
10 src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts
5 src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts
12 src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts
@@ -123,6 +124,8 @@
10 src/main/native-chat/agent-session-journal/journal-sql-rewind-command.ts
4 src/main/native-chat/agent-session-journal/journal-sql-send-admission.ts
7 src/main/native-chat/agent-session-journal/journal-sql-steps.ts
10 src/main/native-chat/agent-session-journal/journal-stored-output-compaction.ts
4 src/main/native-chat/agent-session-journal/journal-stored-output-rows.ts
4 src/main/native-chat/agent-session-journal/journal-writer-effects.ts
4 src/main/native-chat/agent-session-journal/journal-writer-operation-expiry.ts
12 src/main/native-chat/agent-session-journal/journal-writer-store-mutation.ts
@@ -20,4 +20,15 @@ export function migrateJournalDomainSchema(db: Database.Database): void {
db.pragma(`user_version = ${JOURNAL_DOMAIN_SCHEMA_VERSION}`)
})
}
ensureJournalStoredOutputColumn(db)
}
/** Added without a user_version change: released builds name their columns and never read it. */
function ensureJournalStoredOutputColumn(db: Database.Database): void {
const present = db
.prepare(`SELECT 1 FROM pragma_table_info('journal_items') WHERE name='output_blob'`)
.get()
if (!present) {
db.exec('ALTER TABLE journal_items ADD COLUMN output_blob BLOB')
}
}
@@ -32,6 +32,7 @@ import { assertJournalFence } from './writing/journal-write-guards'
import { queueClearRestatement, writeJournalDomainQueueClear } from './journal-domain-queue-clear'
import { restoreRetainedFailures, retainedFailures } from './journal-domain-retained-failures'
import { invalidateJournalDomainQueue } from './journal-domain-queue-revision'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
@@ -200,7 +201,7 @@ export function replaceJournalDomainContext(
const itemId = itemIds[index]!
const stored = current.get(...prefix, itemId)
const previous: AgentJournalRenderItem | undefined = stored
? JSON.parse(String(stored.payload_json))
? readJournalStoredItem(db, prefix, stored.payload_json)
: undefined
const tombstone = db
.prepare(
@@ -35,6 +35,7 @@ import {
import { appendJournalSqlEvent } from './journal-sql-event-tail'
import { restoreJournalSqlFailure } from './journal-sql-failure'
import { invalidateJournalDomainQueue } from './journal-domain-queue-revision'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
@@ -299,7 +300,7 @@ function visibleItem(
WHERE session_id=? AND epoch=? AND hidden=0 AND item_id=COALESCE((SELECT item_id FROM journal_item_aliases
WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`)
.get(target.sessionId, target.epoch, target.sessionId, target.epoch, key, key)
return row ? JSON.parse(String(row.payload_json)) : null
return row ? readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json) : null
}
function physicalCoordinate(position: JournalDomainInputCoordinate): JournalWriterCoordinate {
@@ -27,6 +27,11 @@ import {
import { invalidateJournalDomainQueue } from './journal-domain-queue-revision'
import { readAgentJournalItemBody } from '../../../shared/agent-session/journal/agent-session-journal-body-admission'
import { AgentSessionJournalError } from './writing/journal-write-guards'
import {
hydrateJournalStoredItem,
journalStoredItemRow,
readJournalStoredItem
} from './journal-stored-output-rows'
export type JournalDomainCurrentItemWrite = {
item: AgentJournalRenderItem
@@ -59,11 +64,16 @@ function nextJournalDomainItemOrdinal(
export function writeJournalDomainCurrentItem(
db: Database.Database,
target: { sessionId: string; epoch: string },
input: JournalDomainCurrentItemWrite
write: JournalDomainCurrentItemWrite
): AgentJournalRenderItem[] {
if (!db.isTransaction) {
throw new Error('journal item persistence requires its writer transaction')
}
// A stored form handed back is written from its text, so every derived column stays exact.
const input = {
...write,
item: hydrateJournalStoredItem(db, [target.sessionId, target.epoch], write.item)
}
const body = normalizeJournalDomainItemBody(
db,
target,
@@ -98,7 +108,7 @@ export function writeJournalDomainCurrentItem(
(SELECT item_id FROM journal_item_aliases
WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`)
.get(...prefix, ...prefix, id, id)
return row ? JSON.parse(String(row.payload_json)) : undefined
return row ? readJournalStoredItem(db, prefix, row.payload_json) : undefined
})
const next = stopNote.item
const sequence = input.writtenSequence ?? next.sequence
@@ -111,11 +121,13 @@ export function writeJournalDomainCurrentItem(
.get(...prefix, next.itemId)
: undefined
const payload = JSON.stringify(item)
const { metadata, values } = journalDomainItemColumns(item, payload, next)
const columns = journalDomainItemColumns(item, payload, next)
const { metadata } = columns
const row = journalStoredItemRow(db, prefix, item, payload, columns.values)
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json,
hidden,original_stop_note_json,queue_command_turn,queue_failed_outcome,queue_failed_at,
queue_failure_witnessed_through,queue_failure_report_turn_item_id,placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')})
VALUES(${Array.from({ length: 18 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')})
queue_failure_witnessed_through,queue_failure_report_turn_item_id,placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')},output_blob)
VALUES(${Array.from({ length: 19 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')})
ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
ordinal=excluded.ordinal, revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index,
owner_fence=excluded.owner_fence, execution_owner=excluded.execution_owner, payload_json=excluded.payload_json,
@@ -123,7 +135,8 @@ export function writeJournalDomainCurrentItem(
queue_command_turn=excluded.queue_command_turn, queue_failed_outcome=excluded.queue_failed_outcome,
queue_failed_at=excluded.queue_failed_at, queue_failure_witnessed_through=excluded.queue_failure_witnessed_through,
queue_failure_report_turn_item_id=excluded.queue_failure_report_turn_item_id,placement_json=excluded.placement_json,
${JOURNAL_DOMAIN_ITEM_COLUMNS.map((column) => `${column}=excluded.${column}`).join(',')}`).run(
${JOURNAL_DOMAIN_ITEM_COLUMNS.map((column) => `${column}=excluded.${column}`).join(',')},
output_blob=CASE WHEN ? THEN journal_items.output_blob ELSE excluded.output_blob END`).run(
...prefix,
next.itemId,
ordinal,
@@ -132,7 +145,7 @@ export function writeJournalDomainCurrentItem(
next.sequenceIndex ?? 0,
input.ownerFence,
input.executionOwner ?? null,
payload,
row.payload,
Number(input.hidden),
stopNote.originalStopNoteJson,
failure.commandTurn,
@@ -141,7 +154,9 @@ export function writeJournalDomainCurrentItem(
failure.witnessedThrough,
failure.reportTurnItemId,
journalDomainItemPlacement(next),
...values
...row.values,
row.blob,
Number(row.keep)
)
writeJournalDomainItemNames(
db,
@@ -3,6 +3,7 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session/journ
import type { AgentJournalTurnScope } from '../../../shared/agent-session/journal/agent-session-journal-turn-scope'
import { writeJournalDomainCurrentItem } from './journal-domain-current-item'
import type { JournalDomainSubmissionTarget } from './journal-domain-submission-records'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type JournalDomainDispatchItem = {
item: AgentJournalRenderItem
@@ -23,7 +24,7 @@ export function readJournalDomainDispatchItem(
.get(target.sessionId, target.epoch, itemId)
return row
? {
item: JSON.parse(String(row.payload_json)),
item: readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json),
ordinal: Number(row.ordinal),
hidden: Boolean(row.hidden),
ownerFence: Number(row.owner_fence),
@@ -11,6 +11,7 @@ import { readJournalDomainSubmissionBody } from './journal-domain-submission-rec
import type { JournalHistoryOrigin } from './context-epochs/journal-history-origin'
import type { JournalLatestStop } from './context-epochs/journal-stop-turn-end'
import { providerTimelineKeyPart } from '../agent-session-timeline/provider-timeline-identity'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type JournalDomainEntityRead = Extract<
JournalSqlPointRead['operation'],
@@ -107,7 +108,9 @@ export function readJournalDomainEntity(
.prepare(`SELECT payload_json FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`)
.get(...prefix, operation.itemId)
const item: AgentJournalRenderItem | null = row ? JSON.parse(String(row.payload_json)) : null
const item: AgentJournalRenderItem | null = row
? readJournalStoredItem(db, prefix, row.payload_json)
: null
return { kind: operation.kind, item }
}
case 'submission': {
@@ -13,6 +13,7 @@ import {
} from './journal-domain-item-normalization'
import { journalDomainContextHead } from './journal-sql-continuation-context'
import type { JournalStopSettleFact } from './journal-writer-host-facts'
import { readJournalStoredItem } from './journal-stored-output-rows'
/** Select current lifecycle obligations and their named turn/opener evidence, in render order. */
export function readJournalDomainGeneration(
@@ -28,7 +29,7 @@ export function readJournalDomainGeneration(
{ item: AgentJournalRenderItem; ordinal: number; fence: number | undefined }
>()
const include = (row: Record<string, unknown>) => {
const item: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const item: AgentJournalRenderItem = readJournalStoredItem(db, prefix, row.payload_json)
selected.set(item.itemId, {
item,
ordinal: Number(row.ordinal),
@@ -3,6 +3,7 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session/journ
import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer'
import { namedSubagents } from '../../../shared/native-chat/transcript/subagents/native-chat-subagent-items'
import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster'
import { readJournalStoredItem } from './journal-stored-output-rows'
/** One indexed lookup per unnamed page subagent finds its earliest visible naming row. */
export function readJournalDomainHistoryRoster(
@@ -42,7 +43,7 @@ export function readJournalDomainHistoryRoster(
const named = [...naming]
.sort(([, a], [, b]) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2])
.slice(0, ROSTER_ROWS)
.map(([itemId]) => JSON.parse(String(body.get(...prefix, itemId)!.payload_json)))
.map(([itemId]) => readJournalStoredItem(db, prefix, body.get(...prefix, itemId)!.payload_json))
return offPageSubagentRoster(named, page)
}
@@ -10,6 +10,7 @@ import { insertJournalDomainQueueRow, journalDomainQueueRow } from './journal-do
import type { JournalDomainImportHeader, JournalDomainImportPage } from './journal-domain-import'
import { writeJournalDomainItemNames } from './journal-domain-item-names'
import { recordJournalLegacyChatRevision } from './database/journal-legacy-source'
import { hydrateJournalStoredItem, journalStoredItemRow } from './journal-stored-output-rows'
export type JournalDomainImportState =
| { state: 'converting'; epoch: null }
@@ -230,15 +231,15 @@ export function insertJournalDomainItemRows(
const insert = db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,
sequence,sequence_index,owner_fence,execution_owner,payload_json,hidden,original_stop_note_json,
queue_command_turn,queue_failed_outcome,queue_failed_at,queue_failure_witnessed_through,queue_failure_report_turn_item_id,
placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')})
VALUES(${Array.from({ length: 18 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) ${replace ? `ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET ${['ordinal', 'revision', 'sequence', 'sequence_index', 'owner_fence', 'execution_owner', 'payload_json', 'hidden', 'original_stop_note_json', 'queue_command_turn', 'queue_failed_outcome', 'queue_failed_at', 'queue_failure_witnessed_through', 'queue_failure_report_turn_item_id', 'placement_json', ...JOURNAL_DOMAIN_ITEM_COLUMNS].map((column) => `${column}=excluded.${column}`).join(',')}` : 'ON CONFLICT DO NOTHING'}`)
placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')},output_blob)
VALUES(${Array.from({ length: 19 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) ${replace ? `ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET ${['ordinal', 'revision', 'sequence', 'sequence_index', 'owner_fence', 'execution_owner', 'payload_json', 'hidden', 'original_stop_note_json', 'queue_command_turn', 'queue_failed_outcome', 'queue_failed_at', 'queue_failure_witnessed_through', 'queue_failure_report_turn_item_id', 'placement_json', ...JOURNAL_DOMAIN_ITEM_COLUMNS].map((column) => `${column}=excluded.${column}`).join(',')},output_blob=CASE WHEN ? THEN journal_items.output_blob ELSE excluded.output_blob END` : 'ON CONFLICT DO NOTHING'}`)
for (const {
ordinal,
ownerFence,
executionOwner,
hidden,
item,
displayed,
item: given,
displayed: shown,
originalStopNoteJson,
queueCommandTurn,
queueFailedOutcome,
@@ -246,8 +247,13 @@ export function insertJournalDomainItemRows(
queueFailureWitnessedThrough,
queueFailureReportTurnItemId
} of rows) {
const { payload, metadata, values, placement } =
prepared ?? prepareJournalDomainItemRow(item, displayed)
const item = hydrateJournalStoredItem(db, prefix, given)
const displayed = shown && hydrateJournalStoredItem(db, prefix, shown)
const encoded =
(item === given && displayed === shown ? prepared : undefined) ??
prepareJournalDomainItemRow(item, displayed)
const { metadata, placement } = encoded
const row = journalStoredItemRow(db, prefix, item, encoded.payload, encoded.values)
const result = insert.run(
...prefix,
item.itemId,
@@ -257,7 +263,7 @@ export function insertJournalDomainItemRows(
item.sequenceIndex ?? 0,
ownerFence,
executionOwner,
payload,
row.payload,
Number(hidden),
originalStopNoteJson,
Number(queueCommandTurn),
@@ -266,7 +272,9 @@ export function insertJournalDomainItemRows(
queueFailureWitnessedThrough,
queueFailureReportTurnItemId,
placement,
...values
...row.values,
row.blob,
...(replace ? [Number(row.keep)] : [])
)
names = metadata.namedSubagentIds
if (writeNames && Number(result.changes) > 0) {
@@ -11,6 +11,7 @@ import { readCodexErrorWillRetry } from '../../codex/app-server/codex-structured
import type { JournalInputEnd } from './input-order/journal-input-obligations'
import type { JournalDomainSubmissionStoredFacts } from './journal-domain-submission-records'
import type { JournalItemRow, JournalStopEvent } from './durable-rows/journal-row-schema'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
export type JournalDomainInputItemObservation = Pick<
@@ -184,7 +185,7 @@ export function observeJournalDomainInputStop(
WHERE session_id=? AND epoch=? AND item_id=?`)
.get(target.sessionId, target.epoch, scope.turnItemId)
const item: AgentJournalRenderItem | null = current
? JSON.parse(String(current.payload_json))
? readJournalStoredItem(db, [target.sessionId, target.epoch], current.payload_json)
: null
turnId = readAgentJournalTurn(item?.body)?.turnId
}
@@ -3,6 +3,7 @@ import type Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract'
import { projectJournalStopNoteForTurnState } from './context-epochs/journal-stop-note-projection'
import { hydrateJournalStoredItem } from './journal-stored-output-rows'
export type JournalDomainItemTarget = { sessionId: string; epoch: string }
@@ -83,7 +84,14 @@ export function readJournalDomainDisplayedRow(
row: SqliteRow,
stopStates?: ReadonlyMap<string, unknown>
): { item: AgentJournalRenderItem; payload: string } {
const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const parsed: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
// A row that did not select `output_blob` has it read by key.
const source = hydrateJournalStoredItem(
db,
[target.sessionId, target.epoch],
parsed,
row.output_blob
)
const states =
row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? undefined
@@ -96,6 +104,7 @@ export function readJournalDomainDisplayedRow(
)
return {
item,
payload: item === source ? String(row.payload_json) : JSON.stringify(item)
// The row text is the item only when nothing was read back from its stored output.
payload: item === parsed ? String(row.payload_json) : JSON.stringify(item)
}
}
@@ -8,6 +8,7 @@ import {
} from '../agent-session-wire/publication/history/agent-session-item-wire-form'
import type { HistoryItemForm } from '../agent-session-wire/publication/history/agent-session-history-page-bounds'
import { journalDomainItemWire } from './journal-domain-item-wire'
import { hydrateJournalStoredItem, readJournalStoredItem } from './journal-stored-output-rows'
import {
journalDomainDisplayTurnState,
readJournalDomainStopStates,
@@ -20,7 +21,9 @@ export function journalDomainClientFormColumns(alias = 'item'): string {
CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION}
THEN COALESCE(${alias}.wire_json,${alias}.payload_json) ELSE ${alias}.payload_json END AS client_json,
CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION}
THEN COALESCE(${alias}.wire_bytes,octet_length(${alias}.payload_json)) END AS client_bytes`
THEN COALESCE(${alias}.wire_bytes,octet_length(${alias}.payload_json)) END AS client_bytes,
CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION} AND ${alias}.wire_json IS NOT NULL
THEN NULL ELSE ${alias}.output_blob END AS output_blob`
}
/** Decode only this selected set; callers retain admitted forms and discard a rejected whole group. */
@@ -44,16 +47,24 @@ export function readJournalDomainClientForms(
if (row.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION && states === undefined) {
states = readJournalDomainStopStates(db, target, wanted, wantedJson)
}
forms.set(String(row.item_id), readJournalDomainClientForm(row, states))
forms.set(String(row.item_id), readJournalDomainClientForm(db, target, row, states))
}
return forms
}
export function readJournalDomainClientForm(
db: Database.Database,
target: JournalDomainItemTarget,
row: SqliteRow,
states?: ReadonlyMap<string, unknown>
): HistoryItemForm {
const item: AgentJournalRenderItem = JSON.parse(String(row.client_json))
// The blob is selected only when the client form is the stored payload, never beside a saved head.
const item = readJournalStoredItem(
db,
[target.sessionId, target.epoch],
row.client_json,
row.output_blob
)
return row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? rememberAgentJournalItemWireForm(item, Number(row.client_bytes))
: agentJournalItemForWire(
@@ -91,7 +102,8 @@ export function readJournalDomainItemDetail(
if (!row) {
return null
}
const item: AgentJournalRenderItem = JSON.parse(String(row.detail_json))
const parsed: AgentJournalRenderItem = JSON.parse(String(row.detail_json))
const item = hydrateJournalStoredItem(db, [target.sessionId, target.epoch], parsed)
if (row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION) {
return item
}
@@ -102,6 +114,10 @@ export function readJournalDomainItemDetail(
null,
journalDomainDisplayTurnState(item, states)
)
const wire = journalDomainItemWire(item, String(row.detail_json), displayed)
const wire = journalDomainItemWire(
item,
item === parsed ? String(row.detail_json) : undefined,
displayed
)
return wire.detailJson === null ? displayed : JSON.parse(wire.detailJson)
}
@@ -14,6 +14,7 @@ import { observeJournalDomainTurnScope } from './journal-domain-turn-scope'
import { appendJournalSqlEvent } from './journal-sql-event-tail'
import { assertJournalFence } from './writing/journal-write-guards'
import { invalidateJournalDomainQueue } from './journal-domain-queue-revision'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
@@ -72,7 +73,7 @@ export function removeJournalDomainItem(
return { itemId, revision, removed: null, items: [] }
}
const removed: AgentJournalRenderItem | null =
row.payload_json === null ? null : JSON.parse(String(row.payload_json))
row.payload_json === null ? null : readJournalStoredItem(db, prefix, row.payload_json)
if (
row.turn_marker ||
row.prompt ||
@@ -35,6 +35,7 @@ import { writeJournalDomainCurrentItem } from './journal-domain-current-item'
import { writeJournalDomainTextSnapshot } from './journal-domain-text-snapshot'
import { appendJournalSqlEvent } from './journal-sql-event-tail'
import { readJournalSqlAcceptedInputs } from './journal-sql-accepted-inputs'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type { JournalDomainItemWrite, JournalDomainItemCommit } from './journal-domain-item-write'
/** Runs only on the owning writer connection, inside its command SAVEPOINT. */
@@ -137,7 +138,7 @@ export function applyJournalDomainItem(
sequence: Number(stored.sequence),
body: input.body
}
: JSON.parse(String(stored.payload_json))
: readJournalStoredItem(db, prefix, stored.payload_json)
: undefined
const item = (key: string): AgentJournalRenderItem | null => {
if (key === rawId || key === itemId) {
@@ -148,7 +149,7 @@ export function applyJournalDomainItem(
WHERE item.session_id=? AND item.epoch=? AND item.item_id=COALESCE(
(SELECT item_id FROM journal_item_aliases WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`)
.get(...prefix, ...prefix, key, key)
return row ? JSON.parse(String(row.payload_json)) : null
return row ? readJournalStoredItem(db, prefix, row.payload_json) : null
}
let scope = input.options.turnScope ?? { kind: 'thread' as const }
for (const key of input.scopeItemIds ?? []) {
@@ -1,16 +1,19 @@
import { journalForkStageEpochs } from './journal-fork-stage-state'
import type Database from '../../sqlite/sync-database'
import type { JournalStoredOutputCompaction } from './journal-stored-output-compaction'
export type JournalDomainMaintenance =
| { kind: 'domain-retired-prune'; sessionId: string; epoch: string }
| { kind: 'page-reclaim' }
| { kind: 'compaction-state' }
| { kind: 'compaction-complete'; now: number }
| { kind: 'stored-output-compaction' }
export type JournalDomainMaintenanceResult =
| { kind: 'domain-retired'; deleted: number; more: boolean }
| { kind: 'page-reclaim'; more: boolean }
| { kind: 'compaction-state'; completedAt: number | null }
| JournalStoredOutputCompaction
const RETIRED_PAGE_BYTES = 1024 * 1024
// Only a ready head naming another epoch proves retirement; import shells have no epoch.
@@ -27,7 +30,8 @@ export const JOURNAL_EPOCH_TABLES: readonly {
'original_stop_note_json',
'wire_json',
'detail_json',
'outline_source_json'
'outline_source_json',
'output_blob'
]
},
{ name: 'journal_outline_cache', key: '', payload: ['outline_json'] },
@@ -18,6 +18,7 @@ import {
import { JournalSqlScope } from './journal-sql-scope'
import type { JournalRequestMark } from './journal-request-failure'
import type { JournalStopSettleFact } from './journal-writer-host-facts'
import { readJournalStoredItem } from './journal-stored-output-rows'
export class JournalDomainRequestFailure {
private readonly scope: JournalSqlScope
@@ -54,7 +55,7 @@ export class JournalDomainRequestFailure {
item(id: string): AgentJournalRenderItem | null {
const row = this.row(id)
return row ? JSON.parse(String(row.payload_json)) : null
return row ? readJournalStoredItem(this.scope.db, this.scope.prefix, row.payload_json) : null
}
request(kind: JournalRequestMark['kind'], id: string): JournalRequestMark | null {
@@ -64,7 +65,9 @@ export class JournalDomainRequestFailure {
return null
}
const row = this.row(itemId)
const item: AgentJournalRenderItem | null = row ? JSON.parse(String(row.payload_json)) : null
const item: AgentJournalRenderItem | null = row
? readJournalStoredItem(this.scope.db, this.scope.prefix, row.payload_json)
: null
if (!item || !row?.root || item.sequence <= this.floor) {
return null
}
@@ -2,6 +2,7 @@ import type Database from '../../sqlite/sync-database'
import { isDeepStrictEqual } from 'node:util'
import { agentJournalLinkageFields } from '../../../shared/agent-session/journal/agent-session-journal-producer'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
@@ -30,7 +31,7 @@ export function retainedFailures(
const failedAt = Number(row.queue_failed_at)
return {
itemId,
item: JSON.parse(String(row.payload_json)),
item: readJournalStoredItem(db, prefix, row.payload_json),
at: Math.max(
failedAt,
boundary > Number(row.queue_failure_witnessed_through) ? boundary : 0
@@ -51,7 +52,7 @@ export function restoreRetainedFailures(
for (const { itemId, item, at, reports } of retained) {
const row = kept.get(...prefix, itemId)
const next: AgentJournalRenderItem | undefined = row
? JSON.parse(String(row.payload_json))
? readJournalStoredItem(db, prefix, row.payload_json)
: undefined
if (
!next ||
@@ -6,6 +6,7 @@ import { projectJournalStopNote } from './context-epochs/journal-stop-note-proje
import { journalDomainItemWire } from './journal-domain-item-wire'
import { journalDomainOutlineSource } from './journal-domain-outline-source'
import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract'
import { readJournalStoredItem } from './journal-stored-output-rows'
type Target = { sessionId: string; epoch: string }
@@ -52,7 +53,7 @@ export function refreshJournalDomainStopNotes(
WHERE session_id=? AND epoch=? AND item_id=?`)
const get = (id: string): AgentJournalRenderItem | undefined => {
const row = turn.get(...prefix, id)
return row ? JSON.parse(String(row.payload_json)) : undefined
return row ? readJournalStoredItem(db, prefix, row.payload_json) : undefined
}
const priorGet = (id: string) => (previousTurns?.has(id) ? previousTurns.get(id) : get(id))
const update =
@@ -60,7 +61,7 @@ export function refreshJournalDomainStopNotes(
WHERE session_id=? AND epoch=? AND item_id=?`)
const changed: AgentJournalRenderItem[] = []
for (const row of notes) {
const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const source: AgentJournalRenderItem = readJournalStoredItem(db, prefix, row.payload_json)
const prior: AgentJournalRenderItem =
row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION
? resolveJournalDomainDisplay(source, row.wire_version, row.display_json, undefined)
@@ -6,6 +6,7 @@ import { runningTurnLifecycleRevisions } from '../agent-session-wire/termination
import { applyJournalDomainLifecycle } from './journal-domain-lifecycle'
import type { JournalWriterOperation } from './journal-writer-command'
import { readJournalStoredItem } from './journal-stored-output-rows'
export function applyJournalDomainLifecycleOperation(
db: Database.Database,
@@ -49,7 +50,9 @@ function applyJournalDomainTakenStop(
WHERE session_id=? AND epoch=? AND turn_id=? AND hidden=0 AND lifecycle_state='running'
ORDER BY sequence,sequence_index,ordinal`)
.all(...prefix)
.map((row): AgentJournalRenderItem => JSON.parse(String(row.payload_json)))
.map((row): AgentJournalRenderItem =>
readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json)
)
const result = applyJournalDomainLifecycle(
db,
target,
@@ -8,6 +8,7 @@ import {
} from './journal-domain-item-normalization'
import type { JournalStopSettleFact } from './journal-writer-host-facts'
import { assertJournalDomainItemAuthority } from './journal-domain-item'
import { readJournalStoredItem } from './journal-stored-output-rows'
/** Typed named selections used only during the owning synchronous writer command. */
export function journalDomainTimelineReads(
@@ -18,7 +19,7 @@ export function journalDomainTimelineReads(
): JournalTimelineView {
const prefix = [target.sessionId, target.epoch] as const
const decode = (row: Record<string, unknown> | undefined): AgentJournalRenderItem | null =>
row ? JSON.parse(String(row.payload_json)) : null
row ? readJournalStoredItem(db, prefix, row.payload_json) : null
const item = (key: string) =>
decode(
db
@@ -54,7 +54,8 @@ export class JournalForkCapture {
for (const row of this.db
.prepare(`SELECT header.rank,header.item_id,header.provider_key,
item.wire_version,CASE WHEN item.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION}
AND item.original_stop_note_json IS NOT NULL THEN item.wire_json END AS wire_json,item.payload_json
AND item.original_stop_note_json IS NOT NULL THEN item.wire_json END AS wire_json,item.payload_json,
item.output_blob
FROM temp.fork_headers header LEFT JOIN journal_items item
ON item.session_id=?3 AND item.epoch=?4 AND item.item_id=header.item_id
WHERE header.rank>?1 AND (header.rank<?2 OR EXISTS(SELECT 1 FROM temp.fork_kept WHERE member=header.member))
@@ -96,7 +97,8 @@ export class JournalForkCapture {
this.db.prepare('INSERT OR IGNORE INTO temp.fork_omitted(id) VALUES(?)').run(item.itemId)
}
this.after = Number(row.rank)
account(journalForkRowBytes(row))
// The staged body is the restored text, not the row's stored form.
account(journalForkRowBytes({ ...row, payload_json: payload, output_blob: null }))
drainEdges()
if (!budget()) {
exhausted = false
@@ -24,6 +24,7 @@ import {
import { journalDomainStopNote } from './journal-domain-stop-notes'
import type { Stage } from './journal-fork-stage-state'
import { readJournalStoredItem } from './journal-stored-output-rows'
export function applyJournalForkStagePage(
db: Database.Database,
@@ -108,7 +109,8 @@ export function applyJournalForkStagePage(
)
.get(stage.target.sessionId, stage.epoch, itemId)
const previous: AgentJournalRenderItem | undefined =
stored && JSON.parse(String(stored.payload_json))
stored &&
readJournalStoredItem(db, [stage.target.sessionId, stage.epoch], stored.payload_json)
const scope = JournalDerivedTurnScope.restore(next.derived)
const rendered: AgentJournalRenderItem = {
itemId,
@@ -135,7 +137,9 @@ export function applyJournalForkStagePage(
'SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
)
.get(stageTarget.sessionId, stageTarget.epoch, id)
return saved ? JSON.parse(String(saved.payload_json)) : undefined
return saved
? readJournalStoredItem(db, [stageTarget.sessionId, stageTarget.epoch], saved.payload_json)
: undefined
})
const names = insertJournalDomainItemRows(
db,
@@ -8,6 +8,7 @@ import type { JournalStopSettleFact } from './journal-writer-host-facts'
import type { JournalSqlScope } from './journal-sql-scope'
import { readAgentJournalTurn } from '../../../shared/agent-session/lifecycle/records/agent-session-turn-record'
import { readJournalDomainGeneration } from './journal-domain-generation-reads'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type JournalGenerationRead = {
startKey?: string
@@ -78,7 +79,11 @@ export function readJournalGeneration(
)
.get(...scope.prefix, itemId)
if (row) {
const item: AgentJournalRenderItem = JSON.parse(String(row.payload_json))
const item: AgentJournalRenderItem = readJournalStoredItem(
scope.db,
scope.prefix,
row.payload_json
)
items.set(item.itemId, item)
}
}
@@ -1,5 +1,7 @@
import type { SqliteRow } from '../../sqlite/sqlite-statement'
import type Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { measureJournalStoredRow } from './journal-stored-output-rows'
/**
* A row-count page of tool output can reach hundreds of MB, which main decodes on its shared heap;
@@ -8,29 +10,38 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session/journ
*/
export const ITEMS_BY_KIND_PAGE_JSON_BUDGET = 8 * 2 ** 20
/** A row's decoded JSON length, and its item, decoded only once the row fits the page. */
export type ItemsByKindRowRead = { bytes: number; item: () => AgentJournalRenderItem }
function readInlineRow(row: Record<string, unknown>): ItemsByKindRowRead {
const json = String(row.payload_json)
return { bytes: json.length, item: () => JSON.parse(json) }
}
/** `rows` are the chosen items in ordinal order, read lazily so rows past the budget stay unread. */
export function budgetItemsByKindPage(
rows: Iterable<Record<string, unknown>>,
nextOrdinal: number | null,
budget = ITEMS_BY_KIND_PAGE_JSON_BUDGET
budget = ITEMS_BY_KIND_PAGE_JSON_BUDGET,
read: (row: Record<string, unknown>) => ItemsByKindRowRead = readInlineRow
): { items: AgentJournalRenderItem[]; nextOrdinal: number | null } {
const items: AgentJournalRenderItem[] = []
let used = 0
let lastKept = -1
for (const row of rows) {
const json = String(row.payload_json)
if (items.length > 0 && used + json.length > budget) {
const { bytes, item } = read(row)
if (items.length > 0 && used + bytes > budget) {
return { items, nextOrdinal: lastKept }
}
used += json.length
used += bytes
lastKept = Number(row.ordinal)
items.push(JSON.parse(json))
items.push(item())
}
return { items, nextOrdinal }
}
/** Payload by primary key, so a page cut by the budget looks up no further rows. */
export const ITEMS_BY_KIND_PAYLOAD_SQL = `SELECT payload_json,ordinal FROM journal_items
export const ITEMS_BY_KIND_PAYLOAD_SQL = `SELECT payload_json,ordinal,output_blob FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=?`
/** Each chosen item's row, in order, read only as the page budget asks for it. */
@@ -45,3 +56,19 @@ export function* itemsByKindPayloads(
}
}
}
/** One budgeted page of the chosen items, stored outputs restored and counted at full size. */
export function readItemsByKindPage(
db: Database.Database,
prefix: readonly [sessionId: string, epoch: string],
chosen: readonly SqliteRow[],
nextOrdinal: number | null
): { items: AgentJournalRenderItem[]; nextOrdinal: number | null } {
const payload = db.prepare(ITEMS_BY_KIND_PAYLOAD_SQL)
return budgetItemsByKindPage(
itemsByKindPayloads(chosen, (itemId) => payload.get(...prefix, itemId)),
nextOrdinal,
undefined,
(row) => measureJournalStoredRow(db, prefix, row)
)
}
@@ -3,10 +3,13 @@ import type { JournalSqlOwner } from './journal-sql-owner'
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
import type { JournalSqlConversions } from './journal-sql-conversions'
import type { JournalSqlActivity } from './journal-sql-activity'
import type { JournalSqlMaintenance } from './journal-sql-maintenance'
import type { JournalPagedMaintenanceKind, JournalSqlMaintenance } from './journal-sql-maintenance'
import { JOURNAL_STORED_OUTPUT_WRITES } from './journal-stored-output'
const MINUTE = 60_000
const COMPACTION_INTERVAL = 6 * 60 * MINUTE
// About 8k saved outputs a minute at the shared 10% duty cycle; none while writes are off.
const STORED_OUTPUT_PAGES_PER_TICK = JOURNAL_STORED_OUTPUT_WRITES ? 256 : 0
/** Idle conversion and periodic pruning share one owner; activity never blocks retired cleanup. */
export class JournalSqlBackgroundMaintenance {
@@ -141,6 +144,7 @@ export class JournalSqlBackgroundMaintenance {
}
}
await this.pages({ kind: 'page-reclaim' }, this.reclaim)
await this.pages({ kind: 'stored-output-compaction' }, STORED_OUTPUT_PAGES_PER_TICK)
} catch (error) {
this.report(error)
}
@@ -215,10 +219,7 @@ export class JournalSqlBackgroundMaintenance {
}
private async pages(
operation: Extract<
JournalSqlMaintenance,
{ kind: 'event-tail-prune' | 'domain-retired-prune' | 'page-reclaim' | 'chat-discard' }
>,
operation: Extract<JournalSqlMaintenance, { kind: JournalPagedMaintenanceKind }>,
limit = Infinity
): Promise<void> {
for (let unit = 0; unit < limit && !this.stopped.signal.aborted; unit++) {
@@ -7,6 +7,7 @@ import type { AgentJournalCursor } from '../../../shared/agent-session/journal/a
import type { JournalContinuationContext } from './journal-reads'
import { journalContextEpoch } from './context-epochs/journal-context-epoch'
import type { SqliteBindings, SqliteRow } from '../../sqlite/sqlite-statement'
import { readJournalStoredRow } from './journal-stored-output-rows'
const PAGE = 256
@@ -45,14 +46,14 @@ export function readJournalDomainContinuationContext(
let afterItem: SqliteBindings = []
do {
const rows = db
.prepare(`SELECT payload_json, sequence, sequence_index, ordinal
.prepare(`SELECT payload_json, output_blob, sequence, sequence_index, ordinal
FROM journal_items INDEXED BY journal_items_visible_position
WHERE session_id=? AND epoch=? AND hidden=0 AND sequence>?
${afterItem.length ? 'AND (sequence, sequence_index, ordinal)>(?, ?, ?)' : ''}
ORDER BY sequence, sequence_index, ordinal LIMIT ?`)
.all(...prefix, clear, ...afterItem, PAGE)
for (const row of rows) {
items.push(JSON.parse(String(row.payload_json)))
items.push(readJournalStoredRow(db, prefix, row))
}
afterItem = nextPage(rows, ['sequence', 'sequence_index', 'ordinal'])
} while (afterItem.length)
@@ -1,3 +1,4 @@
import { readJournalStoredItem } from './journal-stored-output-rows'
import type Database from '../../sqlite/sync-database'
import {
readJournalHistoryItems,
@@ -7,11 +8,7 @@ import {
type JournalHistoryPrompt
} from './journal-history-items'
import { journalDomainHistoryOpeners, journalDomainUserItemIds } from './journal-sql-history-items'
import {
budgetItemsByKindPage,
itemsByKindPayloads,
ITEMS_BY_KIND_PAYLOAD_SQL
} from './journal-items-by-kind-page'
import { readItemsByKindPage } from './journal-items-by-kind-page'
import type {
AgentJournalItemBody,
AgentJournalRenderItem
@@ -191,10 +188,8 @@ export function readJournalDomainItems(
)
.sort((a, b) => Number(a.ordinal) - Number(b.ordinal))
const chosen = headers.slice(0, operation.limit)
const payload = db.prepare(ITEMS_BY_KIND_PAYLOAD_SQL)
const page = itemsByKindPayloads(chosen, (itemId) => payload.get(...prefix, itemId))
const next = headers.length > operation.limit ? Number(chosen.at(-1)!.ordinal) : null
return { kind: 'items-by-kind', ...budgetItemsByKindPage(page, next) }
return { kind: 'items-by-kind', ...readItemsByKindPage(db, prefix, chosen, next) }
}
case 'context-floor': {
const row = db
@@ -212,7 +207,10 @@ export function readJournalDomainItems(
const row = db.prepare(`${TURNS_NEWEST_FIRST} LIMIT 1`).get(...prefix)
return {
kind: 'running-turn',
running: row ? runningStructuredAgentSessionTurnItemBySequence([parseItem(row)]) : null
// A turn row never stores tool output.
running: row
? runningStructuredAgentSessionTurnItemBySequence([JSON.parse(String(row.payload_json))])
: null
}
}
case 'context-usage':
@@ -250,7 +248,7 @@ function visibleItem(
.prepare(`SELECT payload_json FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`)
.get(...prefix, itemId)
return row ? parseItem(row) : null
return row ? readJournalStoredItem(db, prefix, row.payload_json) : null
}
/** The input order's newest turn: latest position, then the greatest item ID among ties. */
@@ -264,7 +262,7 @@ export function newestJournalDomainTurnOrClear(
WHERE session_id=? AND epoch=? AND hidden=0 AND turn_marker=1
ORDER BY sequence DESC, sequence_index DESC`)
.iterate(...prefix)) {
const item = parseItem(row)
const item = readJournalStoredItem(db, prefix, row.payload_json)
if (
newest &&
(item.sequence !== newest.sequence ||
@@ -285,7 +283,7 @@ function readContextUsage(db: Database.Database, prefix: Prefix): AgentSessionCo
let used: number | null = null
let window: number | null = null
for (const row of db.prepare(TURNS_NEWEST_FIRST).iterate(...prefix)) {
const item = parseItem(row)
const item = readJournalStoredItem(db, prefix, row.payload_json)
if (used !== null && window !== null && item.sequence < Math.min(used, window)) {
break
}
@@ -307,13 +305,9 @@ export function readJournalSqlGoalRows(db: Database.Database, prefix: Prefix) {
WHERE session_id=? AND epoch=? AND hidden=0 AND goal_transition=1
ORDER BY sequence DESC, ordinal ASC LIMIT 1`)
.get(...prefix)
return row ? [parseItem(row)] : []
return row ? [readJournalStoredItem(db, prefix, row.payload_json)] : []
}
function readThreadGoal(db: Database.Database, prefix: Prefix): JournalThreadGoalSource {
return journalThreadGoalSource(readJournalSqlGoalRows(db, prefix))
}
function parseItem(row: Record<string, unknown>): AgentJournalRenderItem {
return JSON.parse(String(row.payload_json))
}
@@ -11,6 +11,7 @@ import type {
JournalOutgrownChildRead,
JournalRestartProgress
} from './journal-lifecycle-reads'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type JournalSqlLifecycleRead =
| { kind: 'latest-prompt' }
@@ -37,7 +38,9 @@ export function readJournalDomainLifecycle(
.get(...prefix)
return {
kind: 'latest-prompt',
prompt: row ? latestStructuredAgentSessionPrompt([JSON.parse(String(row.payload_json))]) : ''
prompt: row
? latestStructuredAgentSessionPrompt([readJournalStoredItem(db, prefix, row.payload_json)])
: ''
}
}
if (operation.kind === 'restart-progress') {
@@ -74,7 +77,7 @@ export function readJournalDomainLifecycle(
FROM journal_items WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`)
.get(...prefix, agentJournalSubmissionKey(nextClientMessageId))
const nextItem: AgentJournalRenderItem | null = next
? JSON.parse(String(next.payload_json))
? readJournalStoredItem(db, prefix, next.payload_json)
: null
const body = nextItem?.body
return {
@@ -17,6 +17,7 @@ import {
type JournalDomainMaintenance,
type JournalDomainMaintenanceResult
} from './journal-domain-maintenance'
import { compactJournalStoredOutputPage } from './journal-stored-output-compaction'
import { pruneJournalSqlEventTail, type JournalSqlEventTailResult } from './journal-sql-event-tail'
import {
discardJournalDomainImportPage,
@@ -26,6 +27,14 @@ import { discardForgottenJournalChatPage } from './journal-forgotten-chats'
import { installAgentSessionStoreRows } from '../../runtime/agent-session/records/agent-session-record-rows'
import type { JournalSqlStoredRecords } from './journal-sql-database-read'
/** Operations the background pass runs a bounded page at a time. */
export type JournalPagedMaintenanceKind =
| 'event-tail-prune'
| 'domain-retired-prune'
| 'page-reclaim'
| 'chat-discard'
| 'stored-output-compaction'
export type JournalSqlMaintenance =
| { kind: 'records-install' }
| { kind: 'wsl-workspace-rekey'; worktreeIds: readonly (readonly [string, string])[] }
@@ -99,5 +108,7 @@ export function applyJournalSqlMaintenance(
return completeJournalDomainCompaction(db, operation.now)
case 'event-tail-prune':
return pruneJournalSqlEventTail(db, operation.now)
case 'stored-output-compaction':
return compactJournalStoredOutputPage(db)
}
}
@@ -15,6 +15,7 @@ import {
import { staleSubagentRosterRevisions } from './item-lifecycle/journal-subagent-liveness'
import { applyJournalDomainLifecycle } from './journal-domain-lifecycle'
import { withdrawOpeningDerivedQueue } from './journal-open-queue-withdrawal'
import { readJournalStoredItem } from './journal-stored-output-rows'
export type JournalOpenRecoveryResult = {
failures: {
@@ -140,7 +141,9 @@ function settleOpeningLiveWork(
AND (lifecycle_state IN ('running','pending') OR live_work=1) AND live_work=1
${after ? 'AND (ordinal,item_id)>(?,?)' : ''} ORDER BY ordinal,item_id LIMIT 128`)
.all(target.sessionId, target.epoch, ...(after ?? []))
const items: AgentJournalRenderItem[] = rows.map((row) => JSON.parse(String(row.payload_json)))
const items: AgentJournalRenderItem[] = rows.map((row) =>
readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json)
)
const revisions = staleSubagentRosterRevisions(items)
if (revisions.length > 0) {
const head = db
@@ -1,5 +1,7 @@
import type Database from '../../sqlite/sync-database'
import type { SqliteRow } from '../../sqlite/sqlite-statement'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { hydrateJournalStoredItem } from './journal-stored-output-rows'
export type JournalSqlReadWork = { rows: number; decodedBytes: number }
export type JournalSqlIndexValues = {
@@ -33,7 +35,8 @@ export class JournalSqlScope {
throw new Error('Invalid journal projection value')
}
this.work.decodedBytes += Buffer.byteLength(json)
return this.indexValues ? this.indexValues.decode<T>(json) : JSON.parse(json)
const value: T = this.indexValues ? this.indexValues.decode<T>(json) : JSON.parse(json)
return isJournalItemShape(value) ? hydrateJournalStoredItem(this.db, this.prefix, value) : value
}
/** A projection row's own payload; never an index value pointer. */
@@ -72,3 +75,17 @@ export class JournalSqlScope {
)
}
}
function isJournalItemShape<T>(value: T): value is T & AgentJournalRenderItem {
return (
typeof value === 'object' &&
value !== null &&
'itemId' in value &&
typeof value.itemId === 'string' &&
'body' in value &&
typeof value.body === 'object' &&
value.body !== null &&
'kind' in value.body &&
typeof value.body.kind === 'string'
)
}
@@ -0,0 +1,116 @@
import type Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract'
import { journalStoredItemRow } from './journal-stored-output-rows'
import { JOURNAL_DOMAIN_ITEM_COLUMNS } from './journal-domain-item-read-columns'
import {
hydrateJournalItemOutput,
JOURNAL_STORED_OUTPUT_WRITES,
MAX_STORED_TEXT_BYTES
} from './journal-stored-output'
/** The last rowid examined; rows appended later are picked up, rows behind it are never rescanned. */
export const JOURNAL_STORED_OUTPUT_CURSOR = 'journal.stored-output.cursor'
// Each page is one writer transaction: bounded rows scanned, rows rewritten, bytes and time.
const SCAN_ROWS = 512
const PAGE_ROWS = 32
const PAGE_BYTES = 512 * 1024
const PAGE_MS = 5
const MIN_ROW_BYTES = 512
const WIRE_BYTES = JOURNAL_DOMAIN_ITEM_COLUMNS.indexOf('wire_bytes')
export type JournalStoredOutputCompaction = { kind: 'stored-output-compaction'; more: boolean }
function payloadText(item: AgentJournalRenderItem): string | undefined {
const body = item.body
return body.kind === 'tool-call'
? body.output?.head
: body.kind === 'diff'
? body.patch.head
: undefined
}
function compactRow(db: Database.Database, rowid: number, writes: boolean): void {
const row = db
.prepare('SELECT session_id, epoch, payload_json, wire_bytes FROM journal_items WHERE rowid=?')
.get(rowid)!
const payload = String(row.payload_json)
const item: AgentJournalRenderItem = JSON.parse(payload)
const values: (number | null)[] = JOURNAL_DOMAIN_ITEM_COLUMNS.map(() => null)
values[WIRE_BYTES] = row.wire_bytes === null ? null : Number(row.wire_bytes)
const prefix = [String(row.session_id), String(row.epoch)] as const
const stored = journalStoredItemRow(db, prefix, item, payload, values, writes)
if (!stored.blob) {
return
}
const restored = hydrateJournalItemOutput(JSON.parse(stored.payload), stored.blob)
if (payloadText(restored) !== payloadText(item)) {
throw new Error('stored tool output did not read back as written')
}
db.prepare(
'UPDATE journal_items SET payload_json=?, wire_bytes=?, output_blob=? WHERE rowid=?'
).run(stored.payload, stored.values[WIRE_BYTES] ?? null, stored.blob, rowid)
}
/** Moves saved tool output and diff text written inline into compressed row blobs, oldest rows first. */
export function compactJournalStoredOutputPage(
db: Database.Database,
writes = JOURNAL_STORED_OUTPUT_WRITES
): JournalStoredOutputCompaction {
if (!writes) {
return { kind: 'stored-output-compaction', more: false }
}
const started = performance.now()
const saved = db
.prepare('SELECT value FROM agent_session_store_meta WHERE key=?')
.get(JOURNAL_STORED_OUTPUT_CURSOR)
let cursor = saved ? Number(saved.value) : 0
// Lengths and NULL tests come from record headers, so skipped rows never load their text. Headers
// are read lazily: a page cut short by its budget never fetches the rows the next page starts at.
const headers = db
.prepare(`SELECT rowid, kind, wire_version, output_blob IS NULL AS inline,
octet_length(payload_json) AS bytes FROM journal_items WHERE rowid>? ORDER BY rowid LIMIT ?`)
.iterate(cursor, SCAN_ROWS)
let scanned = 0
let rows = 0
let bytes = 0
let more = false
for (const header of headers) {
const size = Number(header.bytes)
if (
rows >= PAGE_ROWS ||
(rows > 0 && (bytes + size > PAGE_BYTES || performance.now() - started >= PAGE_MS))
) {
more = true
break
}
scanned++
cursor = Number(header.rowid)
if (
(header.kind !== 'tool-call' && header.kind !== 'diff') ||
header.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION ||
!header.inline ||
size < MIN_ROW_BYTES ||
// Judged from the record header: a row this large is never loaded inside a page.
size > MAX_STORED_TEXT_BYTES * 2
) {
continue
}
rows++
bytes += size
try {
compactRow(db, cursor, writes)
} catch (error) {
// A row that can't be stored exactly stays inline; rethrowing would retry this page forever.
console.warn('[agent-session-journal] kept a saved tool output inline', error)
}
}
more ||= scanned === SCAN_ROWS
db.prepare(`INSERT INTO agent_session_store_meta(key,value) VALUES(?,?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(
JOURNAL_STORED_OUTPUT_CURSOR,
String(cursor)
)
return { kind: 'stored-output-compaction', more }
}
@@ -0,0 +1,120 @@
import type Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { JOURNAL_DOMAIN_ITEM_COLUMNS } from './journal-domain-item-read-columns'
import type { ItemsByKindRowRead } from './journal-items-by-kind-page'
import {
hasJournalStoredOutput,
hydrateJournalItemOutput,
journalItemOutputPreview,
journalStoredTextBytes,
JOURNAL_STORED_OUTPUT_WRITES,
storeJournalItemOutput
} from './journal-stored-output'
type Prefix = readonly [sessionId: string, epoch: string]
const WIRE_BYTES = JOURNAL_DOMAIN_ITEM_COLUMNS.indexOf('wire_bytes')
/** The one parse of a journal_items item column. */
export function readJournalStoredItem<T extends AgentJournalRenderItem = AgentJournalRenderItem>(
db: Database.Database,
prefix: Prefix,
json: unknown,
/** The row's `output_blob` when the read already selected it, saving a lookup per row. */
blob?: unknown
): T {
const item: T = JSON.parse(String(json))
return hydrateJournalStoredItem(db, prefix, item, blob)
}
/** A row that selected `payload_json` and, to save a lookup, `output_blob`. */
export function readJournalStoredRow(
db: Database.Database,
prefix: Prefix,
row: { payload_json?: unknown; output_blob?: unknown }
): AgentJournalRenderItem {
// A row that did not select the blob reads it by key, never as missing.
return readJournalStoredItem(db, prefix, row.payload_json, row.output_blob)
}
/** A page row's size counts its stored text, which is read only if the row joins the page. */
export function measureJournalStoredRow(
db: Database.Database,
prefix: Prefix,
row: { payload_json?: unknown; output_blob?: unknown }
): ItemsByKindRowRead {
const json = String(row.payload_json)
if (!(row.output_blob instanceof Uint8Array)) {
return { bytes: json.length, item: () => JSON.parse(json) }
}
const parsed: AgentJournalRenderItem = JSON.parse(json)
return {
bytes: json.length + journalStoredTextBytes(parsed),
item: () => hydrateJournalStoredItem(db, prefix, parsed, row.output_blob)
}
}
/** Returns the same object when the item stores nothing, so callers can reuse its row text. */
export function hydrateJournalStoredItem<T extends AgentJournalRenderItem>(
db: Database.Database,
prefix: Prefix,
item: T,
blob?: unknown
): T {
if (!hasJournalStoredOutput(item)) {
return item
}
const held =
blob === undefined
? db
.prepare(
'SELECT output_blob FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?'
)
.get(...prefix, item.itemId)?.output_blob
: blob
return hydrateJournalItemOutput(item, held instanceof Uint8Array ? held : null)
}
export type JournalStoredItemRow<V> = {
payload: string
values: V[]
blob: Buffer | null
/** The row's own blob holds this item's text (an item whose stored output could not be read). */
keep: boolean
}
/** Row values for one item write: stored text moves to the blob; client-form columns stay as computed. */
export function journalStoredItemRow<V>(
db: Database.Database,
prefix: Prefix,
item: AgentJournalRenderItem,
payload: string,
values: readonly V[],
writes = JOURNAL_STORED_OUTPUT_WRITES
): JournalStoredItemRow<V | number> {
if (hasJournalStoredOutput(item)) {
const held = db
.prepare(`SELECT 1 FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=? AND output_blob IS NOT NULL`)
.get(...prefix, item.itemId)
if (held) {
return { payload, values: [...values], blob: null, keep: true }
}
// A fork or copy of an output that could not be read keeps its preview rather than failing.
console.warn(`[agent-session-journal] kept only the preview of a saved output in ${prefix[0]}`)
return {
payload: JSON.stringify(journalItemOutputPreview(item)),
values: [...values],
blob: null,
keep: false
}
}
const stored = writes ? storeJournalItemOutput(item) : null
if (!stored) {
return { payload, values: [...values], blob: null, keep: false }
}
const next: (V | number)[] = [...values]
// A null client size means "the stored payload's length", which the stored form no longer is.
next[WIRE_BYTES] ??= Buffer.byteLength(payload, 'utf8')
return { payload: JSON.stringify(stored.item), values: next, blob: stored.blob, keep: false }
}
@@ -0,0 +1,164 @@
import { afterEach, expect, it, vi } from 'vitest'
import Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { migrateJournalDomainSchema } from './journal-domain-schema'
import { writeJournalDomainCurrentItem } from './journal-domain-current-item'
import { runJournalTransaction } from './database/journal-database'
import { readJournalDomainItems } from './journal-sql-item-reads'
import {
readJournalDomainClientForms,
readJournalDomainItemDetail
} from './journal-domain-item-presentation'
import { compactJournalStoredOutputPage } from './journal-stored-output-compaction'
import { measureJournalStoredRow } from './journal-stored-output-rows'
import type * as StoredOutput from './journal-stored-output'
const switches = vi.hoisted(() => ({ writes: true }))
vi.mock('./journal-stored-output', async (load) => {
const actual = await load<typeof StoredOutput>()
return {
...actual,
get JOURNAL_STORED_OUTPUT_WRITES() {
return switches.writes
}
}
})
const target = { sessionId: 'chat', epoch: 'epoch' }
// Past the 16 KiB client head, with text UTF-8 cannot carry unescaped (a lone surrogate).
const OUTPUT = `${'a'.repeat(20_000)}é\ud800 tail ${'line\n'.repeat(4_000)}`
const databases: Database.Database[] = []
afterEach(() => {
switches.writes = true
for (const db of databases.splice(0)) {
db.close()
}
})
function chat() {
const db = new Database(':memory:')
databases.push(db)
migrateJournalDomainSchema(db)
db.prepare(
`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace',?,?,900,1)`
).run(target.sessionId, target.epoch)
return db
}
function toolCall(
itemId: string,
state: 'running' | 'completed',
revision = 1
): AgentJournalRenderItem {
return {
itemId,
revision,
sequence: 10,
observedAt: 1,
body: {
kind: 'tool-call',
name: 'shell',
callId: itemId,
input: { command: 'cat big.log' },
state,
output: { head: OUTPUT, byteLength: 40_011, digest: 'd', truncated: false }
}
}
}
function write(db: Database.Database, item: AgentJournalRenderItem) {
runJournalTransaction(db, () =>
writeJournalDomainCurrentItem(db, target, { item, hidden: false, ownerFence: 1 })
)
}
function stored(db: Database.Database, itemId: string) {
return db
.prepare(`SELECT output_blob IS NOT NULL AS blob FROM journal_items WHERE item_id=?`)
.get(itemId)
}
function readTool(db: Database.Database) {
const read = readJournalDomainItems(db, target, {
kind: 'items-by-kind',
kinds: ['tool-call'],
afterOrdinal: 0,
limit: 10
})
return read.kind === 'items-by-kind' ? read.items : []
}
it('reads a saved tool output back whole, as the bounded client head, through rewrites and with writes off', () => {
const db = chat()
write(db, toolCall('tool', 'completed'))
expect(stored(db, 'tool')!.blob).toBe(1)
// A tool-output page counts a stored row at its full size, not its preview's.
const row = db.prepare(`SELECT payload_json, output_blob FROM journal_items`).get()!
expect(measureJournalStoredRow(db, [target.sessionId, target.epoch], row).bytes).toBeGreaterThan(
40_011
)
const [whole] = readTool(db)
expect(whole!.body).toMatchObject({ output: { head: OUTPUT, truncated: false } })
expect(readJournalDomainItemDetail(db, target, 'tool')!.body).toMatchObject({
output: { head: OUTPUT, truncated: false }
})
const client = readJournalDomainClientForms(db, target, ['tool']).get('tool')!
expect(client.item.body).toEqual({
kind: 'tool-call',
name: 'shell',
callId: 'tool',
input: { command: 'cat big.log' },
state: 'completed',
output: { head: 'a'.repeat(16_384), byteLength: 40_011, digest: 'd', truncated: true }
})
expect(client.bytes).toBeGreaterThan(16_384)
// A later revision written from the hydrated read keeps every byte.
write(db, { ...whole!, revision: 2 })
// A row form that never hydrated keeps this row's blob; it cannot name another row's.
const raw: AgentJournalRenderItem = JSON.parse(
String(
db.prepare(`SELECT payload_json FROM journal_items WHERE item_id='tool'`).get()!.payload_json
)
)
write(db, { ...raw, revision: 3 })
expect(readTool(db)[0]!.body).toMatchObject({ output: { head: OUTPUT, truncated: false } })
expect(readJournalDomainClientForms(db, target, ['tool']).get('tool')!.bytes).toBe(client.bytes)
// A stored form copied to a row without its blob keeps the preview as a plain truncated output.
write(db, { ...raw, itemId: 'other', sequence: 11 })
const copy = readJournalDomainItemDetail(db, target, 'other')!.body
expect(copy.kind === 'tool-call' && copy.output).toEqual({
head: `${OUTPUT.slice(0, 512)}\n…\n${OUTPUT.slice(-512)}`,
byteLength: 40_011,
digest: 'd',
truncated: true
})
// With writes off, a rewrite stores the whole text inline again.
switches.writes = false
const tool = () => readTool(db).find((item) => item.itemId === 'tool')!
write(db, { ...tool(), revision: 4 })
expect(stored(db, 'tool')!.blob).toBe(0)
expect(tool().body).toMatchObject({ output: { head: OUTPUT, truncated: false } })
})
it('compacts inline rows written before the switch, leaving running calls inline', () => {
const db = chat()
switches.writes = false
write(db, toolCall('done', 'completed'))
write(db, { ...toolCall('live', 'running'), sequence: 11 })
expect(stored(db, 'done')!.blob).toBe(0)
switches.writes = true
while (compactJournalStoredOutputPage(db).more) {
// Pages until the pass reaches the last row.
}
expect(stored(db, 'done')!.blob).toBe(1)
expect(stored(db, 'live')!.blob).toBe(0)
expect(readTool(db).map((item) => item.body)).toMatchObject([
{ state: 'completed', output: { head: OUTPUT, truncated: false } },
{ state: 'running' }
])
})
@@ -0,0 +1,159 @@
import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib'
import type {
AgentJournalBoundedPayload,
AgentJournalItemBody,
AgentJournalRenderItem
} from '../../../shared/agent-session/journal/agent-session-journal-types'
/** Off until builds that cannot read stored outputs are out of the downgrade window; reads always work. */
export const JOURNAL_STORED_OUTPUT_WRITES = false
// Format 1: one zstd frame over the text as a JSON string literal, so lone surrogates survive.
const FORMAT_ZSTD_JSON = 1
const MIN_STORED_TEXT_BYTES = 512
// Encoding and the read-back check run inside the writer transaction; larger text stays inline.
export const MAX_STORED_TEXT_BYTES = 2 * 1024 * 1024
// Builds that predate stored outputs show this inline preview, as does this one if a blob is unreadable.
const PREVIEW_HEAD_CHARS = 512
const PREVIEW_TAIL_CHARS = 512
const ZSTD_PARAMS = {
[constants.ZSTD_c_compressionLevel]: 3,
[constants.ZSTD_c_checksumFlag]: 1
}
type StoredMarker = { truncated: boolean }
type StoredPayload = AgentJournalBoundedPayload & { stored: StoredMarker }
function bodyPayload(body: AgentJournalItemBody): AgentJournalBoundedPayload | undefined {
return body.kind === 'tool-call' ? body.output : body.kind === 'diff' ? body.patch : undefined
}
function withPayload<T extends AgentJournalRenderItem>(
item: T,
payload: AgentJournalBoundedPayload
): T {
const body =
item.body.kind === 'tool-call'
? { ...item.body, output: payload }
: item.body.kind === 'diff'
? { ...item.body, patch: payload }
: item.body
return { ...item, body }
}
function storedMarker(payload: AgentJournalBoundedPayload | undefined): StoredMarker | null {
const stored: unknown = payload && 'stored' in payload ? payload.stored : undefined
return typeof stored === 'object' &&
stored !== null &&
'truncated' in stored &&
typeof stored.truncated === 'boolean'
? { truncated: stored.truncated }
: null
}
export function hasJournalStoredOutput(item: AgentJournalRenderItem): boolean {
return storedMarker(bodyPayload(item.body)) !== null
}
/** The recorded length of the text an item stores outside its row, 0 when it stores none. */
export function journalStoredTextBytes(item: AgentJournalRenderItem): number {
const payload = bodyPayload(item.body)
return payload && storedMarker(payload) ? payload.byteLength : 0
}
/** The row form and its blob, or null when this item's text stays inline. */
export function storeJournalItemOutput<T extends AgentJournalRenderItem>(
item: T
): { item: T; blob: Buffer } | null {
const payload = bodyPayload(item.body)
if (
!payload ||
(item.body.kind === 'tool-call' && item.body.state === 'running') ||
storedMarker(payload) ||
payload.head.length * 3 < MIN_STORED_TEXT_BYTES ||
payload.head.length > MAX_STORED_TEXT_BYTES
) {
return null
}
const textBytes = Buffer.byteLength(payload.head, 'utf8')
// Reads bound and check decoding by the recorded length, so a text that disagrees stays inline.
if (
textBytes < MIN_STORED_TEXT_BYTES ||
textBytes > MAX_STORED_TEXT_BYTES ||
textBytes > payload.byteLength ||
(!payload.truncated && textBytes !== payload.byteLength)
) {
return null
}
const text = payload.head
if (text.length <= PREVIEW_HEAD_CHARS + PREVIEW_TAIL_CHARS + MIN_STORED_TEXT_BYTES) {
return null
}
const preview =
PREVIEW_TAIL_CHARS > 0
? `${text.slice(0, PREVIEW_HEAD_CHARS)}\n…\n${text.slice(-PREVIEW_TAIL_CHARS)}`
: text.slice(0, PREVIEW_HEAD_CHARS)
const frame = zstdCompressSync(Buffer.from(JSON.stringify(text)), { params: ZSTD_PARAMS })
// Small or incompressible text costs more as a blob plus preview than inline.
if (frame.length + 1 + Buffer.byteLength(preview, 'utf8') > textBytes * 0.8) {
return null
}
const blob = Buffer.allocUnsafe(frame.length + 1)
blob.writeUInt8(FORMAT_ZSTD_JSON, 0)
frame.copy(blob, 1)
const stored: StoredPayload = {
head: preview,
byteLength: payload.byteLength,
digest: payload.digest,
truncated: true,
stored: { truncated: payload.truncated }
}
return { item: withPayload(item, stored), blob }
}
/** The inline preview as a plain truncated payload, for a stored form whose row holds no blob. */
export function journalItemOutputPreview<T extends AgentJournalRenderItem>(item: T): T {
const payload = bodyPayload(item.body)
if (!payload || !storedMarker(payload)) {
return item
}
const { head, byteLength, digest } = payload
return withPayload(item, { head, byteLength, digest, truncated: true })
}
/** Unreadable blobs keep the marker, so a rewrite keeps the blob rather than an empty head. */
export function hydrateJournalItemOutput<T extends AgentJournalRenderItem>(
item: T,
blob: Uint8Array | null | undefined
): T {
const payload = bodyPayload(item.body)
const marker = storedMarker(payload)
if (!payload || !marker) {
return item
}
try {
if (!blob || blob.length < 2 || blob[0] !== FORMAT_ZSTD_JSON) {
throw new Error('stored output blob is missing or unknown')
}
// JSON escapes spend at most six bytes per UTF-8 byte; the bound caps inflation.
const json = zstdDecompressSync(blob.subarray(1), {
maxOutputLength: payload.byteLength * 6 + 64
})
const text: unknown = JSON.parse(json.toString('utf8'))
if (
typeof text !== 'string' ||
(!marker.truncated && Buffer.byteLength(text, 'utf8') !== payload.byteLength)
) {
throw new Error('stored output does not match its item')
}
return withPayload(item, {
head: text,
byteLength: payload.byteLength,
digest: payload.digest,
truncated: marker.truncated
})
} catch (error) {
console.warn('[agent-session-journal] a saved tool output could not be read', error)
return item
}
}