diff --git a/config/main-blocking-compression-baseline.txt b/config/main-blocking-compression-baseline.txt index 9b41ba5a702..cc52cbd6728 100644 --- a/config/main-blocking-compression-baseline.txt +++ b/config/main-blocking-compression-baseline.txt @@ -1,6 +1,7 @@ # Legacy synchronous compression uses per file; counts may only go DOWN. # Only explicit worker/utility entry files are exempt; their imports remain guarded. 1 src/main/browser/extensions/actions/browser-extension-action-icon.ts +2 src/main/native-chat/agent-session-journal/journal-stored-output.ts 1 src/main/runtime/rpc/mobile/e2ee/mobile-e2ee-v2-compressed-frame.ts 1 src/main/ssh/remote-exec/ssh-posix-command-wrapper.ts 2 src/main/ssh/remote-exec/ssh-remote-powershell.ts diff --git a/config/main-blocking-sqlite-baseline.txt b/config/main-blocking-sqlite-baseline.txt index 9ed1ca4d8fb..84159eff065 100644 --- a/config/main-blocking-sqlite-baseline.txt +++ b/config/main-blocking-sqlite-baseline.txt @@ -38,7 +38,7 @@ 3 src/main/native-chat/agent-session-journal/database/journal-checkpoint-owner.ts 13 src/main/native-chat/agent-session-journal/database/journal-database.ts 7 src/main/native-chat/agent-session-journal/database/journal-domain-database.ts -2 src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts +5 src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts 20 src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts 1 src/main/native-chat/agent-session-journal/database/journal-host-database.ts 32 src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts @@ -103,6 +103,7 @@ 10 src/main/native-chat/agent-session-journal/journal-fork-stage-page.ts 12 src/main/native-chat/agent-session-journal/journal-fork-stages.ts 4 src/main/native-chat/agent-session-journal/journal-fork-workspace.ts +2 src/main/native-chat/agent-session-journal/journal-items-by-kind-page.ts 2 src/main/native-chat/agent-session-journal/journal-open-queue-withdrawal.ts 4 src/main/native-chat/agent-session-journal/journal-sql-accepted-inputs.ts 6 src/main/native-chat/agent-session-journal/journal-sql-automatic-submission-command.ts @@ -114,7 +115,7 @@ 2 src/main/native-chat/agent-session-journal/journal-sql-edit-lease-command.ts 12 src/main/native-chat/agent-session-journal/journal-sql-event-tail.ts 4 src/main/native-chat/agent-session-journal/journal-sql-history-items.ts -24 src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts +22 src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts 10 src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts 5 src/main/native-chat/agent-session-journal/journal-sql-message-commit.ts 12 src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts @@ -123,6 +124,8 @@ 10 src/main/native-chat/agent-session-journal/journal-sql-rewind-command.ts 4 src/main/native-chat/agent-session-journal/journal-sql-send-admission.ts 7 src/main/native-chat/agent-session-journal/journal-sql-steps.ts +10 src/main/native-chat/agent-session-journal/journal-stored-output-compaction.ts +4 src/main/native-chat/agent-session-journal/journal-stored-output-rows.ts 4 src/main/native-chat/agent-session-journal/journal-writer-effects.ts 4 src/main/native-chat/agent-session-journal/journal-writer-operation-expiry.ts 12 src/main/native-chat/agent-session-journal/journal-writer-store-mutation.ts diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts index 2ac7c3b4580..1324ba6fc49 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts @@ -20,4 +20,15 @@ export function migrateJournalDomainSchema(db: Database.Database): void { db.pragma(`user_version = ${JOURNAL_DOMAIN_SCHEMA_VERSION}`) }) } + ensureJournalStoredOutputColumn(db) +} + +/** Added without a user_version change: released builds name their columns and never read it. */ +function ensureJournalStoredOutputColumn(db: Database.Database): void { + const present = db + .prepare(`SELECT 1 FROM pragma_table_info('journal_items') WHERE name='output_blob'`) + .get() + if (!present) { + db.exec('ALTER TABLE journal_items ADD COLUMN output_blob BLOB') + } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-context-replacement.ts b/src/main/native-chat/agent-session-journal/journal-domain-context-replacement.ts index d0910529399..1a47b2680b3 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-context-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-context-replacement.ts @@ -32,6 +32,7 @@ import { assertJournalFence } from './writing/journal-write-guards' import { queueClearRestatement, writeJournalDomainQueueClear } from './journal-domain-queue-clear' import { restoreRetainedFailures, retainedFailures } from './journal-domain-retained-failures' import { invalidateJournalDomainQueue } from './journal-domain-queue-revision' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } @@ -200,7 +201,7 @@ export function replaceJournalDomainContext( const itemId = itemIds[index]! const stored = current.get(...prefix, itemId) const previous: AgentJournalRenderItem | undefined = stored - ? JSON.parse(String(stored.payload_json)) + ? readJournalStoredItem(db, prefix, stored.payload_json) : undefined const tombstone = db .prepare( diff --git a/src/main/native-chat/agent-session-journal/journal-domain-control-command.ts b/src/main/native-chat/agent-session-journal/journal-domain-control-command.ts index 1f4eb27f953..265840d0dd3 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-control-command.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-control-command.ts @@ -35,6 +35,7 @@ import { import { appendJournalSqlEvent } from './journal-sql-event-tail' import { restoreJournalSqlFailure } from './journal-sql-failure' import { invalidateJournalDomainQueue } from './journal-domain-queue-revision' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } @@ -299,7 +300,7 @@ function visibleItem( WHERE session_id=? AND epoch=? AND hidden=0 AND item_id=COALESCE((SELECT item_id FROM journal_item_aliases WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`) .get(target.sessionId, target.epoch, target.sessionId, target.epoch, key, key) - return row ? JSON.parse(String(row.payload_json)) : null + return row ? readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json) : null } function physicalCoordinate(position: JournalDomainInputCoordinate): JournalWriterCoordinate { diff --git a/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts b/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts index df58a2423dd..9c91009c07e 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts @@ -27,6 +27,11 @@ import { import { invalidateJournalDomainQueue } from './journal-domain-queue-revision' import { readAgentJournalItemBody } from '../../../shared/agent-session/journal/agent-session-journal-body-admission' import { AgentSessionJournalError } from './writing/journal-write-guards' +import { + hydrateJournalStoredItem, + journalStoredItemRow, + readJournalStoredItem +} from './journal-stored-output-rows' export type JournalDomainCurrentItemWrite = { item: AgentJournalRenderItem @@ -59,11 +64,16 @@ function nextJournalDomainItemOrdinal( export function writeJournalDomainCurrentItem( db: Database.Database, target: { sessionId: string; epoch: string }, - input: JournalDomainCurrentItemWrite + write: JournalDomainCurrentItemWrite ): AgentJournalRenderItem[] { if (!db.isTransaction) { throw new Error('journal item persistence requires its writer transaction') } + // A stored form handed back is written from its text, so every derived column stays exact. + const input = { + ...write, + item: hydrateJournalStoredItem(db, [target.sessionId, target.epoch], write.item) + } const body = normalizeJournalDomainItemBody( db, target, @@ -98,7 +108,7 @@ export function writeJournalDomainCurrentItem( (SELECT item_id FROM journal_item_aliases WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`) .get(...prefix, ...prefix, id, id) - return row ? JSON.parse(String(row.payload_json)) : undefined + return row ? readJournalStoredItem(db, prefix, row.payload_json) : undefined }) const next = stopNote.item const sequence = input.writtenSequence ?? next.sequence @@ -111,11 +121,13 @@ export function writeJournalDomainCurrentItem( .get(...prefix, next.itemId) : undefined const payload = JSON.stringify(item) - const { metadata, values } = journalDomainItemColumns(item, payload, next) + const columns = journalDomainItemColumns(item, payload, next) + const { metadata } = columns + const row = journalStoredItemRow(db, prefix, item, payload, columns.values) db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json, hidden,original_stop_note_json,queue_command_turn,queue_failed_outcome,queue_failed_at, - queue_failure_witnessed_through,queue_failure_report_turn_item_id,placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')}) - VALUES(${Array.from({ length: 18 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) + queue_failure_witnessed_through,queue_failure_report_turn_item_id,placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')},output_blob) + VALUES(${Array.from({ length: 19 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET ordinal=excluded.ordinal, revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index, owner_fence=excluded.owner_fence, execution_owner=excluded.execution_owner, payload_json=excluded.payload_json, @@ -123,7 +135,8 @@ export function writeJournalDomainCurrentItem( queue_command_turn=excluded.queue_command_turn, queue_failed_outcome=excluded.queue_failed_outcome, queue_failed_at=excluded.queue_failed_at, queue_failure_witnessed_through=excluded.queue_failure_witnessed_through, queue_failure_report_turn_item_id=excluded.queue_failure_report_turn_item_id,placement_json=excluded.placement_json, - ${JOURNAL_DOMAIN_ITEM_COLUMNS.map((column) => `${column}=excluded.${column}`).join(',')}`).run( + ${JOURNAL_DOMAIN_ITEM_COLUMNS.map((column) => `${column}=excluded.${column}`).join(',')}, + output_blob=CASE WHEN ? THEN journal_items.output_blob ELSE excluded.output_blob END`).run( ...prefix, next.itemId, ordinal, @@ -132,7 +145,7 @@ export function writeJournalDomainCurrentItem( next.sequenceIndex ?? 0, input.ownerFence, input.executionOwner ?? null, - payload, + row.payload, Number(input.hidden), stopNote.originalStopNoteJson, failure.commandTurn, @@ -141,7 +154,9 @@ export function writeJournalDomainCurrentItem( failure.witnessedThrough, failure.reportTurnItemId, journalDomainItemPlacement(next), - ...values + ...row.values, + row.blob, + Number(row.keep) ) writeJournalDomainItemNames( db, diff --git a/src/main/native-chat/agent-session-journal/journal-domain-dispatch-placement.ts b/src/main/native-chat/agent-session-journal/journal-domain-dispatch-placement.ts index f8375cf8f3f..892fd30eb3d 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-dispatch-placement.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-dispatch-placement.ts @@ -3,6 +3,7 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session/journ import type { AgentJournalTurnScope } from '../../../shared/agent-session/journal/agent-session-journal-turn-scope' import { writeJournalDomainCurrentItem } from './journal-domain-current-item' import type { JournalDomainSubmissionTarget } from './journal-domain-submission-records' +import { readJournalStoredItem } from './journal-stored-output-rows' export type JournalDomainDispatchItem = { item: AgentJournalRenderItem @@ -23,7 +24,7 @@ export function readJournalDomainDispatchItem( .get(target.sessionId, target.epoch, itemId) return row ? { - item: JSON.parse(String(row.payload_json)), + item: readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json), ordinal: Number(row.ordinal), hidden: Boolean(row.hidden), ownerFence: Number(row.owner_fence), diff --git a/src/main/native-chat/agent-session-journal/journal-domain-entity-read.ts b/src/main/native-chat/agent-session-journal/journal-domain-entity-read.ts index 338bccf23b5..62460efa384 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-entity-read.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-entity-read.ts @@ -11,6 +11,7 @@ import { readJournalDomainSubmissionBody } from './journal-domain-submission-rec import type { JournalHistoryOrigin } from './context-epochs/journal-history-origin' import type { JournalLatestStop } from './context-epochs/journal-stop-turn-end' import { providerTimelineKeyPart } from '../agent-session-timeline/provider-timeline-identity' +import { readJournalStoredItem } from './journal-stored-output-rows' export type JournalDomainEntityRead = Extract< JournalSqlPointRead['operation'], @@ -107,7 +108,9 @@ export function readJournalDomainEntity( .prepare(`SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`) .get(...prefix, operation.itemId) - const item: AgentJournalRenderItem | null = row ? JSON.parse(String(row.payload_json)) : null + const item: AgentJournalRenderItem | null = row + ? readJournalStoredItem(db, prefix, row.payload_json) + : null return { kind: operation.kind, item } } case 'submission': { diff --git a/src/main/native-chat/agent-session-journal/journal-domain-generation-reads.ts b/src/main/native-chat/agent-session-journal/journal-domain-generation-reads.ts index 59490ec3c3c..38a64be9cbc 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-generation-reads.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-generation-reads.ts @@ -13,6 +13,7 @@ import { } from './journal-domain-item-normalization' import { journalDomainContextHead } from './journal-sql-continuation-context' import type { JournalStopSettleFact } from './journal-writer-host-facts' +import { readJournalStoredItem } from './journal-stored-output-rows' /** Select current lifecycle obligations and their named turn/opener evidence, in render order. */ export function readJournalDomainGeneration( @@ -28,7 +29,7 @@ export function readJournalDomainGeneration( { item: AgentJournalRenderItem; ordinal: number; fence: number | undefined } >() const include = (row: Record) => { - const item: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) + const item: AgentJournalRenderItem = readJournalStoredItem(db, prefix, row.payload_json) selected.set(item.itemId, { item, ordinal: Number(row.ordinal), diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts index 1942deef374..dec8535b25d 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-roster.ts @@ -3,6 +3,7 @@ import type { AgentJournalRenderItem } from '../../../shared/agent-session/journ import { agentJournalItemSubagentId } from '../../../shared/agent-session/journal/agent-session-journal-producer' import { namedSubagents } from '../../../shared/native-chat/transcript/subagents/native-chat-subagent-items' import { offPageSubagentRoster } from '../agent-session-wire/publication/history/agent-session-history-subagent-roster' +import { readJournalStoredItem } from './journal-stored-output-rows' /** One indexed lookup per unnamed page subagent finds its earliest visible naming row. */ export function readJournalDomainHistoryRoster( @@ -42,7 +43,7 @@ export function readJournalDomainHistoryRoster( const named = [...naming] .sort(([, a], [, b]) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2]) .slice(0, ROSTER_ROWS) - .map(([itemId]) => JSON.parse(String(body.get(...prefix, itemId)!.payload_json))) + .map(([itemId]) => readJournalStoredItem(db, prefix, body.get(...prefix, itemId)!.payload_json)) return offPageSubagentRoster(named, page) } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts b/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts index fdf3580faca..d143579bca5 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts @@ -10,6 +10,7 @@ import { insertJournalDomainQueueRow, journalDomainQueueRow } from './journal-do import type { JournalDomainImportHeader, JournalDomainImportPage } from './journal-domain-import' import { writeJournalDomainItemNames } from './journal-domain-item-names' import { recordJournalLegacyChatRevision } from './database/journal-legacy-source' +import { hydrateJournalStoredItem, journalStoredItemRow } from './journal-stored-output-rows' export type JournalDomainImportState = | { state: 'converting'; epoch: null } @@ -230,15 +231,15 @@ export function insertJournalDomainItemRows( const insert = db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision, sequence,sequence_index,owner_fence,execution_owner,payload_json,hidden,original_stop_note_json, queue_command_turn,queue_failed_outcome,queue_failed_at,queue_failure_witnessed_through,queue_failure_report_turn_item_id, - placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')}) - VALUES(${Array.from({ length: 18 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) ${replace ? `ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET ${['ordinal', 'revision', 'sequence', 'sequence_index', 'owner_fence', 'execution_owner', 'payload_json', 'hidden', 'original_stop_note_json', 'queue_command_turn', 'queue_failed_outcome', 'queue_failed_at', 'queue_failure_witnessed_through', 'queue_failure_report_turn_item_id', 'placement_json', ...JOURNAL_DOMAIN_ITEM_COLUMNS].map((column) => `${column}=excluded.${column}`).join(',')}` : 'ON CONFLICT DO NOTHING'}`) + placement_json,${JOURNAL_DOMAIN_ITEM_COLUMNS.join(',')},output_blob) + VALUES(${Array.from({ length: 19 + JOURNAL_DOMAIN_ITEM_COLUMNS.length }, () => '?').join(',')}) ${replace ? `ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET ${['ordinal', 'revision', 'sequence', 'sequence_index', 'owner_fence', 'execution_owner', 'payload_json', 'hidden', 'original_stop_note_json', 'queue_command_turn', 'queue_failed_outcome', 'queue_failed_at', 'queue_failure_witnessed_through', 'queue_failure_report_turn_item_id', 'placement_json', ...JOURNAL_DOMAIN_ITEM_COLUMNS].map((column) => `${column}=excluded.${column}`).join(',')},output_blob=CASE WHEN ? THEN journal_items.output_blob ELSE excluded.output_blob END` : 'ON CONFLICT DO NOTHING'}`) for (const { ordinal, ownerFence, executionOwner, hidden, - item, - displayed, + item: given, + displayed: shown, originalStopNoteJson, queueCommandTurn, queueFailedOutcome, @@ -246,8 +247,13 @@ export function insertJournalDomainItemRows( queueFailureWitnessedThrough, queueFailureReportTurnItemId } of rows) { - const { payload, metadata, values, placement } = - prepared ?? prepareJournalDomainItemRow(item, displayed) + const item = hydrateJournalStoredItem(db, prefix, given) + const displayed = shown && hydrateJournalStoredItem(db, prefix, shown) + const encoded = + (item === given && displayed === shown ? prepared : undefined) ?? + prepareJournalDomainItemRow(item, displayed) + const { metadata, placement } = encoded + const row = journalStoredItemRow(db, prefix, item, encoded.payload, encoded.values) const result = insert.run( ...prefix, item.itemId, @@ -257,7 +263,7 @@ export function insertJournalDomainItemRows( item.sequenceIndex ?? 0, ownerFence, executionOwner, - payload, + row.payload, Number(hidden), originalStopNoteJson, Number(queueCommandTurn), @@ -266,7 +272,9 @@ export function insertJournalDomainItemRows( queueFailureWitnessedThrough, queueFailureReportTurnItemId, placement, - ...values + ...row.values, + row.blob, + ...(replace ? [Number(row.keep)] : []) ) names = metadata.namedSubagentIds if (writeNames && Number(result.changes) > 0) { diff --git a/src/main/native-chat/agent-session-journal/journal-domain-input-ends.ts b/src/main/native-chat/agent-session-journal/journal-domain-input-ends.ts index e7a20454337..5c0e1c56a86 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-input-ends.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-input-ends.ts @@ -11,6 +11,7 @@ import { readCodexErrorWillRetry } from '../../codex/app-server/codex-structured import type { JournalInputEnd } from './input-order/journal-input-obligations' import type { JournalDomainSubmissionStoredFacts } from './journal-domain-submission-records' import type { JournalItemRow, JournalStopEvent } from './durable-rows/journal-row-schema' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } export type JournalDomainInputItemObservation = Pick< @@ -184,7 +185,7 @@ export function observeJournalDomainInputStop( WHERE session_id=? AND epoch=? AND item_id=?`) .get(target.sessionId, target.epoch, scope.turnItemId) const item: AgentJournalRenderItem | null = current - ? JSON.parse(String(current.payload_json)) + ? readJournalStoredItem(db, [target.sessionId, target.epoch], current.payload_json) : null turnId = readAgentJournalTurn(item?.body)?.turnId } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts index c44c86cc732..f667c5a0ec2 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-display.ts @@ -3,6 +3,7 @@ import type Database from '../../sqlite/sync-database' import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract' import { projectJournalStopNoteForTurnState } from './context-epochs/journal-stop-note-projection' +import { hydrateJournalStoredItem } from './journal-stored-output-rows' export type JournalDomainItemTarget = { sessionId: string; epoch: string } @@ -83,7 +84,14 @@ export function readJournalDomainDisplayedRow( row: SqliteRow, stopStates?: ReadonlyMap ): { item: AgentJournalRenderItem; payload: string } { - const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) + const parsed: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) + // A row that did not select `output_blob` has it read by key. + const source = hydrateJournalStoredItem( + db, + [target.sessionId, target.epoch], + parsed, + row.output_blob + ) const states = row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION ? undefined @@ -96,6 +104,7 @@ export function readJournalDomainDisplayedRow( ) return { item, - payload: item === source ? String(row.payload_json) : JSON.stringify(item) + // The row text is the item only when nothing was read back from its stored output. + payload: item === parsed ? String(row.payload_json) : JSON.stringify(item) } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts index 8b043505cc1..b2f9af2a60d 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-presentation.ts @@ -8,6 +8,7 @@ import { } from '../agent-session-wire/publication/history/agent-session-item-wire-form' import type { HistoryItemForm } from '../agent-session-wire/publication/history/agent-session-history-page-bounds' import { journalDomainItemWire } from './journal-domain-item-wire' +import { hydrateJournalStoredItem, readJournalStoredItem } from './journal-stored-output-rows' import { journalDomainDisplayTurnState, readJournalDomainStopStates, @@ -20,7 +21,9 @@ export function journalDomainClientFormColumns(alias = 'item'): string { CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION} THEN COALESCE(${alias}.wire_json,${alias}.payload_json) ELSE ${alias}.payload_json END AS client_json, CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION} - THEN COALESCE(${alias}.wire_bytes,octet_length(${alias}.payload_json)) END AS client_bytes` + THEN COALESCE(${alias}.wire_bytes,octet_length(${alias}.payload_json)) END AS client_bytes, + CASE WHEN ${alias}.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION} AND ${alias}.wire_json IS NOT NULL + THEN NULL ELSE ${alias}.output_blob END AS output_blob` } /** Decode only this selected set; callers retain admitted forms and discard a rejected whole group. */ @@ -44,16 +47,24 @@ export function readJournalDomainClientForms( if (row.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION && states === undefined) { states = readJournalDomainStopStates(db, target, wanted, wantedJson) } - forms.set(String(row.item_id), readJournalDomainClientForm(row, states)) + forms.set(String(row.item_id), readJournalDomainClientForm(db, target, row, states)) } return forms } export function readJournalDomainClientForm( + db: Database.Database, + target: JournalDomainItemTarget, row: SqliteRow, states?: ReadonlyMap ): HistoryItemForm { - const item: AgentJournalRenderItem = JSON.parse(String(row.client_json)) + // The blob is selected only when the client form is the stored payload, never beside a saved head. + const item = readJournalStoredItem( + db, + [target.sessionId, target.epoch], + row.client_json, + row.output_blob + ) return row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION ? rememberAgentJournalItemWireForm(item, Number(row.client_bytes)) : agentJournalItemForWire( @@ -91,7 +102,8 @@ export function readJournalDomainItemDetail( if (!row) { return null } - const item: AgentJournalRenderItem = JSON.parse(String(row.detail_json)) + const parsed: AgentJournalRenderItem = JSON.parse(String(row.detail_json)) + const item = hydrateJournalStoredItem(db, [target.sessionId, target.epoch], parsed) if (row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION) { return item } @@ -102,6 +114,10 @@ export function readJournalDomainItemDetail( null, journalDomainDisplayTurnState(item, states) ) - const wire = journalDomainItemWire(item, String(row.detail_json), displayed) + const wire = journalDomainItemWire( + item, + item === parsed ? String(row.detail_json) : undefined, + displayed + ) return wire.detailJson === null ? displayed : JSON.parse(wire.detailJson) } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-removal.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-removal.ts index 7746678ef6f..8858912e7bb 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-removal.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-removal.ts @@ -14,6 +14,7 @@ import { observeJournalDomainTurnScope } from './journal-domain-turn-scope' import { appendJournalSqlEvent } from './journal-sql-event-tail' import { assertJournalFence } from './writing/journal-write-guards' import { invalidateJournalDomainQueue } from './journal-domain-queue-revision' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } @@ -72,7 +73,7 @@ export function removeJournalDomainItem( return { itemId, revision, removed: null, items: [] } } const removed: AgentJournalRenderItem | null = - row.payload_json === null ? null : JSON.parse(String(row.payload_json)) + row.payload_json === null ? null : readJournalStoredItem(db, prefix, row.payload_json) if ( row.turn_marker || row.prompt || diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item.ts b/src/main/native-chat/agent-session-journal/journal-domain-item.ts index 0e1bbc96e6f..450f4c1d316 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item.ts @@ -35,6 +35,7 @@ import { writeJournalDomainCurrentItem } from './journal-domain-current-item' import { writeJournalDomainTextSnapshot } from './journal-domain-text-snapshot' import { appendJournalSqlEvent } from './journal-sql-event-tail' import { readJournalSqlAcceptedInputs } from './journal-sql-accepted-inputs' +import { readJournalStoredItem } from './journal-stored-output-rows' export type { JournalDomainItemWrite, JournalDomainItemCommit } from './journal-domain-item-write' /** Runs only on the owning writer connection, inside its command SAVEPOINT. */ @@ -137,7 +138,7 @@ export function applyJournalDomainItem( sequence: Number(stored.sequence), body: input.body } - : JSON.parse(String(stored.payload_json)) + : readJournalStoredItem(db, prefix, stored.payload_json) : undefined const item = (key: string): AgentJournalRenderItem | null => { if (key === rawId || key === itemId) { @@ -148,7 +149,7 @@ export function applyJournalDomainItem( WHERE item.session_id=? AND item.epoch=? AND item.item_id=COALESCE( (SELECT item_id FROM journal_item_aliases WHERE session_id=? AND epoch=? AND provider_item_id=?),?)`) .get(...prefix, ...prefix, key, key) - return row ? JSON.parse(String(row.payload_json)) : null + return row ? readJournalStoredItem(db, prefix, row.payload_json) : null } let scope = input.options.turnScope ?? { kind: 'thread' as const } for (const key of input.scopeItemIds ?? []) { diff --git a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts index 59ec5874788..5c71449ebfa 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts @@ -1,16 +1,19 @@ import { journalForkStageEpochs } from './journal-fork-stage-state' import type Database from '../../sqlite/sync-database' +import type { JournalStoredOutputCompaction } from './journal-stored-output-compaction' export type JournalDomainMaintenance = | { kind: 'domain-retired-prune'; sessionId: string; epoch: string } | { kind: 'page-reclaim' } | { kind: 'compaction-state' } | { kind: 'compaction-complete'; now: number } + | { kind: 'stored-output-compaction' } export type JournalDomainMaintenanceResult = | { kind: 'domain-retired'; deleted: number; more: boolean } | { kind: 'page-reclaim'; more: boolean } | { kind: 'compaction-state'; completedAt: number | null } + | JournalStoredOutputCompaction const RETIRED_PAGE_BYTES = 1024 * 1024 // Only a ready head naming another epoch proves retirement; import shells have no epoch. @@ -27,7 +30,8 @@ export const JOURNAL_EPOCH_TABLES: readonly { 'original_stop_note_json', 'wire_json', 'detail_json', - 'outline_source_json' + 'outline_source_json', + 'output_blob' ] }, { name: 'journal_outline_cache', key: '', payload: ['outline_json'] }, diff --git a/src/main/native-chat/agent-session-journal/journal-domain-request-failure.ts b/src/main/native-chat/agent-session-journal/journal-domain-request-failure.ts index 797dc15840d..65a13d41b46 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-request-failure.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-request-failure.ts @@ -18,6 +18,7 @@ import { import { JournalSqlScope } from './journal-sql-scope' import type { JournalRequestMark } from './journal-request-failure' import type { JournalStopSettleFact } from './journal-writer-host-facts' +import { readJournalStoredItem } from './journal-stored-output-rows' export class JournalDomainRequestFailure { private readonly scope: JournalSqlScope @@ -54,7 +55,7 @@ export class JournalDomainRequestFailure { item(id: string): AgentJournalRenderItem | null { const row = this.row(id) - return row ? JSON.parse(String(row.payload_json)) : null + return row ? readJournalStoredItem(this.scope.db, this.scope.prefix, row.payload_json) : null } request(kind: JournalRequestMark['kind'], id: string): JournalRequestMark | null { @@ -64,7 +65,9 @@ export class JournalDomainRequestFailure { return null } const row = this.row(itemId) - const item: AgentJournalRenderItem | null = row ? JSON.parse(String(row.payload_json)) : null + const item: AgentJournalRenderItem | null = row + ? readJournalStoredItem(this.scope.db, this.scope.prefix, row.payload_json) + : null if (!item || !row?.root || item.sequence <= this.floor) { return null } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-retained-failures.ts b/src/main/native-chat/agent-session-journal/journal-domain-retained-failures.ts index 7941b2e4ff3..1e37f2ca98a 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-retained-failures.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-retained-failures.ts @@ -2,6 +2,7 @@ import type Database from '../../sqlite/sync-database' import { isDeepStrictEqual } from 'node:util' import { agentJournalLinkageFields } from '../../../shared/agent-session/journal/agent-session-journal-producer' import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } @@ -30,7 +31,7 @@ export function retainedFailures( const failedAt = Number(row.queue_failed_at) return { itemId, - item: JSON.parse(String(row.payload_json)), + item: readJournalStoredItem(db, prefix, row.payload_json), at: Math.max( failedAt, boundary > Number(row.queue_failure_witnessed_through) ? boundary : 0 @@ -51,7 +52,7 @@ export function restoreRetainedFailures( for (const { itemId, item, at, reports } of retained) { const row = kept.get(...prefix, itemId) const next: AgentJournalRenderItem | undefined = row - ? JSON.parse(String(row.payload_json)) + ? readJournalStoredItem(db, prefix, row.payload_json) : undefined if ( !next || diff --git a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts index d60f9c11ca1..835a12f16f2 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-stop-notes.ts @@ -6,6 +6,7 @@ import { projectJournalStopNote } from './context-epochs/journal-stop-note-proje import { journalDomainItemWire } from './journal-domain-item-wire' import { journalDomainOutlineSource } from './journal-domain-outline-source' import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract' +import { readJournalStoredItem } from './journal-stored-output-rows' type Target = { sessionId: string; epoch: string } @@ -52,7 +53,7 @@ export function refreshJournalDomainStopNotes( WHERE session_id=? AND epoch=? AND item_id=?`) const get = (id: string): AgentJournalRenderItem | undefined => { const row = turn.get(...prefix, id) - return row ? JSON.parse(String(row.payload_json)) : undefined + return row ? readJournalStoredItem(db, prefix, row.payload_json) : undefined } const priorGet = (id: string) => (previousTurns?.has(id) ? previousTurns.get(id) : get(id)) const update = @@ -60,7 +61,7 @@ export function refreshJournalDomainStopNotes( WHERE session_id=? AND epoch=? AND item_id=?`) const changed: AgentJournalRenderItem[] = [] for (const row of notes) { - const source: AgentJournalRenderItem = JSON.parse(String(row.payload_json)) + const source: AgentJournalRenderItem = readJournalStoredItem(db, prefix, row.payload_json) const prior: AgentJournalRenderItem = row.wire_version === AGENT_SESSION_ITEM_WIRE_VERSION ? resolveJournalDomainDisplay(source, row.wire_version, row.display_json, undefined) diff --git a/src/main/native-chat/agent-session-journal/journal-domain-taken-stop.ts b/src/main/native-chat/agent-session-journal/journal-domain-taken-stop.ts index 49b23b6b411..847166999c8 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-taken-stop.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-taken-stop.ts @@ -6,6 +6,7 @@ import { runningTurnLifecycleRevisions } from '../agent-session-wire/termination import { applyJournalDomainLifecycle } from './journal-domain-lifecycle' import type { JournalWriterOperation } from './journal-writer-command' +import { readJournalStoredItem } from './journal-stored-output-rows' export function applyJournalDomainLifecycleOperation( db: Database.Database, @@ -49,7 +50,9 @@ function applyJournalDomainTakenStop( WHERE session_id=? AND epoch=? AND turn_id=? AND hidden=0 AND lifecycle_state='running' ORDER BY sequence,sequence_index,ordinal`) .all(...prefix) - .map((row): AgentJournalRenderItem => JSON.parse(String(row.payload_json))) + .map((row): AgentJournalRenderItem => + readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json) + ) const result = applyJournalDomainLifecycle( db, target, diff --git a/src/main/native-chat/agent-session-journal/journal-domain-timeline-reads.ts b/src/main/native-chat/agent-session-journal/journal-domain-timeline-reads.ts index 5fa0b7be170..209633a13d1 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-timeline-reads.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-timeline-reads.ts @@ -8,6 +8,7 @@ import { } from './journal-domain-item-normalization' import type { JournalStopSettleFact } from './journal-writer-host-facts' import { assertJournalDomainItemAuthority } from './journal-domain-item' +import { readJournalStoredItem } from './journal-stored-output-rows' /** Typed named selections used only during the owning synchronous writer command. */ export function journalDomainTimelineReads( @@ -18,7 +19,7 @@ export function journalDomainTimelineReads( ): JournalTimelineView { const prefix = [target.sessionId, target.epoch] as const const decode = (row: Record | undefined): AgentJournalRenderItem | null => - row ? JSON.parse(String(row.payload_json)) : null + row ? readJournalStoredItem(db, prefix, row.payload_json) : null const item = (key: string) => decode( db diff --git a/src/main/native-chat/agent-session-journal/journal-fork-capture.ts b/src/main/native-chat/agent-session-journal/journal-fork-capture.ts index be33dfe1c28..bc2665c1463 100644 --- a/src/main/native-chat/agent-session-journal/journal-fork-capture.ts +++ b/src/main/native-chat/agent-session-journal/journal-fork-capture.ts @@ -54,7 +54,8 @@ export class JournalForkCapture { for (const row of this.db .prepare(`SELECT header.rank,header.item_id,header.provider_key, item.wire_version,CASE WHEN item.wire_version=${AGENT_SESSION_ITEM_WIRE_VERSION} - AND item.original_stop_note_json IS NOT NULL THEN item.wire_json END AS wire_json,item.payload_json + AND item.original_stop_note_json IS NOT NULL THEN item.wire_json END AS wire_json,item.payload_json, + item.output_blob FROM temp.fork_headers header LEFT JOIN journal_items item ON item.session_id=?3 AND item.epoch=?4 AND item.item_id=header.item_id WHERE header.rank>?1 AND (header.rank AgentJournalRenderItem } + +function readInlineRow(row: Record): ItemsByKindRowRead { + const json = String(row.payload_json) + return { bytes: json.length, item: () => JSON.parse(json) } +} + /** `rows` are the chosen items in ordinal order, read lazily so rows past the budget stay unread. */ export function budgetItemsByKindPage( rows: Iterable>, nextOrdinal: number | null, - budget = ITEMS_BY_KIND_PAGE_JSON_BUDGET + budget = ITEMS_BY_KIND_PAGE_JSON_BUDGET, + read: (row: Record) => ItemsByKindRowRead = readInlineRow ): { items: AgentJournalRenderItem[]; nextOrdinal: number | null } { const items: AgentJournalRenderItem[] = [] let used = 0 let lastKept = -1 for (const row of rows) { - const json = String(row.payload_json) - if (items.length > 0 && used + json.length > budget) { + const { bytes, item } = read(row) + if (items.length > 0 && used + bytes > budget) { return { items, nextOrdinal: lastKept } } - used += json.length + used += bytes lastKept = Number(row.ordinal) - items.push(JSON.parse(json)) + items.push(item()) } return { items, nextOrdinal } } /** Payload by primary key, so a page cut by the budget looks up no further rows. */ -export const ITEMS_BY_KIND_PAYLOAD_SQL = `SELECT payload_json,ordinal FROM journal_items +export const ITEMS_BY_KIND_PAYLOAD_SQL = `SELECT payload_json,ordinal,output_blob FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?` /** Each chosen item's row, in order, read only as the page budget asks for it. */ @@ -45,3 +56,19 @@ export function* itemsByKindPayloads( } } } + +/** One budgeted page of the chosen items, stored outputs restored and counted at full size. */ +export function readItemsByKindPage( + db: Database.Database, + prefix: readonly [sessionId: string, epoch: string], + chosen: readonly SqliteRow[], + nextOrdinal: number | null +): { items: AgentJournalRenderItem[]; nextOrdinal: number | null } { + const payload = db.prepare(ITEMS_BY_KIND_PAYLOAD_SQL) + return budgetItemsByKindPage( + itemsByKindPayloads(chosen, (itemId) => payload.get(...prefix, itemId)), + nextOrdinal, + undefined, + (row) => measureJournalStoredRow(db, prefix, row) + ) +} diff --git a/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts index 5d46a204c40..3048ca19a88 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts @@ -3,10 +3,13 @@ import type { JournalSqlOwner } from './journal-sql-owner' import type { JournalSqlReaderPool } from './journal-sql-reader-pool' import type { JournalSqlConversions } from './journal-sql-conversions' import type { JournalSqlActivity } from './journal-sql-activity' -import type { JournalSqlMaintenance } from './journal-sql-maintenance' +import type { JournalPagedMaintenanceKind, JournalSqlMaintenance } from './journal-sql-maintenance' +import { JOURNAL_STORED_OUTPUT_WRITES } from './journal-stored-output' const MINUTE = 60_000 const COMPACTION_INTERVAL = 6 * 60 * MINUTE +// About 8k saved outputs a minute at the shared 10% duty cycle; none while writes are off. +const STORED_OUTPUT_PAGES_PER_TICK = JOURNAL_STORED_OUTPUT_WRITES ? 256 : 0 /** Idle conversion and periodic pruning share one owner; activity never blocks retired cleanup. */ export class JournalSqlBackgroundMaintenance { @@ -141,6 +144,7 @@ export class JournalSqlBackgroundMaintenance { } } await this.pages({ kind: 'page-reclaim' }, this.reclaim) + await this.pages({ kind: 'stored-output-compaction' }, STORED_OUTPUT_PAGES_PER_TICK) } catch (error) { this.report(error) } @@ -215,10 +219,7 @@ export class JournalSqlBackgroundMaintenance { } private async pages( - operation: Extract< - JournalSqlMaintenance, - { kind: 'event-tail-prune' | 'domain-retired-prune' | 'page-reclaim' | 'chat-discard' } - >, + operation: Extract, limit = Infinity ): Promise { for (let unit = 0; unit < limit && !this.stopped.signal.aborted; unit++) { diff --git a/src/main/native-chat/agent-session-journal/journal-sql-continuation-context.ts b/src/main/native-chat/agent-session-journal/journal-sql-continuation-context.ts index 39f8ced9caf..c77ef7bb8a5 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-continuation-context.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-continuation-context.ts @@ -7,6 +7,7 @@ import type { AgentJournalCursor } from '../../../shared/agent-session/journal/a import type { JournalContinuationContext } from './journal-reads' import { journalContextEpoch } from './context-epochs/journal-context-epoch' import type { SqliteBindings, SqliteRow } from '../../sqlite/sqlite-statement' +import { readJournalStoredRow } from './journal-stored-output-rows' const PAGE = 256 @@ -45,14 +46,14 @@ export function readJournalDomainContinuationContext( let afterItem: SqliteBindings = [] do { const rows = db - .prepare(`SELECT payload_json, sequence, sequence_index, ordinal + .prepare(`SELECT payload_json, output_blob, sequence, sequence_index, ordinal FROM journal_items INDEXED BY journal_items_visible_position WHERE session_id=? AND epoch=? AND hidden=0 AND sequence>? ${afterItem.length ? 'AND (sequence, sequence_index, ordinal)>(?, ?, ?)' : ''} ORDER BY sequence, sequence_index, ordinal LIMIT ?`) .all(...prefix, clear, ...afterItem, PAGE) for (const row of rows) { - items.push(JSON.parse(String(row.payload_json))) + items.push(readJournalStoredRow(db, prefix, row)) } afterItem = nextPage(rows, ['sequence', 'sequence_index', 'ordinal']) } while (afterItem.length) diff --git a/src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts b/src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts index 85f02da7325..37b7ed208fd 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-item-reads.ts @@ -1,3 +1,4 @@ +import { readJournalStoredItem } from './journal-stored-output-rows' import type Database from '../../sqlite/sync-database' import { readJournalHistoryItems, @@ -7,11 +8,7 @@ import { type JournalHistoryPrompt } from './journal-history-items' import { journalDomainHistoryOpeners, journalDomainUserItemIds } from './journal-sql-history-items' -import { - budgetItemsByKindPage, - itemsByKindPayloads, - ITEMS_BY_KIND_PAYLOAD_SQL -} from './journal-items-by-kind-page' +import { readItemsByKindPage } from './journal-items-by-kind-page' import type { AgentJournalItemBody, AgentJournalRenderItem @@ -191,10 +188,8 @@ export function readJournalDomainItems( ) .sort((a, b) => Number(a.ordinal) - Number(b.ordinal)) const chosen = headers.slice(0, operation.limit) - const payload = db.prepare(ITEMS_BY_KIND_PAYLOAD_SQL) - const page = itemsByKindPayloads(chosen, (itemId) => payload.get(...prefix, itemId)) const next = headers.length > operation.limit ? Number(chosen.at(-1)!.ordinal) : null - return { kind: 'items-by-kind', ...budgetItemsByKindPage(page, next) } + return { kind: 'items-by-kind', ...readItemsByKindPage(db, prefix, chosen, next) } } case 'context-floor': { const row = db @@ -212,7 +207,10 @@ export function readJournalDomainItems( const row = db.prepare(`${TURNS_NEWEST_FIRST} LIMIT 1`).get(...prefix) return { kind: 'running-turn', - running: row ? runningStructuredAgentSessionTurnItemBySequence([parseItem(row)]) : null + // A turn row never stores tool output. + running: row + ? runningStructuredAgentSessionTurnItemBySequence([JSON.parse(String(row.payload_json))]) + : null } } case 'context-usage': @@ -250,7 +248,7 @@ function visibleItem( .prepare(`SELECT payload_json FROM journal_items WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`) .get(...prefix, itemId) - return row ? parseItem(row) : null + return row ? readJournalStoredItem(db, prefix, row.payload_json) : null } /** The input order's newest turn: latest position, then the greatest item ID among ties. */ @@ -264,7 +262,7 @@ export function newestJournalDomainTurnOrClear( WHERE session_id=? AND epoch=? AND hidden=0 AND turn_marker=1 ORDER BY sequence DESC, sequence_index DESC`) .iterate(...prefix)) { - const item = parseItem(row) + const item = readJournalStoredItem(db, prefix, row.payload_json) if ( newest && (item.sequence !== newest.sequence || @@ -285,7 +283,7 @@ function readContextUsage(db: Database.Database, prefix: Prefix): AgentSessionCo let used: number | null = null let window: number | null = null for (const row of db.prepare(TURNS_NEWEST_FIRST).iterate(...prefix)) { - const item = parseItem(row) + const item = readJournalStoredItem(db, prefix, row.payload_json) if (used !== null && window !== null && item.sequence < Math.min(used, window)) { break } @@ -307,13 +305,9 @@ export function readJournalSqlGoalRows(db: Database.Database, prefix: Prefix) { WHERE session_id=? AND epoch=? AND hidden=0 AND goal_transition=1 ORDER BY sequence DESC, ordinal ASC LIMIT 1`) .get(...prefix) - return row ? [parseItem(row)] : [] + return row ? [readJournalStoredItem(db, prefix, row.payload_json)] : [] } function readThreadGoal(db: Database.Database, prefix: Prefix): JournalThreadGoalSource { return journalThreadGoalSource(readJournalSqlGoalRows(db, prefix)) } - -function parseItem(row: Record): AgentJournalRenderItem { - return JSON.parse(String(row.payload_json)) -} diff --git a/src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts b/src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts index 6dc5c3cc016..cfad86dcc5b 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-lifecycle-reads.ts @@ -11,6 +11,7 @@ import type { JournalOutgrownChildRead, JournalRestartProgress } from './journal-lifecycle-reads' +import { readJournalStoredItem } from './journal-stored-output-rows' export type JournalSqlLifecycleRead = | { kind: 'latest-prompt' } @@ -37,7 +38,9 @@ export function readJournalDomainLifecycle( .get(...prefix) return { kind: 'latest-prompt', - prompt: row ? latestStructuredAgentSessionPrompt([JSON.parse(String(row.payload_json))]) : '' + prompt: row + ? latestStructuredAgentSessionPrompt([readJournalStoredItem(db, prefix, row.payload_json)]) + : '' } } if (operation.kind === 'restart-progress') { @@ -74,7 +77,7 @@ export function readJournalDomainLifecycle( FROM journal_items WHERE session_id=? AND epoch=? AND item_id=? AND hidden=0`) .get(...prefix, agentJournalSubmissionKey(nextClientMessageId)) const nextItem: AgentJournalRenderItem | null = next - ? JSON.parse(String(next.payload_json)) + ? readJournalStoredItem(db, prefix, next.payload_json) : null const body = nextItem?.body return { diff --git a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts index dc50d7cee98..d08321b8b58 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts @@ -17,6 +17,7 @@ import { type JournalDomainMaintenance, type JournalDomainMaintenanceResult } from './journal-domain-maintenance' +import { compactJournalStoredOutputPage } from './journal-stored-output-compaction' import { pruneJournalSqlEventTail, type JournalSqlEventTailResult } from './journal-sql-event-tail' import { discardJournalDomainImportPage, @@ -26,6 +27,14 @@ import { discardForgottenJournalChatPage } from './journal-forgotten-chats' import { installAgentSessionStoreRows } from '../../runtime/agent-session/records/agent-session-record-rows' import type { JournalSqlStoredRecords } from './journal-sql-database-read' +/** Operations the background pass runs a bounded page at a time. */ +export type JournalPagedMaintenanceKind = + | 'event-tail-prune' + | 'domain-retired-prune' + | 'page-reclaim' + | 'chat-discard' + | 'stored-output-compaction' + export type JournalSqlMaintenance = | { kind: 'records-install' } | { kind: 'wsl-workspace-rekey'; worktreeIds: readonly (readonly [string, string])[] } @@ -99,5 +108,7 @@ export function applyJournalSqlMaintenance( return completeJournalDomainCompaction(db, operation.now) case 'event-tail-prune': return pruneJournalSqlEventTail(db, operation.now) + case 'stored-output-compaction': + return compactJournalStoredOutputPage(db) } } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-open-command.ts b/src/main/native-chat/agent-session-journal/journal-sql-open-command.ts index 8dd8fd95ec3..dc092caf37a 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-open-command.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-open-command.ts @@ -15,6 +15,7 @@ import { import { staleSubagentRosterRevisions } from './item-lifecycle/journal-subagent-liveness' import { applyJournalDomainLifecycle } from './journal-domain-lifecycle' import { withdrawOpeningDerivedQueue } from './journal-open-queue-withdrawal' +import { readJournalStoredItem } from './journal-stored-output-rows' export type JournalOpenRecoveryResult = { failures: { @@ -140,7 +141,9 @@ function settleOpeningLiveWork( AND (lifecycle_state IN ('running','pending') OR live_work=1) AND live_work=1 ${after ? 'AND (ordinal,item_id)>(?,?)' : ''} ORDER BY ordinal,item_id LIMIT 128`) .all(target.sessionId, target.epoch, ...(after ?? [])) - const items: AgentJournalRenderItem[] = rows.map((row) => JSON.parse(String(row.payload_json))) + const items: AgentJournalRenderItem[] = rows.map((row) => + readJournalStoredItem(db, [target.sessionId, target.epoch], row.payload_json) + ) const revisions = staleSubagentRosterRevisions(items) if (revisions.length > 0) { const head = db diff --git a/src/main/native-chat/agent-session-journal/journal-sql-scope.ts b/src/main/native-chat/agent-session-journal/journal-sql-scope.ts index 752cf03bd22..db2e875d746 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-scope.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-scope.ts @@ -1,5 +1,7 @@ import type Database from '../../sqlite/sync-database' import type { SqliteRow } from '../../sqlite/sqlite-statement' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { hydrateJournalStoredItem } from './journal-stored-output-rows' export type JournalSqlReadWork = { rows: number; decodedBytes: number } export type JournalSqlIndexValues = { @@ -33,7 +35,8 @@ export class JournalSqlScope { throw new Error('Invalid journal projection value') } this.work.decodedBytes += Buffer.byteLength(json) - return this.indexValues ? this.indexValues.decode(json) : JSON.parse(json) + const value: T = this.indexValues ? this.indexValues.decode(json) : JSON.parse(json) + return isJournalItemShape(value) ? hydrateJournalStoredItem(this.db, this.prefix, value) : value } /** A projection row's own payload; never an index value pointer. */ @@ -72,3 +75,17 @@ export class JournalSqlScope { ) } } + +function isJournalItemShape(value: T): value is T & AgentJournalRenderItem { + return ( + typeof value === 'object' && + value !== null && + 'itemId' in value && + typeof value.itemId === 'string' && + 'body' in value && + typeof value.body === 'object' && + value.body !== null && + 'kind' in value.body && + typeof value.body.kind === 'string' + ) +} diff --git a/src/main/native-chat/agent-session-journal/journal-stored-output-compaction.ts b/src/main/native-chat/agent-session-journal/journal-stored-output-compaction.ts new file mode 100644 index 00000000000..0de8d1498ea --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stored-output-compaction.ts @@ -0,0 +1,116 @@ +import type Database from '../../sqlite/sync-database' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { AGENT_SESSION_ITEM_WIRE_VERSION } from '../agent-session-wire/publication/history/agent-session-item-presentation-contract' +import { journalStoredItemRow } from './journal-stored-output-rows' +import { JOURNAL_DOMAIN_ITEM_COLUMNS } from './journal-domain-item-read-columns' +import { + hydrateJournalItemOutput, + JOURNAL_STORED_OUTPUT_WRITES, + MAX_STORED_TEXT_BYTES +} from './journal-stored-output' + +/** The last rowid examined; rows appended later are picked up, rows behind it are never rescanned. */ +export const JOURNAL_STORED_OUTPUT_CURSOR = 'journal.stored-output.cursor' + +// Each page is one writer transaction: bounded rows scanned, rows rewritten, bytes and time. +const SCAN_ROWS = 512 +const PAGE_ROWS = 32 +const PAGE_BYTES = 512 * 1024 +const PAGE_MS = 5 +const MIN_ROW_BYTES = 512 +const WIRE_BYTES = JOURNAL_DOMAIN_ITEM_COLUMNS.indexOf('wire_bytes') + +export type JournalStoredOutputCompaction = { kind: 'stored-output-compaction'; more: boolean } + +function payloadText(item: AgentJournalRenderItem): string | undefined { + const body = item.body + return body.kind === 'tool-call' + ? body.output?.head + : body.kind === 'diff' + ? body.patch.head + : undefined +} + +function compactRow(db: Database.Database, rowid: number, writes: boolean): void { + const row = db + .prepare('SELECT session_id, epoch, payload_json, wire_bytes FROM journal_items WHERE rowid=?') + .get(rowid)! + const payload = String(row.payload_json) + const item: AgentJournalRenderItem = JSON.parse(payload) + const values: (number | null)[] = JOURNAL_DOMAIN_ITEM_COLUMNS.map(() => null) + values[WIRE_BYTES] = row.wire_bytes === null ? null : Number(row.wire_bytes) + const prefix = [String(row.session_id), String(row.epoch)] as const + const stored = journalStoredItemRow(db, prefix, item, payload, values, writes) + if (!stored.blob) { + return + } + const restored = hydrateJournalItemOutput(JSON.parse(stored.payload), stored.blob) + if (payloadText(restored) !== payloadText(item)) { + throw new Error('stored tool output did not read back as written') + } + db.prepare( + 'UPDATE journal_items SET payload_json=?, wire_bytes=?, output_blob=? WHERE rowid=?' + ).run(stored.payload, stored.values[WIRE_BYTES] ?? null, stored.blob, rowid) +} + +/** Moves saved tool output and diff text written inline into compressed row blobs, oldest rows first. */ +export function compactJournalStoredOutputPage( + db: Database.Database, + writes = JOURNAL_STORED_OUTPUT_WRITES +): JournalStoredOutputCompaction { + if (!writes) { + return { kind: 'stored-output-compaction', more: false } + } + const started = performance.now() + const saved = db + .prepare('SELECT value FROM agent_session_store_meta WHERE key=?') + .get(JOURNAL_STORED_OUTPUT_CURSOR) + let cursor = saved ? Number(saved.value) : 0 + // Lengths and NULL tests come from record headers, so skipped rows never load their text. Headers + // are read lazily: a page cut short by its budget never fetches the rows the next page starts at. + const headers = db + .prepare(`SELECT rowid, kind, wire_version, output_blob IS NULL AS inline, + octet_length(payload_json) AS bytes FROM journal_items WHERE rowid>? ORDER BY rowid LIMIT ?`) + .iterate(cursor, SCAN_ROWS) + let scanned = 0 + let rows = 0 + let bytes = 0 + let more = false + for (const header of headers) { + const size = Number(header.bytes) + if ( + rows >= PAGE_ROWS || + (rows > 0 && (bytes + size > PAGE_BYTES || performance.now() - started >= PAGE_MS)) + ) { + more = true + break + } + scanned++ + cursor = Number(header.rowid) + if ( + (header.kind !== 'tool-call' && header.kind !== 'diff') || + header.wire_version !== AGENT_SESSION_ITEM_WIRE_VERSION || + !header.inline || + size < MIN_ROW_BYTES || + // Judged from the record header: a row this large is never loaded inside a page. + size > MAX_STORED_TEXT_BYTES * 2 + ) { + continue + } + rows++ + bytes += size + try { + compactRow(db, cursor, writes) + } catch (error) { + // A row that can't be stored exactly stays inline; rethrowing would retry this page forever. + console.warn('[agent-session-journal] kept a saved tool output inline', error) + } + } + more ||= scanned === SCAN_ROWS + db.prepare(`INSERT INTO agent_session_store_meta(key,value) VALUES(?,?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run( + JOURNAL_STORED_OUTPUT_CURSOR, + String(cursor) + ) + return { kind: 'stored-output-compaction', more } +} diff --git a/src/main/native-chat/agent-session-journal/journal-stored-output-rows.ts b/src/main/native-chat/agent-session-journal/journal-stored-output-rows.ts new file mode 100644 index 00000000000..8135c9634f6 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stored-output-rows.ts @@ -0,0 +1,120 @@ +import type Database from '../../sqlite/sync-database' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { JOURNAL_DOMAIN_ITEM_COLUMNS } from './journal-domain-item-read-columns' +import type { ItemsByKindRowRead } from './journal-items-by-kind-page' +import { + hasJournalStoredOutput, + hydrateJournalItemOutput, + journalItemOutputPreview, + journalStoredTextBytes, + JOURNAL_STORED_OUTPUT_WRITES, + storeJournalItemOutput +} from './journal-stored-output' + +type Prefix = readonly [sessionId: string, epoch: string] + +const WIRE_BYTES = JOURNAL_DOMAIN_ITEM_COLUMNS.indexOf('wire_bytes') + +/** The one parse of a journal_items item column. */ +export function readJournalStoredItem( + db: Database.Database, + prefix: Prefix, + json: unknown, + /** The row's `output_blob` when the read already selected it, saving a lookup per row. */ + blob?: unknown +): T { + const item: T = JSON.parse(String(json)) + return hydrateJournalStoredItem(db, prefix, item, blob) +} + +/** A row that selected `payload_json` and, to save a lookup, `output_blob`. */ +export function readJournalStoredRow( + db: Database.Database, + prefix: Prefix, + row: { payload_json?: unknown; output_blob?: unknown } +): AgentJournalRenderItem { + // A row that did not select the blob reads it by key, never as missing. + return readJournalStoredItem(db, prefix, row.payload_json, row.output_blob) +} + +/** A page row's size counts its stored text, which is read only if the row joins the page. */ +export function measureJournalStoredRow( + db: Database.Database, + prefix: Prefix, + row: { payload_json?: unknown; output_blob?: unknown } +): ItemsByKindRowRead { + const json = String(row.payload_json) + if (!(row.output_blob instanceof Uint8Array)) { + return { bytes: json.length, item: () => JSON.parse(json) } + } + const parsed: AgentJournalRenderItem = JSON.parse(json) + return { + bytes: json.length + journalStoredTextBytes(parsed), + item: () => hydrateJournalStoredItem(db, prefix, parsed, row.output_blob) + } +} + +/** Returns the same object when the item stores nothing, so callers can reuse its row text. */ +export function hydrateJournalStoredItem( + db: Database.Database, + prefix: Prefix, + item: T, + blob?: unknown +): T { + if (!hasJournalStoredOutput(item)) { + return item + } + const held = + blob === undefined + ? db + .prepare( + 'SELECT output_blob FROM journal_items WHERE session_id=? AND epoch=? AND item_id=?' + ) + .get(...prefix, item.itemId)?.output_blob + : blob + return hydrateJournalItemOutput(item, held instanceof Uint8Array ? held : null) +} + +export type JournalStoredItemRow = { + payload: string + values: V[] + blob: Buffer | null + /** The row's own blob holds this item's text (an item whose stored output could not be read). */ + keep: boolean +} + +/** Row values for one item write: stored text moves to the blob; client-form columns stay as computed. */ +export function journalStoredItemRow( + db: Database.Database, + prefix: Prefix, + item: AgentJournalRenderItem, + payload: string, + values: readonly V[], + writes = JOURNAL_STORED_OUTPUT_WRITES +): JournalStoredItemRow { + if (hasJournalStoredOutput(item)) { + const held = db + .prepare(`SELECT 1 FROM journal_items + WHERE session_id=? AND epoch=? AND item_id=? AND output_blob IS NOT NULL`) + .get(...prefix, item.itemId) + if (held) { + return { payload, values: [...values], blob: null, keep: true } + } + // A fork or copy of an output that could not be read keeps its preview rather than failing. + console.warn(`[agent-session-journal] kept only the preview of a saved output in ${prefix[0]}`) + return { + payload: JSON.stringify(journalItemOutputPreview(item)), + values: [...values], + blob: null, + keep: false + } + } + const stored = writes ? storeJournalItemOutput(item) : null + if (!stored) { + return { payload, values: [...values], blob: null, keep: false } + } + const next: (V | number)[] = [...values] + // A null client size means "the stored payload's length", which the stored form no longer is. + next[WIRE_BYTES] ??= Buffer.byteLength(payload, 'utf8') + return { payload: JSON.stringify(stored.item), values: next, blob: stored.blob, keep: false } +} diff --git a/src/main/native-chat/agent-session-journal/journal-stored-output.test.ts b/src/main/native-chat/agent-session-journal/journal-stored-output.test.ts new file mode 100644 index 00000000000..cf35977074c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stored-output.test.ts @@ -0,0 +1,164 @@ +import { afterEach, expect, it, vi } from 'vitest' +import Database from '../../sqlite/sync-database' +import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types' +import { migrateJournalDomainSchema } from './journal-domain-schema' +import { writeJournalDomainCurrentItem } from './journal-domain-current-item' +import { runJournalTransaction } from './database/journal-database' +import { readJournalDomainItems } from './journal-sql-item-reads' +import { + readJournalDomainClientForms, + readJournalDomainItemDetail +} from './journal-domain-item-presentation' +import { compactJournalStoredOutputPage } from './journal-stored-output-compaction' +import { measureJournalStoredRow } from './journal-stored-output-rows' +import type * as StoredOutput from './journal-stored-output' + +const switches = vi.hoisted(() => ({ writes: true })) +vi.mock('./journal-stored-output', async (load) => { + const actual = await load() + return { + ...actual, + get JOURNAL_STORED_OUTPUT_WRITES() { + return switches.writes + } + } +}) + +const target = { sessionId: 'chat', epoch: 'epoch' } +// Past the 16 KiB client head, with text UTF-8 cannot carry unescaped (a lone surrogate). +const OUTPUT = `${'a'.repeat(20_000)}é\ud800 tail ${'line\n'.repeat(4_000)}` +const databases: Database.Database[] = [] +afterEach(() => { + switches.writes = true + for (const db of databases.splice(0)) { + db.close() + } +}) + +function chat() { + const db = new Database(':memory:') + databases.push(db) + migrateJournalDomainSchema(db) + db.prepare( + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace',?,?,900,1)` + ).run(target.sessionId, target.epoch) + return db +} + +function toolCall( + itemId: string, + state: 'running' | 'completed', + revision = 1 +): AgentJournalRenderItem { + return { + itemId, + revision, + sequence: 10, + observedAt: 1, + body: { + kind: 'tool-call', + name: 'shell', + callId: itemId, + input: { command: 'cat big.log' }, + state, + output: { head: OUTPUT, byteLength: 40_011, digest: 'd', truncated: false } + } + } +} + +function write(db: Database.Database, item: AgentJournalRenderItem) { + runJournalTransaction(db, () => + writeJournalDomainCurrentItem(db, target, { item, hidden: false, ownerFence: 1 }) + ) +} + +function stored(db: Database.Database, itemId: string) { + return db + .prepare(`SELECT output_blob IS NOT NULL AS blob FROM journal_items WHERE item_id=?`) + .get(itemId) +} + +function readTool(db: Database.Database) { + const read = readJournalDomainItems(db, target, { + kind: 'items-by-kind', + kinds: ['tool-call'], + afterOrdinal: 0, + limit: 10 + }) + return read.kind === 'items-by-kind' ? read.items : [] +} + +it('reads a saved tool output back whole, as the bounded client head, through rewrites and with writes off', () => { + const db = chat() + write(db, toolCall('tool', 'completed')) + expect(stored(db, 'tool')!.blob).toBe(1) + // A tool-output page counts a stored row at its full size, not its preview's. + const row = db.prepare(`SELECT payload_json, output_blob FROM journal_items`).get()! + expect(measureJournalStoredRow(db, [target.sessionId, target.epoch], row).bytes).toBeGreaterThan( + 40_011 + ) + + const [whole] = readTool(db) + expect(whole!.body).toMatchObject({ output: { head: OUTPUT, truncated: false } }) + expect(readJournalDomainItemDetail(db, target, 'tool')!.body).toMatchObject({ + output: { head: OUTPUT, truncated: false } + }) + const client = readJournalDomainClientForms(db, target, ['tool']).get('tool')! + expect(client.item.body).toEqual({ + kind: 'tool-call', + name: 'shell', + callId: 'tool', + input: { command: 'cat big.log' }, + state: 'completed', + output: { head: 'a'.repeat(16_384), byteLength: 40_011, digest: 'd', truncated: true } + }) + + expect(client.bytes).toBeGreaterThan(16_384) + + // A later revision written from the hydrated read keeps every byte. + write(db, { ...whole!, revision: 2 }) + // A row form that never hydrated keeps this row's blob; it cannot name another row's. + const raw: AgentJournalRenderItem = JSON.parse( + String( + db.prepare(`SELECT payload_json FROM journal_items WHERE item_id='tool'`).get()!.payload_json + ) + ) + write(db, { ...raw, revision: 3 }) + expect(readTool(db)[0]!.body).toMatchObject({ output: { head: OUTPUT, truncated: false } }) + expect(readJournalDomainClientForms(db, target, ['tool']).get('tool')!.bytes).toBe(client.bytes) + // A stored form copied to a row without its blob keeps the preview as a plain truncated output. + write(db, { ...raw, itemId: 'other', sequence: 11 }) + const copy = readJournalDomainItemDetail(db, target, 'other')!.body + expect(copy.kind === 'tool-call' && copy.output).toEqual({ + head: `${OUTPUT.slice(0, 512)}\n…\n${OUTPUT.slice(-512)}`, + byteLength: 40_011, + digest: 'd', + truncated: true + }) + + // With writes off, a rewrite stores the whole text inline again. + switches.writes = false + const tool = () => readTool(db).find((item) => item.itemId === 'tool')! + write(db, { ...tool(), revision: 4 }) + expect(stored(db, 'tool')!.blob).toBe(0) + expect(tool().body).toMatchObject({ output: { head: OUTPUT, truncated: false } }) +}) + +it('compacts inline rows written before the switch, leaving running calls inline', () => { + const db = chat() + switches.writes = false + write(db, toolCall('done', 'completed')) + write(db, { ...toolCall('live', 'running'), sequence: 11 }) + expect(stored(db, 'done')!.blob).toBe(0) + + switches.writes = true + while (compactJournalStoredOutputPage(db).more) { + // Pages until the pass reaches the last row. + } + expect(stored(db, 'done')!.blob).toBe(1) + expect(stored(db, 'live')!.blob).toBe(0) + expect(readTool(db).map((item) => item.body)).toMatchObject([ + { state: 'completed', output: { head: OUTPUT, truncated: false } }, + { state: 'running' } + ]) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-stored-output.ts b/src/main/native-chat/agent-session-journal/journal-stored-output.ts new file mode 100644 index 00000000000..fd75cfb1772 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-stored-output.ts @@ -0,0 +1,159 @@ +import { constants, zstdCompressSync, zstdDecompressSync } from 'node:zlib' +import type { + AgentJournalBoundedPayload, + AgentJournalItemBody, + AgentJournalRenderItem +} from '../../../shared/agent-session/journal/agent-session-journal-types' + +/** Off until builds that cannot read stored outputs are out of the downgrade window; reads always work. */ +export const JOURNAL_STORED_OUTPUT_WRITES = false + +// Format 1: one zstd frame over the text as a JSON string literal, so lone surrogates survive. +const FORMAT_ZSTD_JSON = 1 +const MIN_STORED_TEXT_BYTES = 512 +// Encoding and the read-back check run inside the writer transaction; larger text stays inline. +export const MAX_STORED_TEXT_BYTES = 2 * 1024 * 1024 +// Builds that predate stored outputs show this inline preview, as does this one if a blob is unreadable. +const PREVIEW_HEAD_CHARS = 512 +const PREVIEW_TAIL_CHARS = 512 +const ZSTD_PARAMS = { + [constants.ZSTD_c_compressionLevel]: 3, + [constants.ZSTD_c_checksumFlag]: 1 +} + +type StoredMarker = { truncated: boolean } +type StoredPayload = AgentJournalBoundedPayload & { stored: StoredMarker } + +function bodyPayload(body: AgentJournalItemBody): AgentJournalBoundedPayload | undefined { + return body.kind === 'tool-call' ? body.output : body.kind === 'diff' ? body.patch : undefined +} + +function withPayload( + item: T, + payload: AgentJournalBoundedPayload +): T { + const body = + item.body.kind === 'tool-call' + ? { ...item.body, output: payload } + : item.body.kind === 'diff' + ? { ...item.body, patch: payload } + : item.body + return { ...item, body } +} + +function storedMarker(payload: AgentJournalBoundedPayload | undefined): StoredMarker | null { + const stored: unknown = payload && 'stored' in payload ? payload.stored : undefined + return typeof stored === 'object' && + stored !== null && + 'truncated' in stored && + typeof stored.truncated === 'boolean' + ? { truncated: stored.truncated } + : null +} + +export function hasJournalStoredOutput(item: AgentJournalRenderItem): boolean { + return storedMarker(bodyPayload(item.body)) !== null +} + +/** The recorded length of the text an item stores outside its row, 0 when it stores none. */ +export function journalStoredTextBytes(item: AgentJournalRenderItem): number { + const payload = bodyPayload(item.body) + return payload && storedMarker(payload) ? payload.byteLength : 0 +} + +/** The row form and its blob, or null when this item's text stays inline. */ +export function storeJournalItemOutput( + item: T +): { item: T; blob: Buffer } | null { + const payload = bodyPayload(item.body) + if ( + !payload || + (item.body.kind === 'tool-call' && item.body.state === 'running') || + storedMarker(payload) || + payload.head.length * 3 < MIN_STORED_TEXT_BYTES || + payload.head.length > MAX_STORED_TEXT_BYTES + ) { + return null + } + const textBytes = Buffer.byteLength(payload.head, 'utf8') + // Reads bound and check decoding by the recorded length, so a text that disagrees stays inline. + if ( + textBytes < MIN_STORED_TEXT_BYTES || + textBytes > MAX_STORED_TEXT_BYTES || + textBytes > payload.byteLength || + (!payload.truncated && textBytes !== payload.byteLength) + ) { + return null + } + const text = payload.head + if (text.length <= PREVIEW_HEAD_CHARS + PREVIEW_TAIL_CHARS + MIN_STORED_TEXT_BYTES) { + return null + } + const preview = + PREVIEW_TAIL_CHARS > 0 + ? `${text.slice(0, PREVIEW_HEAD_CHARS)}\n…\n${text.slice(-PREVIEW_TAIL_CHARS)}` + : text.slice(0, PREVIEW_HEAD_CHARS) + const frame = zstdCompressSync(Buffer.from(JSON.stringify(text)), { params: ZSTD_PARAMS }) + // Small or incompressible text costs more as a blob plus preview than inline. + if (frame.length + 1 + Buffer.byteLength(preview, 'utf8') > textBytes * 0.8) { + return null + } + const blob = Buffer.allocUnsafe(frame.length + 1) + blob.writeUInt8(FORMAT_ZSTD_JSON, 0) + frame.copy(blob, 1) + const stored: StoredPayload = { + head: preview, + byteLength: payload.byteLength, + digest: payload.digest, + truncated: true, + stored: { truncated: payload.truncated } + } + return { item: withPayload(item, stored), blob } +} + +/** The inline preview as a plain truncated payload, for a stored form whose row holds no blob. */ +export function journalItemOutputPreview(item: T): T { + const payload = bodyPayload(item.body) + if (!payload || !storedMarker(payload)) { + return item + } + const { head, byteLength, digest } = payload + return withPayload(item, { head, byteLength, digest, truncated: true }) +} + +/** Unreadable blobs keep the marker, so a rewrite keeps the blob rather than an empty head. */ +export function hydrateJournalItemOutput( + item: T, + blob: Uint8Array | null | undefined +): T { + const payload = bodyPayload(item.body) + const marker = storedMarker(payload) + if (!payload || !marker) { + return item + } + try { + if (!blob || blob.length < 2 || blob[0] !== FORMAT_ZSTD_JSON) { + throw new Error('stored output blob is missing or unknown') + } + // JSON escapes spend at most six bytes per UTF-8 byte; the bound caps inflation. + const json = zstdDecompressSync(blob.subarray(1), { + maxOutputLength: payload.byteLength * 6 + 64 + }) + const text: unknown = JSON.parse(json.toString('utf8')) + if ( + typeof text !== 'string' || + (!marker.truncated && Buffer.byteLength(text, 'utf8') !== payload.byteLength) + ) { + throw new Error('stored output does not match its item') + } + return withPayload(item, { + head: text, + byteLength: payload.byteLength, + digest: payload.digest, + truncated: marker.truncated + }) + } catch (error) { + console.warn('[agent-session-journal] a saved tool output could not be read', error) + return item + } +}