refactor: write current chat items once per revision

This commit is contained in:
Brennan Benson
2026-11-08 08:06:00 -08:00
parent 7f482fbd0a
commit ac9c052d02
11 changed files with 122 additions and 87 deletions
@@ -79,8 +79,7 @@ CREATE TABLE journal_items (
wire_bytes INTEGER,
turn_item_id TEXT,
PRIMARY KEY(session_id, epoch, item_id)
) WITHOUT ROWID;
CREATE INDEX journal_items_position ON journal_items(session_id, epoch, sequence, sequence_index, ordinal);
);
CREATE TABLE journal_item_subagent_names (
session_id TEXT NOT NULL,
epoch TEXT NOT NULL,
@@ -129,7 +128,7 @@ CREATE TABLE journal_submission_attempts (
admitted_personal INTEGER NOT NULL DEFAULT 0 CHECK(admitted_personal IN (0,1)),
payload_json TEXT NOT NULL,
PRIMARY KEY(session_id, epoch, message_id, attempt_sequence, attempt_index)
) WITHOUT ROWID;
);
CREATE UNIQUE INDEX journal_submission_head ON journal_submission_attempts(session_id, epoch, message_id) WHERE current=1;
CREATE UNIQUE INDEX journal_submission_item ON journal_submission_attempts(session_id, epoch, item_id) WHERE current=1;
CREATE INDEX journal_submission_echo ON journal_submission_attempts(session_id, epoch, fingerprint, submitted_at, ordinal) WHERE current=1;
@@ -155,7 +154,7 @@ CREATE TABLE journal_events (
saved_at INTEGER NOT NULL,
changes_json TEXT NOT NULL,
PRIMARY KEY(session_id, epoch, sequence)
) WITHOUT ROWID;
);
CREATE TABLE journal_queue_cards (
session_id TEXT NOT NULL,
epoch TEXT NOT NULL,
@@ -166,7 +165,7 @@ CREATE TABLE journal_queue_cards (
command_turn_item_id TEXT,
payload_json TEXT NOT NULL,
PRIMARY KEY(session_id, epoch, message_id)
) WITHOUT ROWID;
);
CREATE INDEX journal_queue_order ON journal_queue_cards(session_id, epoch, position, message_id)
WHERE state IN ('waiting','returned');
CREATE INDEX journal_queue_last_position ON journal_queue_cards(session_id, epoch, position);
@@ -1,45 +0,0 @@
import type Database from '../../sqlite/sync-database'
import type { JournalDomainHistoryMetadata } from './journal-domain-history-metadata'
type HistoryTarget = { sessionId: string; epoch: string }
/** Every journal_items upsert calls this after writing the row, in the same transaction. */
export function writeJournalDomainHistoryBounds(
db: Database.Database,
target: HistoryTarget,
itemId: string,
metadata: Pick<JournalDomainHistoryMetadata, 'wireBytes' | 'namedSubagentIds'>
): void {
const key = [target.sessionId, target.epoch, itemId] as const
db.prepare(
'UPDATE journal_items SET wire_bytes=? WHERE session_id=? AND epoch=? AND item_id=?'
).run(metadata.wireBytes, ...key)
db.prepare(
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
).run(...key)
if (metadata.namedSubagentIds.length === 0) {
return
}
const row = db
.prepare(`SELECT sequence, sequence_index, ordinal, hidden FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=? AND root=1`)
.get(...key)
if (!row) {
return
}
const insert = db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(
session_id,epoch,subagent_id,item_id,sequence,sequence_index,ordinal,hidden)
VALUES(?,?,?,?,?,?,?,?)`)
for (const subagentId of metadata.namedSubagentIds) {
insert.run(
target.sessionId,
target.epoch,
subagentId,
itemId,
row.sequence,
row.sequence_index,
row.ordinal,
row.hidden
)
}
}
@@ -31,15 +31,11 @@ export function journalDomainHistoryMetadata(
): JournalDomainHistoryMetadata {
const root = isRootAgentJournalItem(item)
const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null
const named =
root && item.body.kind === 'message'
? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id)
: []
return {
root: Number(root),
subagentRow: Number(isSubagentRow(item)),
subagentGroupKey: block ? subagentFoldKey(block) : null,
namedSubagentIds: named,
namedSubagentIds: journalDomainNamedSubagentIds(item),
turnMarker: Number(
readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body)
),
@@ -47,6 +43,12 @@ export function journalDomainHistoryMetadata(
}
}
export function journalDomainNamedSubagentIds(item: AgentJournalRenderItem): string[] {
return isRootAgentJournalItem(item) && item.body.kind === 'message'
? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id)
: []
}
// The client form cuts tool/diff heads and whole items over the page budget; skipping the
// form elsewhere only overstates a size, which shortens a page but never drops content.
function wireBytes(item: AgentJournalRenderItem, payloadJson: string | undefined): number | null {
@@ -123,7 +123,8 @@ it('keeps revised naming items at their visible history position and removes obs
])
// A restored hidden row stays hidden when its body is revised.
db.prepare("UPDATE journal_items SET hidden=1 WHERE item_id='orca:first'").run()
db.exec(`UPDATE journal_items SET hidden=1 WHERE item_id='orca:first';
UPDATE journal_item_subagent_names SET hidden=1 WHERE item_id='orca:first'`)
await commit(revision)
expect(read({ direction: 'tail', limit: 1 }).page.subagentRoster).toMatchObject([
{ itemId: 'orca:second', sequence: 2, entry: { id: 'child', label: 'Second' } }
@@ -3,7 +3,7 @@ import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
import { insertJournalDomainQueueRow, journalDomainQueueRow } from './journal-domain-queue-rows'
import type { JournalDomainImportHeader, JournalDomainImportPage } from './journal-domain-import'
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
import { writeJournalDomainItemNames } from './journal-domain-item-names'
export type JournalDomainImportState =
| { state: 'converting'; epoch: null }
@@ -45,8 +45,8 @@ export function insertJournalDomainImportPage(
const insert =
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,
sequence,sequence_index,owner_fence,execution_owner,payload_json,hidden,root,subagent_row,
subagent_group_key,turn_marker)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`)
subagent_group_key,turn_marker,wire_bytes)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`)
for (const { ordinal, ownerFence, executionOwner, hidden, item } of page.rows) {
const payload = JSON.stringify(item)
const metadata = journalDomainHistoryMetadata(item, payload)
@@ -64,10 +64,17 @@ export function insertJournalDomainImportPage(
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.turnMarker
metadata.turnMarker,
metadata.wireBytes
)
if (Number(result.changes) > 0) {
writeJournalDomainHistoryBounds(db, target, item.itemId, metadata)
writeJournalDomainItemNames(
db,
target,
item,
{ ordinal, hidden },
metadata.namedSubagentIds
)
}
}
break
@@ -0,0 +1,48 @@
import type Database from '../../sqlite/sync-database'
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
import { journalDomainNamedSubagentIds } from './journal-domain-history-metadata'
type HistoryTarget = { sessionId: string; epoch: string }
/** Ordinary content revisions keep the same memberships without rewriting their rows. */
export function writeJournalDomainItemNames(
db: Database.Database,
target: HistoryTarget,
item: AgentJournalRenderItem,
placement: { ordinal: number; hidden: boolean },
namedIds: readonly string[],
previous?: { item: AgentJournalRenderItem; ordinal: number; hidden: boolean }
): void {
const names = new Set(namedIds)
const priorNames = new Set(previous ? journalDomainNamedSubagentIds(previous.item) : [])
if (
names.size === priorNames.size &&
[...names].every((id) => priorNames.has(id)) &&
(!previous ||
(previous.item.sequence === item.sequence &&
(previous.item.sequenceIndex ?? 0) === (item.sequenceIndex ?? 0) &&
previous.ordinal === placement.ordinal &&
previous.hidden === placement.hidden))
) {
return
}
const key = [target.sessionId, target.epoch, item.itemId] as const
db.prepare(
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
).run(...key)
const insert = db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(
session_id,epoch,subagent_id,item_id,sequence,sequence_index,ordinal,hidden)
VALUES(?,?,?,?,?,?,?,?)`)
for (const subagentId of names) {
insert.run(
target.sessionId,
target.epoch,
subagentId,
item.itemId,
item.sequence,
item.sequenceIndex ?? 0,
placement.ordinal,
Number(placement.hidden)
)
}
}
@@ -32,7 +32,7 @@ import { assertJournalFence } from './writing/journal-write-guards'
import type { JournalItemRow } from './durable-rows/journal-row-schema'
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
import { writeJournalDomainItemNames } from './journal-domain-item-names'
import { appendJournalSqlEvent } from './journal-sql-event-tail'
import {
readJournalSqlAcceptedInputs,
@@ -128,7 +128,7 @@ export function applyJournalDomainItem(
}
}
const stored = db
.prepare(`SELECT revision, ordinal, owner_fence, execution_owner, payload_json FROM journal_items
.prepare(`SELECT revision, ordinal, owner_fence, execution_owner, hidden, payload_json FROM journal_items
WHERE session_id=? AND epoch=? AND item_id=?`)
.get(...prefix, itemId)
const previous: AgentJournalRenderItem | undefined = stored
@@ -197,12 +197,12 @@ export function applyJournalDomainItem(
const payload = JSON.stringify(next)
const metadata = journalDomainHistoryMetadata(next, payload)
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json,
root,subagent_row,subagent_group_key,turn_marker)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
root,subagent_row,subagent_group_key,turn_marker,wire_bytes)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index,
execution_owner=excluded.execution_owner, payload_json=excluded.payload_json,
root=excluded.root, subagent_row=excluded.subagent_row, subagent_group_key=excluded.subagent_group_key,
turn_marker=excluded.turn_marker`).run(
turn_marker=excluded.turn_marker, wire_bytes=excluded.wire_bytes`).run(
...prefix,
itemId,
stored?.ordinal ?? sequence,
@@ -215,9 +215,19 @@ export function applyJournalDomainItem(
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.turnMarker
metadata.turnMarker,
metadata.wireBytes
)
writeJournalDomainItemNames(
db,
target,
next,
{ ordinal: Number(stored?.ordinal ?? sequence), hidden: Boolean(stored?.hidden) },
metadata.namedSubagentIds,
previous
? { item: previous, ordinal: Number(stored!.ordinal), hidden: Boolean(stored!.hidden) }
: undefined
)
writeJournalDomainHistoryBounds(db, target, itemId, metadata)
db.prepare(
'DELETE FROM journal_item_tombstones WHERE session_id=? AND epoch=? AND item_id=?'
).run(...prefix, itemId)
@@ -251,12 +261,19 @@ export function applyJournalDomainItem(
savedAt
)
}
db.prepare(`UPDATE journal_chat_state SET last_sequence=?,highest_fence=MAX(highest_fence,?),last_activity_at=MAX(last_activity_at,?)
WHERE session_id=? AND epoch=?`).run(sequence, row.fence, row.ts, ...prefix)
appendJournalSqlEvent(db, target, sequence, savedAt, {
items: [{ itemId, revision, sequence: next.sequence, sequenceIndex: next.sequenceIndex ?? 0 }],
submissions: result.receipts.map((receipt) => receipt.clientMessageId),
cards: result.cards.map((card) => card.messageId)
})
appendJournalSqlEvent(
db,
target,
sequence,
savedAt,
{
items: [
{ itemId, revision, sequence: next.sequence, sequenceIndex: next.sequenceIndex ?? 0 }
],
submissions: result.receipts.map((receipt) => receipt.clientMessageId),
cards: result.cards.map((card) => card.messageId)
},
{ fence: row.fence, observedAt: row.ts }
)
return result
}
@@ -304,7 +304,6 @@ it.each([100, 1_000])(
expect(plans).toContain(
'SEARCH journal_chat_state USING INDEX sqlite_autoindex_journal_chat_state_1 (session_id=?)'
)
expect(plans).toContain('SEARCH journal_events USING PRIMARY KEY (session_id=? AND epoch>?)')
expect(plans.some((detail) => /^SCAN (journal_|ready|retired)/.test(detail))).toBe(false)
expect(
plans
@@ -221,11 +221,6 @@ export function applyJournalDomainQueueCommand(
placeJournalDomainQueueRow(db, target, messageId, position)
}
retireHolds(db, target, retire, commit, savedAt)
db.prepare('UPDATE journal_chat_state SET last_sequence=? WHERE session_id=? AND epoch=?').run(
sequence,
target.sessionId,
target.epoch
)
appendJournalSqlEvent(db, target, sequence, savedAt, {
items: [],
submissions: [],
@@ -11,14 +11,19 @@ export function appendJournalSqlEvent(
target: { sessionId: string; epoch: string },
sequence: number,
savedAt: number,
changes: JournalDomainEventChanges
changes: JournalDomainEventChanges,
activity?: { fence: number; observedAt: number }
): void {
const prefix = [target.sessionId, target.epoch] as const
const json = JSON.stringify(changes)
const oversized = Buffer.byteLength(json, 'utf8') > MAX_EVENT_BYTES
db.prepare(`UPDATE journal_chat_state SET retained_sequence=MAX(retained_sequence,?)
db.prepare(`UPDATE journal_chat_state SET last_sequence=?, retained_sequence=MAX(retained_sequence,?),
highest_fence=MAX(highest_fence,?), last_activity_at=MAX(last_activity_at,?)
WHERE session_id=? AND epoch=?`).run(
sequence,
oversized ? sequence : Math.max(0, sequence - MAX_EVENT_SEQUENCES),
activity?.fence ?? 0,
activity?.observedAt ?? 0,
...prefix
)
db.prepare(`INSERT INTO journal_events(session_id,epoch,sequence,saved_at,changes_json)
@@ -8,7 +8,7 @@ import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal
import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation'
import { migrateJournalDomainSchema } from './journal-domain-schema'
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
import { writeJournalDomainItemNames } from './journal-domain-item-names'
import { JournalSqlBatch } from './journal-sql-batch'
import {
commitJournalSqlMessages,
@@ -59,7 +59,7 @@ export function messageFixture(db = new Database(':memory:')) {
const metadata = journalDomainHistoryMetadata(item, payload)
db.prepare(`INSERT INTO journal_items
(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json,
root,subagent_row,subagent_group_key,turn_marker) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(
root,subagent_row,subagent_group_key,turn_marker,wire_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(
target.sessionId,
target.epoch,
item.itemId,
@@ -72,9 +72,16 @@ export function messageFixture(db = new Database(':memory:')) {
metadata.root,
metadata.subagentRow,
metadata.subagentGroupKey,
metadata.turnMarker
metadata.turnMarker,
metadata.wireBytes
)
writeJournalDomainItemNames(
db,
target,
item,
{ ordinal: item.sequence, hidden: false },
metadata.namedSubagentIds
)
writeJournalDomainHistoryBounds(db, target, item.itemId, metadata)
}
const seedSend = () => {
const body: AgentJournalMessageItem = {