mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
refactor: write current chat items once per revision
This commit is contained in:
@@ -79,8 +79,7 @@ CREATE TABLE journal_items (
|
||||
wire_bytes INTEGER,
|
||||
turn_item_id TEXT,
|
||||
PRIMARY KEY(session_id, epoch, item_id)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX journal_items_position ON journal_items(session_id, epoch, sequence, sequence_index, ordinal);
|
||||
);
|
||||
CREATE TABLE journal_item_subagent_names (
|
||||
session_id TEXT NOT NULL,
|
||||
epoch TEXT NOT NULL,
|
||||
@@ -129,7 +128,7 @@ CREATE TABLE journal_submission_attempts (
|
||||
admitted_personal INTEGER NOT NULL DEFAULT 0 CHECK(admitted_personal IN (0,1)),
|
||||
payload_json TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, epoch, message_id, attempt_sequence, attempt_index)
|
||||
) WITHOUT ROWID;
|
||||
);
|
||||
CREATE UNIQUE INDEX journal_submission_head ON journal_submission_attempts(session_id, epoch, message_id) WHERE current=1;
|
||||
CREATE UNIQUE INDEX journal_submission_item ON journal_submission_attempts(session_id, epoch, item_id) WHERE current=1;
|
||||
CREATE INDEX journal_submission_echo ON journal_submission_attempts(session_id, epoch, fingerprint, submitted_at, ordinal) WHERE current=1;
|
||||
@@ -155,7 +154,7 @@ CREATE TABLE journal_events (
|
||||
saved_at INTEGER NOT NULL,
|
||||
changes_json TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, epoch, sequence)
|
||||
) WITHOUT ROWID;
|
||||
);
|
||||
CREATE TABLE journal_queue_cards (
|
||||
session_id TEXT NOT NULL,
|
||||
epoch TEXT NOT NULL,
|
||||
@@ -166,7 +165,7 @@ CREATE TABLE journal_queue_cards (
|
||||
command_turn_item_id TEXT,
|
||||
payload_json TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, epoch, message_id)
|
||||
) WITHOUT ROWID;
|
||||
);
|
||||
CREATE INDEX journal_queue_order ON journal_queue_cards(session_id, epoch, position, message_id)
|
||||
WHERE state IN ('waiting','returned');
|
||||
CREATE INDEX journal_queue_last_position ON journal_queue_cards(session_id, epoch, position);
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { JournalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
|
||||
type HistoryTarget = { sessionId: string; epoch: string }
|
||||
|
||||
/** Every journal_items upsert calls this after writing the row, in the same transaction. */
|
||||
export function writeJournalDomainHistoryBounds(
|
||||
db: Database.Database,
|
||||
target: HistoryTarget,
|
||||
itemId: string,
|
||||
metadata: Pick<JournalDomainHistoryMetadata, 'wireBytes' | 'namedSubagentIds'>
|
||||
): void {
|
||||
const key = [target.sessionId, target.epoch, itemId] as const
|
||||
db.prepare(
|
||||
'UPDATE journal_items SET wire_bytes=? WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(metadata.wireBytes, ...key)
|
||||
db.prepare(
|
||||
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(...key)
|
||||
if (metadata.namedSubagentIds.length === 0) {
|
||||
return
|
||||
}
|
||||
const row = db
|
||||
.prepare(`SELECT sequence, sequence_index, ordinal, hidden FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND item_id=? AND root=1`)
|
||||
.get(...key)
|
||||
if (!row) {
|
||||
return
|
||||
}
|
||||
const insert = db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(
|
||||
session_id,epoch,subagent_id,item_id,sequence,sequence_index,ordinal,hidden)
|
||||
VALUES(?,?,?,?,?,?,?,?)`)
|
||||
for (const subagentId of metadata.namedSubagentIds) {
|
||||
insert.run(
|
||||
target.sessionId,
|
||||
target.epoch,
|
||||
subagentId,
|
||||
itemId,
|
||||
row.sequence,
|
||||
row.sequence_index,
|
||||
row.ordinal,
|
||||
row.hidden
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -31,15 +31,11 @@ export function journalDomainHistoryMetadata(
|
||||
): JournalDomainHistoryMetadata {
|
||||
const root = isRootAgentJournalItem(item)
|
||||
const block = item.body.kind === 'message' ? subagentItemBlock(item.body) : null
|
||||
const named =
|
||||
root && item.body.kind === 'message'
|
||||
? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id)
|
||||
: []
|
||||
return {
|
||||
root: Number(root),
|
||||
subagentRow: Number(isSubagentRow(item)),
|
||||
subagentGroupKey: block ? subagentFoldKey(block) : null,
|
||||
namedSubagentIds: named,
|
||||
namedSubagentIds: journalDomainNamedSubagentIds(item),
|
||||
turnMarker: Number(
|
||||
readAgentJournalTurn(item.body) !== null || isAgentSessionContextClear(item.body)
|
||||
),
|
||||
@@ -47,6 +43,12 @@ export function journalDomainHistoryMetadata(
|
||||
}
|
||||
}
|
||||
|
||||
export function journalDomainNamedSubagentIds(item: AgentJournalRenderItem): string[] {
|
||||
return isRootAgentJournalItem(item) && item.body.kind === 'message'
|
||||
? item.body.blocks.flatMap(namedSubagents).map((entry) => entry.id)
|
||||
: []
|
||||
}
|
||||
|
||||
// The client form cuts tool/diff heads and whole items over the page budget; skipping the
|
||||
// form elsewhere only overstates a size, which shortens a page but never drops content.
|
||||
function wireBytes(item: AgentJournalRenderItem, payloadJson: string | undefined): number | null {
|
||||
|
||||
@@ -123,7 +123,8 @@ it('keeps revised naming items at their visible history position and removes obs
|
||||
])
|
||||
|
||||
// A restored hidden row stays hidden when its body is revised.
|
||||
db.prepare("UPDATE journal_items SET hidden=1 WHERE item_id='orca:first'").run()
|
||||
db.exec(`UPDATE journal_items SET hidden=1 WHERE item_id='orca:first';
|
||||
UPDATE journal_item_subagent_names SET hidden=1 WHERE item_id='orca:first'`)
|
||||
await commit(revision)
|
||||
expect(read({ direction: 'tail', limit: 1 }).page.subagentRoster).toMatchObject([
|
||||
{ itemId: 'orca:second', sequence: 2, entry: { id: 'child', label: 'Second' } }
|
||||
|
||||
@@ -3,7 +3,7 @@ import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal
|
||||
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
import { insertJournalDomainQueueRow, journalDomainQueueRow } from './journal-domain-queue-rows'
|
||||
import type { JournalDomainImportHeader, JournalDomainImportPage } from './journal-domain-import'
|
||||
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
|
||||
import { writeJournalDomainItemNames } from './journal-domain-item-names'
|
||||
|
||||
export type JournalDomainImportState =
|
||||
| { state: 'converting'; epoch: null }
|
||||
@@ -45,8 +45,8 @@ export function insertJournalDomainImportPage(
|
||||
const insert =
|
||||
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,
|
||||
sequence,sequence_index,owner_fence,execution_owner,payload_json,hidden,root,subagent_row,
|
||||
subagent_group_key,turn_marker)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`)
|
||||
subagent_group_key,turn_marker,wire_bytes)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`)
|
||||
for (const { ordinal, ownerFence, executionOwner, hidden, item } of page.rows) {
|
||||
const payload = JSON.stringify(item)
|
||||
const metadata = journalDomainHistoryMetadata(item, payload)
|
||||
@@ -64,10 +64,17 @@ export function insertJournalDomainImportPage(
|
||||
metadata.root,
|
||||
metadata.subagentRow,
|
||||
metadata.subagentGroupKey,
|
||||
metadata.turnMarker
|
||||
metadata.turnMarker,
|
||||
metadata.wireBytes
|
||||
)
|
||||
if (Number(result.changes) > 0) {
|
||||
writeJournalDomainHistoryBounds(db, target, item.itemId, metadata)
|
||||
writeJournalDomainItemNames(
|
||||
db,
|
||||
target,
|
||||
item,
|
||||
{ ordinal, hidden },
|
||||
metadata.namedSubagentIds
|
||||
)
|
||||
}
|
||||
}
|
||||
break
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
import type { AgentJournalRenderItem } from '../../../shared/agent-session/journal/agent-session-journal-types'
|
||||
import { journalDomainNamedSubagentIds } from './journal-domain-history-metadata'
|
||||
|
||||
type HistoryTarget = { sessionId: string; epoch: string }
|
||||
|
||||
/** Ordinary content revisions keep the same memberships without rewriting their rows. */
|
||||
export function writeJournalDomainItemNames(
|
||||
db: Database.Database,
|
||||
target: HistoryTarget,
|
||||
item: AgentJournalRenderItem,
|
||||
placement: { ordinal: number; hidden: boolean },
|
||||
namedIds: readonly string[],
|
||||
previous?: { item: AgentJournalRenderItem; ordinal: number; hidden: boolean }
|
||||
): void {
|
||||
const names = new Set(namedIds)
|
||||
const priorNames = new Set(previous ? journalDomainNamedSubagentIds(previous.item) : [])
|
||||
if (
|
||||
names.size === priorNames.size &&
|
||||
[...names].every((id) => priorNames.has(id)) &&
|
||||
(!previous ||
|
||||
(previous.item.sequence === item.sequence &&
|
||||
(previous.item.sequenceIndex ?? 0) === (item.sequenceIndex ?? 0) &&
|
||||
previous.ordinal === placement.ordinal &&
|
||||
previous.hidden === placement.hidden))
|
||||
) {
|
||||
return
|
||||
}
|
||||
const key = [target.sessionId, target.epoch, item.itemId] as const
|
||||
db.prepare(
|
||||
'DELETE FROM journal_item_subagent_names WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(...key)
|
||||
const insert = db.prepare(`INSERT OR IGNORE INTO journal_item_subagent_names(
|
||||
session_id,epoch,subagent_id,item_id,sequence,sequence_index,ordinal,hidden)
|
||||
VALUES(?,?,?,?,?,?,?,?)`)
|
||||
for (const subagentId of names) {
|
||||
insert.run(
|
||||
target.sessionId,
|
||||
target.epoch,
|
||||
subagentId,
|
||||
item.itemId,
|
||||
item.sequence,
|
||||
item.sequenceIndex ?? 0,
|
||||
placement.ordinal,
|
||||
Number(placement.hidden)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ import { assertJournalFence } from './writing/journal-write-guards'
|
||||
import type { JournalItemRow } from './durable-rows/journal-row-schema'
|
||||
import type { QueuedMessageRow } from './queued-messages/queued-message-table'
|
||||
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
|
||||
import { writeJournalDomainItemNames } from './journal-domain-item-names'
|
||||
import { appendJournalSqlEvent } from './journal-sql-event-tail'
|
||||
import {
|
||||
readJournalSqlAcceptedInputs,
|
||||
@@ -128,7 +128,7 @@ export function applyJournalDomainItem(
|
||||
}
|
||||
}
|
||||
const stored = db
|
||||
.prepare(`SELECT revision, ordinal, owner_fence, execution_owner, payload_json FROM journal_items
|
||||
.prepare(`SELECT revision, ordinal, owner_fence, execution_owner, hidden, payload_json FROM journal_items
|
||||
WHERE session_id=? AND epoch=? AND item_id=?`)
|
||||
.get(...prefix, itemId)
|
||||
const previous: AgentJournalRenderItem | undefined = stored
|
||||
@@ -197,12 +197,12 @@ export function applyJournalDomainItem(
|
||||
const payload = JSON.stringify(next)
|
||||
const metadata = journalDomainHistoryMetadata(next, payload)
|
||||
db.prepare(`INSERT INTO journal_items(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,execution_owner,payload_json,
|
||||
root,subagent_row,subagent_group_key,turn_marker)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
|
||||
root,subagent_row,subagent_group_key,turn_marker,wire_bytes)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(session_id,epoch,item_id) DO UPDATE SET
|
||||
revision=excluded.revision, sequence=excluded.sequence, sequence_index=excluded.sequence_index,
|
||||
execution_owner=excluded.execution_owner, payload_json=excluded.payload_json,
|
||||
root=excluded.root, subagent_row=excluded.subagent_row, subagent_group_key=excluded.subagent_group_key,
|
||||
turn_marker=excluded.turn_marker`).run(
|
||||
turn_marker=excluded.turn_marker, wire_bytes=excluded.wire_bytes`).run(
|
||||
...prefix,
|
||||
itemId,
|
||||
stored?.ordinal ?? sequence,
|
||||
@@ -215,9 +215,19 @@ export function applyJournalDomainItem(
|
||||
metadata.root,
|
||||
metadata.subagentRow,
|
||||
metadata.subagentGroupKey,
|
||||
metadata.turnMarker
|
||||
metadata.turnMarker,
|
||||
metadata.wireBytes
|
||||
)
|
||||
writeJournalDomainItemNames(
|
||||
db,
|
||||
target,
|
||||
next,
|
||||
{ ordinal: Number(stored?.ordinal ?? sequence), hidden: Boolean(stored?.hidden) },
|
||||
metadata.namedSubagentIds,
|
||||
previous
|
||||
? { item: previous, ordinal: Number(stored!.ordinal), hidden: Boolean(stored!.hidden) }
|
||||
: undefined
|
||||
)
|
||||
writeJournalDomainHistoryBounds(db, target, itemId, metadata)
|
||||
db.prepare(
|
||||
'DELETE FROM journal_item_tombstones WHERE session_id=? AND epoch=? AND item_id=?'
|
||||
).run(...prefix, itemId)
|
||||
@@ -251,12 +261,19 @@ export function applyJournalDomainItem(
|
||||
savedAt
|
||||
)
|
||||
}
|
||||
db.prepare(`UPDATE journal_chat_state SET last_sequence=?,highest_fence=MAX(highest_fence,?),last_activity_at=MAX(last_activity_at,?)
|
||||
WHERE session_id=? AND epoch=?`).run(sequence, row.fence, row.ts, ...prefix)
|
||||
appendJournalSqlEvent(db, target, sequence, savedAt, {
|
||||
items: [{ itemId, revision, sequence: next.sequence, sequenceIndex: next.sequenceIndex ?? 0 }],
|
||||
submissions: result.receipts.map((receipt) => receipt.clientMessageId),
|
||||
cards: result.cards.map((card) => card.messageId)
|
||||
})
|
||||
appendJournalSqlEvent(
|
||||
db,
|
||||
target,
|
||||
sequence,
|
||||
savedAt,
|
||||
{
|
||||
items: [
|
||||
{ itemId, revision, sequence: next.sequence, sequenceIndex: next.sequenceIndex ?? 0 }
|
||||
],
|
||||
submissions: result.receipts.map((receipt) => receipt.clientMessageId),
|
||||
cards: result.cards.map((card) => card.messageId)
|
||||
},
|
||||
{ fence: row.fence, observedAt: row.ts }
|
||||
)
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -304,7 +304,6 @@ it.each([100, 1_000])(
|
||||
expect(plans).toContain(
|
||||
'SEARCH journal_chat_state USING INDEX sqlite_autoindex_journal_chat_state_1 (session_id=?)'
|
||||
)
|
||||
expect(plans).toContain('SEARCH journal_events USING PRIMARY KEY (session_id=? AND epoch>?)')
|
||||
expect(plans.some((detail) => /^SCAN (journal_|ready|retired)/.test(detail))).toBe(false)
|
||||
expect(
|
||||
plans
|
||||
|
||||
@@ -221,11 +221,6 @@ export function applyJournalDomainQueueCommand(
|
||||
placeJournalDomainQueueRow(db, target, messageId, position)
|
||||
}
|
||||
retireHolds(db, target, retire, commit, savedAt)
|
||||
db.prepare('UPDATE journal_chat_state SET last_sequence=? WHERE session_id=? AND epoch=?').run(
|
||||
sequence,
|
||||
target.sessionId,
|
||||
target.epoch
|
||||
)
|
||||
appendJournalSqlEvent(db, target, sequence, savedAt, {
|
||||
items: [],
|
||||
submissions: [],
|
||||
|
||||
@@ -11,14 +11,19 @@ export function appendJournalSqlEvent(
|
||||
target: { sessionId: string; epoch: string },
|
||||
sequence: number,
|
||||
savedAt: number,
|
||||
changes: JournalDomainEventChanges
|
||||
changes: JournalDomainEventChanges,
|
||||
activity?: { fence: number; observedAt: number }
|
||||
): void {
|
||||
const prefix = [target.sessionId, target.epoch] as const
|
||||
const json = JSON.stringify(changes)
|
||||
const oversized = Buffer.byteLength(json, 'utf8') > MAX_EVENT_BYTES
|
||||
db.prepare(`UPDATE journal_chat_state SET retained_sequence=MAX(retained_sequence,?)
|
||||
db.prepare(`UPDATE journal_chat_state SET last_sequence=?, retained_sequence=MAX(retained_sequence,?),
|
||||
highest_fence=MAX(highest_fence,?), last_activity_at=MAX(last_activity_at,?)
|
||||
WHERE session_id=? AND epoch=?`).run(
|
||||
sequence,
|
||||
oversized ? sequence : Math.max(0, sequence - MAX_EVENT_SEQUENCES),
|
||||
activity?.fence ?? 0,
|
||||
activity?.observedAt ?? 0,
|
||||
...prefix
|
||||
)
|
||||
db.prepare(`INSERT INTO journal_events(session_id,epoch,sequence,saved_at,changes_json)
|
||||
|
||||
@@ -8,7 +8,7 @@ import { agentJournalSubmissionKey } from '../../../shared/agent-session/journal
|
||||
import { agentSessionSendBodyFingerprint } from '../../../shared/agent-session/requests/structured-agent-session-send-mutation'
|
||||
import { migrateJournalDomainSchema } from './journal-domain-schema'
|
||||
import { journalDomainHistoryMetadata } from './journal-domain-history-metadata'
|
||||
import { writeJournalDomainHistoryBounds } from './journal-domain-history-bounds'
|
||||
import { writeJournalDomainItemNames } from './journal-domain-item-names'
|
||||
import { JournalSqlBatch } from './journal-sql-batch'
|
||||
import {
|
||||
commitJournalSqlMessages,
|
||||
@@ -59,7 +59,7 @@ export function messageFixture(db = new Database(':memory:')) {
|
||||
const metadata = journalDomainHistoryMetadata(item, payload)
|
||||
db.prepare(`INSERT INTO journal_items
|
||||
(session_id,epoch,item_id,ordinal,revision,sequence,sequence_index,owner_fence,payload_json,
|
||||
root,subagent_row,subagent_group_key,turn_marker) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(
|
||||
root,subagent_row,subagent_group_key,turn_marker,wire_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`).run(
|
||||
target.sessionId,
|
||||
target.epoch,
|
||||
item.itemId,
|
||||
@@ -72,9 +72,16 @@ export function messageFixture(db = new Database(':memory:')) {
|
||||
metadata.root,
|
||||
metadata.subagentRow,
|
||||
metadata.subagentGroupKey,
|
||||
metadata.turnMarker
|
||||
metadata.turnMarker,
|
||||
metadata.wireBytes
|
||||
)
|
||||
writeJournalDomainItemNames(
|
||||
db,
|
||||
target,
|
||||
item,
|
||||
{ ordinal: item.sequence, hidden: false },
|
||||
metadata.namedSubagentIds
|
||||
)
|
||||
writeJournalDomainHistoryBounds(db, target, item.itemId, metadata)
|
||||
}
|
||||
const seedSend = () => {
|
||||
const body: AgentJournalMessageItem = {
|
||||
|
||||
Reference in New Issue
Block a user