From 9de3f2811d680ae2a3669577d4bbfeb593b17cab Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:04:20 -0800 Subject: [PATCH] refactor: connect named journal commands to the new database --- .../database/journal-domain-database.test.ts | 20 +- .../database/journal-domain-profile-import.ts | 24 +- .../database/journal-domain-schema-001.ts | 42 +-- .../database/journal-sql-database.ts | 68 ++-- .../journal-domain-current-item.ts | 15 + .../journal-domain-dispatch-attempt.ts | 37 +-- .../journal-domain-history-page.ts | 2 +- .../journal-domain-import-discard.ts | 2 +- .../journal-domain-import-rows.ts | 27 +- .../journal-domain-import-submissions.ts | 42 +-- .../journal-domain-import.test.ts | 46 ++- .../journal-domain-import.ts | 8 + .../journal-domain-item-reads.test.ts | 140 +++++++- .../journal-domain-item.ts | 7 +- .../journal-domain-maintenance.test.ts | 12 +- .../journal-domain-maintenance.ts | 4 +- .../journal-domain-queue.test.ts | 4 +- .../journal-domain-submission-admission.ts | 32 +- .../journal-domain-submission-records.ts | 97 ++---- .../journal-domain-submissions.test.ts | 12 +- .../journal-sql-command.ts | 314 ++++++++++-------- .../journal-sql-database-read.ts | 26 +- .../journal-sql-message-effects.ts | 2 +- .../journal-sql-message.test-fixture.ts | 2 +- .../journal-sql-message.test.ts | 4 +- .../journal-sql-owner.test.ts | 6 +- .../journal-writer-changes.ts | 21 +- .../journal-writer-effects.ts | 18 +- .../startup/main-process-ready-foundation.ts | 2 +- .../wsl-drive-unc-conversion-startup.test.ts | 21 +- .../wsl-drive-unc-conversion-startup.ts | 12 +- .../wsl-drive-unc-conversion-stores.ts | 9 +- 32 files changed, 680 insertions(+), 398 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-database.test.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-database.test.ts index 5141faa8c42..c65e6bce7c8 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-database.test.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-database.test.ts @@ -90,8 +90,8 @@ it('publishes an empty incremental store that retains new authoritative state on expect(filename).toBe(journalDomainDatabasePath(stateDirectory)) const db = open(filename) db.exec(`INSERT INTO agent_session_records VALUES('new','{"title":"new chat"}'); - INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES('new','epoch',2,1)`) + INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace','new','epoch',2,1)`) expect(db.pragma('auto_vacuum', { simple: true })).toBe(2) expect(db.prepare('PRAGMA integrity_check').get()).toEqual({ integrity_check: 'ok' }) const reopened = open(await initializeJournalDomainDatabase(stateDirectory)) @@ -118,9 +118,9 @@ it('copies opaque profile metadata in source order from the committed WAL, witho { session_id: 'chat', record_json: '{"title":"original"}' }, { session_id: 'a', record_json: 'unreadable opaque record' } ]) - expect(current.prepare('SELECT * FROM journal_sessions').all()).toEqual([ - { session_id: 'chat', workspace_id: 'workspace', epoch: 'epoch' } - ]) + expect( + current.prepare('SELECT session_id,workspace_id,epoch FROM journal_chat_state').all() + ).toEqual([{ session_id: 'chat', workspace_id: 'workspace', epoch: null }]) expect( current.prepare('SELECT tab_id,session_id,position FROM agent_session_tabs').get() ).toEqual({ tab_id: 'tab', session_id: 'chat', position: 1 }) @@ -184,9 +184,9 @@ it('accepts supported early schemas with no additive metadata tables without mig source.close() const before = hash(journalDatabasePath(stateDirectory)) const current = open(await initializeJournalDomainDatabase(stateDirectory)) - expect(current.prepare('SELECT * FROM journal_sessions').all()).toEqual([ - { session_id: 'early', workspace_id: 'workspace', epoch: 'epoch' } - ]) + expect( + current.prepare('SELECT session_id,workspace_id,epoch FROM journal_chat_state').all() + ).toEqual([{ session_id: 'early', workspace_id: 'workspace', epoch: null }]) expect(current.prepare('SELECT record_json FROM agent_session_records').all()).toEqual([]) expect(hash(journalDatabasePath(stateDirectory))).toBe(before) }) @@ -207,11 +207,11 @@ it('publishes complete metadata under competing initialization without replacing result_json: '{"kind":"stop"}' }) winner.exec( - `INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) VALUES('winner','epoch',7,1)` + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','winner','epoch',7,1)` ) expect( open(await initializeJournalDomainDatabase(stateDirectory)) - .prepare('SELECT session_id,last_sequence FROM journal_chat_state') + .prepare('SELECT session_id,last_sequence FROM journal_chat_state WHERE epoch IS NOT NULL') .get() ).toEqual({ session_id: 'winner', last_sequence: 7 }) expect(readFileSync(join(foreign, 'snapshot.db'), 'utf8')).toBe('foreign evidence') diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts index 53e8880afdd..0ea4c601d44 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts @@ -25,7 +25,7 @@ export function importJournalDomainProfile( `the original chat store uses unreleased schema ${version}` ) } - copyProfileTable(source, destination, 'journal_sessions', ['session_id', 'workspace_id', 'epoch']) + copyChatCatalog(source, destination) copyProfileTable( source, destination, @@ -90,6 +90,28 @@ export function importJournalDomainProfile( const PAGE_ROWS = 128 +function copyChatCatalog(source: Database.Database, destination: Database.Database): void { + const insert = destination.prepare( + 'INSERT INTO journal_chat_state(session_id,workspace_id) VALUES(?,?)' + ) + runJournalTransaction(destination, () => { + let after: string | undefined + for (;;) { + const rows = source + .prepare(`SELECT session_id,workspace_id FROM journal_sessions + ${after === undefined ? '' : 'WHERE session_id>?'} ORDER BY session_id LIMIT ?`) + .all(...(after === undefined ? [] : [after]), PAGE_ROWS) + if (rows.length === 0) { + return + } + for (const row of rows) { + insert.run(row.session_id, row.workspace_id) + } + after = String(rows.at(-1)!.session_id) + } + }) +} + /** The fixed schema above is the whole copy surface; no JSON is interpreted or rewritten. */ function copyProfileTable( source: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-schema-001.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-schema-001.ts index 4ab5faac4a1..c3323ebaf97 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-schema-001.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-schema-001.ts @@ -1,9 +1,4 @@ export const JOURNAL_DOMAIN_SCHEMA_001 = ` -CREATE TABLE journal_sessions ( - session_id TEXT PRIMARY KEY, - workspace_id TEXT NOT NULL, - epoch TEXT NOT NULL -); CREATE TABLE agent_session_records ( session_id TEXT PRIMARY KEY, record_json TEXT NOT NULL @@ -53,13 +48,15 @@ CREATE TABLE agent_session_attachment_sweeps ( CREATE TABLE journal_chat_state ( session_id TEXT PRIMARY KEY, - epoch TEXT NOT NULL, - last_sequence INTEGER NOT NULL, + workspace_id TEXT NOT NULL, + epoch TEXT, + last_sequence INTEGER NOT NULL DEFAULT 0, retained_sequence INTEGER NOT NULL DEFAULT 0, - highest_fence INTEGER NOT NULL, + highest_fence INTEGER NOT NULL DEFAULT 0, context_boundary INTEGER NOT NULL DEFAULT 0, last_activity_at INTEGER NOT NULL DEFAULT 0, derived_turn_item_id TEXT, + history_origin_json TEXT, queue_latest_stop_json TEXT, queue_manual_stop_json TEXT, queue_resumed_sequence INTEGER NOT NULL DEFAULT 0, @@ -69,6 +66,7 @@ CREATE TABLE journal_chat_state ( queue_clear_lifted INTEGER NOT NULL DEFAULT 0, queue_failure_boundary INTEGER NOT NULL DEFAULT 0 ); +CREATE INDEX journal_chat_ready ON journal_chat_state(session_id) WHERE epoch IS NOT NULL; CREATE TABLE journal_queue_clear_members ( session_id TEXT NOT NULL, epoch TEXT NOT NULL, @@ -113,6 +111,7 @@ CREATE TABLE journal_items ( signed_out INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(session_id, epoch, item_id) ); +CREATE INDEX journal_items_ordinal ON journal_items(session_id,epoch,ordinal); CREATE TABLE journal_item_subagent_names ( session_id TEXT NOT NULL, epoch TEXT NOT NULL, @@ -165,35 +164,14 @@ CREATE TABLE journal_submission_attempts ( dispatch_state TEXT NOT NULL, dispatch_fence INTEGER NOT NULL DEFAULT 0, signed_out INTEGER NOT NULL DEFAULT 0 CHECK(signed_out IN (0,1)), + recovered INTEGER NOT NULL DEFAULT 0 CHECK(recovered IN (0,1)), candidate INTEGER NOT NULL DEFAULT 0, admitted_personal INTEGER NOT NULL DEFAULT 0 CHECK(admitted_personal IN (0,1)), echo_observed INTEGER NOT NULL DEFAULT 0 CHECK(echo_observed IN (0,1)), admission_body_json TEXT, - admission_provider_handle_json TEXT, - admission_fence INTEGER NOT NULL DEFAULT 0, - admission_scope_json TEXT, admission_family_id TEXT, - admission_origin TEXT, - admission_source_kind TEXT, - admission_options_json TEXT, - handover_sequence INTEGER, - handover_at INTEGER, handover_fence INTEGER, - handover_scope_json TEXT, - handover_place_sequence INTEGER, - handover_place_at INTEGER, - handover_place_scope_json TEXT, - carries_json TEXT, - rejection_sequence INTEGER, - rejection_at INTEGER, - rejection_fence INTEGER, - rejection_reason TEXT, - rejection_fact_json TEXT, - rejection_recovered INTEGER, - rejection_kept_message_id TEXT, - rejection_answered_turn_json TEXT, - presentation_member INTEGER NOT NULL DEFAULT 0 CHECK(presentation_member IN (0,1)), - normalized_handoff INTEGER NOT NULL DEFAULT 0 CHECK(normalized_handoff IN (0,1)), + facts_json TEXT NOT NULL DEFAULT '{}', payload_json TEXT NOT NULL, PRIMARY KEY(session_id, epoch, message_id, attempt_sequence, attempt_index) ); @@ -203,6 +181,8 @@ CREATE INDEX journal_submission_echo ON journal_submission_attempts(session_id, CREATE INDEX journal_submission_person_acceptance ON journal_submission_attempts(session_id, epoch, accepted_sequence DESC) WHERE dispatch_state='accepted' AND admitted_personal=1; CREATE INDEX journal_submission_family ON journal_submission_attempts(session_id,epoch,admission_family_id,attempt_sequence,attempt_index,ordinal) WHERE current=1; CREATE INDEX journal_submission_ordinal ON journal_submission_attempts(session_id,epoch,ordinal); +CREATE INDEX journal_submission_recovery ON journal_submission_attempts(session_id,epoch,ordinal,message_id) + WHERE current=1 AND (dispatch_state='pending' OR (dispatch_state='unknown' AND recovered=0)); CREATE INDEX journal_submission_acceptance ON journal_submission_attempts(session_id, epoch, accepted_sequence DESC) WHERE dispatch_state='accepted'; CREATE TABLE journal_send_receipts ( session_id TEXT NOT NULL, diff --git a/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts b/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts index 972c87f49f4..b350a6d3f9b 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts @@ -9,11 +9,14 @@ import { JournalSqlActivity } from '../journal-sql-activity' import { JournalSqlBackgroundMaintenance } from '../journal-sql-background-maintenance' import type { JournalWriterCommand } from '../journal-writer-command' import type { JournalWriterCommit } from '../journal-writer-changes' -import type { JournalWriterHostFacts } from '../journal-writer-host-facts' import type { JournalWriterEffect, JournalWriterEffectResult } from '../journal-writer-effects' import type { CommandReceiptScope } from '../writing/command-receipt-schema' import { AgentSessionJournalError } from '../writing/journal-write-guards' import { journalDatabasePath } from './journal-host-database' +import { + initializeJournalDomainDatabase, + journalDomainDatabasePath +} from './journal-domain-database' /** The runtime owns workers, never a writable connection or a retained chat projection. */ export class JournalSqlDatabase { @@ -35,7 +38,7 @@ export class JournalSqlDatabase { } ) { this.readers = new JournalSqlReaderPool( - journalDatabasePath(stateDirectory), + journalDomainDatabasePath(stateDirectory), journalDatabasePath(stateDirectory), options.readerPath ) @@ -57,11 +60,8 @@ export class JournalSqlDatabase { onMaintenanceError: (error: unknown, sessionId?: string) => void } ): Promise { - const writer = new JournalSqlOwner( - journalDatabasePath(stateDirectory), - stateDirectory, - options.writerPath - ) + const filename = await initializeJournalDomainDatabase(stateDirectory) + const writer = new JournalSqlOwner(filename, stateDirectory, options.writerPath) try { return new JournalSqlDatabase(writer, stateDirectory, await writer.opened(), options) } catch (error) { @@ -122,21 +122,49 @@ export class JournalSqlDatabase { return this.readEpoch(sessionId) } - openSession(identity: AgentSessionJournalIdentity, facts?: JournalWriterHostFacts) { + openSession(identity: AgentSessionJournalIdentity) { return this.serialize(identity.sessionId, async () => { - const epoch = await this.conversions.ensure(identity.sessionId) - const target = { sessionId: identity.sessionId, epoch: epoch ?? '', ownerToken: randomUUID() } - const commit = await this.writer.write({ - target, - operation: { kind: 'open' }, - facts: { ...facts, identity } - }) - // Founding assigns the physical epoch on the writer; the public cursor may be logical. - const founded = await this.readEpoch(identity.sessionId) - if (founded === null) { - throw new Error('opened journal has no physical epoch') + let epoch = await this.conversions.ensure(identity.sessionId) + if (epoch === null) { + const founded = await this.writer.maintain({ + kind: 'import-finish', + header: { + sessionId: identity.sessionId, + workspaceId: identity.workspaceId, + epoch: randomUUID(), + head: { + lastSequence: 0, + highestFence: 0, + contextBoundary: 0, + lastActivityAt: 0, + derivedTurnItemId: null, + historyOrigin: null, + queueFailureBoundary: 0, + queuePauseMarks: { latestStop: null, resumedSequence: 0, reopenedSequence: 0 } + } + } + }) + if (typeof founded !== 'string') { + throw new Error('founded journal has no physical epoch') + } + epoch = founded } - return { target: { ...target, epoch: founded }, commit } + const first = await this.readers.domainHistory({ + sessionId: identity.sessionId, + direction: 'hydrate' + }) + const target = { sessionId: identity.sessionId, epoch, ownerToken: randomUUID() } + const cursor = first.page.liveCursor ?? first.page.window.nextCursor + const commit: JournalWriterCommit = { + cursor, + value: { epoch, sequence: cursor.sequence }, + inputChanged: false, + items: [], + keys: [], + inputKeys: [], + effects: [] + } + return { target, commit } }) } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts b/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts index 00fd702aa69..b758d55447d 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-current-item.ts @@ -23,6 +23,21 @@ export type JournalDomainCurrentItemWrite = { writtenSequence?: number } +export function nextJournalDomainItemOrdinal( + db: Database.Database, + target: { sessionId: string; epoch: string } +): number { + if (!db.isTransaction) { + throw new Error('journal item ordinal requires its writer transaction') + } + return Number( + db + .prepare(`SELECT COALESCE(MAX(ordinal),0)+1 AS ordinal + FROM journal_items WHERE session_id=? AND epoch=?`) + .get(target.sessionId, target.epoch)!.ordinal + ) +} + /** The owning command chooses revision, placement and authority before its one event append. */ export function writeJournalDomainCurrentItem( db: Database.Database, diff --git a/src/main/native-chat/agent-session-journal/journal-domain-dispatch-attempt.ts b/src/main/native-chat/agent-session-journal/journal-domain-dispatch-attempt.ts index 6bb1bf6322f..dc49c2cac29 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-dispatch-attempt.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-dispatch-attempt.ts @@ -9,6 +9,7 @@ import type { JournalDomainSubmissionAttempt, JournalDomainSubmissionTarget } from './journal-domain-submission-records' +import { journalDomainSubmissionFactsJson } from './journal-domain-submission-records' import { readStoredRejectionFact } from './projection/journal-dispatch-reducer' import { boundInlineText, @@ -21,20 +22,9 @@ export function writeJournalDomainDispatchAttempt( target: JournalDomainSubmissionTarget, attempt: JournalDomainSubmissionAttempt ): void { - const { - submission: next, - candidate, - handover, - handoverPlace, - carries, - rejection, - presentationMember, - normalizedHandOff - } = attempt - db.prepare(`UPDATE journal_submission_attempts SET dispatch_state=?,candidate=?,payload_json=?,dispatch_fence=?,signed_out=?, - handover_sequence=?,handover_at=?,handover_fence=?,handover_scope_json=?,handover_place_sequence=?,handover_place_at=?,handover_place_scope_json=?,carries_json=?, - rejection_sequence=?,rejection_at=?,rejection_fence=?,rejection_reason=?,rejection_fact_json=?,rejection_recovered=?, - rejection_kept_message_id=?,rejection_answered_turn_json=?,presentation_member=?,normalized_handoff=?, + const { submission: next, candidate, handover } = attempt + db.prepare(`UPDATE journal_submission_attempts SET dispatch_state=?,candidate=?,payload_json=?,dispatch_fence=?,signed_out=?,recovered=?, + handover_fence=?,facts_json=?, admission_body_json=CASE WHEN ? THEN admission_body_json ELSE NULL END WHERE session_id=? AND epoch=? AND message_id=? AND current=1`).run( next.dispatchState, @@ -45,24 +35,9 @@ export function writeJournalDomainDispatchAttempt( next.dispatchState === 'rejected' && readWholeAgentSessionFailureFact(next.rejection)?.kind === 'notSignedIn' ), - handover?.sequence ?? null, - handover?.at ?? null, + Number(next.recovered === true), handover?.fence ?? null, - handover ? JSON.stringify(handover.scope) : null, - handoverPlace?.sequence ?? null, - handoverPlace?.at ?? null, - handoverPlace?.scope === undefined ? null : JSON.stringify(handoverPlace.scope), - JSON.stringify(carries), - rejection?.sequence ?? null, - rejection?.at ?? null, - rejection?.fence ?? null, - rejection?.reason ?? null, - rejection?.fact ? JSON.stringify(rejection.fact) : null, - rejection ? Number(rejection.recovered) : null, - rejection?.keptMessageId ?? null, - rejection?.answeredInTurn === undefined ? null : JSON.stringify(rejection.answeredInTurn), - Number(presentationMember), - Number(normalizedHandOff), + journalDomainSubmissionFactsJson(attempt), Number( next.dispatchState === 'pending' || next.dispatchState === 'unknown' || diff --git a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts index db33d812635..b4f066c303b 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-history-page.ts @@ -66,7 +66,7 @@ export function readJournalDomainHistoryPage( function readHead(db: Database.Database, sessionId: string): JournalDomainHistoryHead { const row = db .prepare(`SELECT epoch, last_sequence, retained_sequence, context_boundary FROM journal_chat_state - WHERE session_id=?`) + WHERE session_id=? AND epoch IS NOT NULL`) .get(sessionId) if (!row) { throw new Error('journal history targets a chat without domain state') diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import-discard.ts b/src/main/native-chat/agent-session-journal/journal-domain-import-discard.ts index 7fe7704fdb4..0aea9e664a9 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import-discard.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import-discard.ts @@ -11,7 +11,7 @@ export function discardJournalDomainImportPage( throw new Error('journal import discard requires its writer transaction') } const ready = db.prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?').get(sessionId) - if (ready) { + if (ready?.epoch !== null && ready?.epoch !== undefined) { return { more: false, epoch: String(ready.epoch) } } let remaining = 512 diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts b/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts index 570fe932596..d60c928cfd7 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import-rows.ts @@ -20,7 +20,7 @@ export function journalDomainImportState( sessionId: string ): JournalDomainImportState { const ready = db.prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?').get(sessionId) - if (ready) { + if (ready?.epoch !== null && ready?.epoch !== undefined) { return { state: 'ready', epoch: String(ready.epoch) } } return { state: 'converting', epoch: null } @@ -38,7 +38,11 @@ export function insertJournalDomainImportPage( if (!db.isTransaction) { throw new Error('journal domain import requires its writer transaction') } - if (db.prepare('SELECT 1 FROM journal_chat_state WHERE session_id=?').get(target.sessionId)) { + if ( + db + .prepare('SELECT 1 FROM journal_chat_state WHERE session_id=? AND epoch IS NOT NULL') + .get(target.sessionId) + ) { return false } const prefix = [target.sessionId, target.epoch] as const @@ -191,11 +195,21 @@ export function finishJournalDomainImport( throw new Error('journal domain import requires its writer transaction') } const { head } = header - db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,retained_sequence, - highest_fence,context_boundary,last_activity_at,derived_turn_item_id,queue_latest_stop_json, + db.prepare(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,retained_sequence, + highest_fence,context_boundary,last_activity_at,derived_turn_item_id,history_origin_json,queue_latest_stop_json, queue_manual_stop_json,queue_resumed_sequence,queue_reopened_sequence,queue_clear_sequence, - queue_clear_operation_id,queue_clear_lifted,queue_failure_boundary) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - ON CONFLICT(session_id) DO NOTHING`).run( + queue_clear_operation_id,queue_clear_lifted,queue_failure_boundary) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(session_id) DO UPDATE SET + epoch=excluded.epoch,last_sequence=excluded.last_sequence,retained_sequence=excluded.retained_sequence, + highest_fence=excluded.highest_fence,context_boundary=excluded.context_boundary, + last_activity_at=excluded.last_activity_at,derived_turn_item_id=excluded.derived_turn_item_id, + history_origin_json=excluded.history_origin_json, + queue_latest_stop_json=excluded.queue_latest_stop_json,queue_manual_stop_json=excluded.queue_manual_stop_json, + queue_resumed_sequence=excluded.queue_resumed_sequence,queue_reopened_sequence=excluded.queue_reopened_sequence, + queue_clear_sequence=excluded.queue_clear_sequence,queue_clear_operation_id=excluded.queue_clear_operation_id, + queue_clear_lifted=excluded.queue_clear_lifted,queue_failure_boundary=excluded.queue_failure_boundary + WHERE journal_chat_state.epoch IS NULL`).run( + header.workspaceId, header.sessionId, header.epoch, head.lastSequence, @@ -204,6 +218,7 @@ export function finishJournalDomainImport( head.contextBoundary, head.lastActivityAt, head.derivedTurnItemId, + head.historyOrigin === null ? null : JSON.stringify(head.historyOrigin), head.queuePauseMarks.latestStop ? JSON.stringify(head.queuePauseMarks.latestStop) : null, head.queuePauseMarks.manualStop === undefined ? null diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import-submissions.ts b/src/main/native-chat/agent-session-journal/journal-domain-import-submissions.ts index 05f0ac7dd39..0c3bedb269c 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import-submissions.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import-submissions.ts @@ -5,7 +5,10 @@ import type { AgentJournalSubmission } from '../../../shared/agent-session/journ import { readWholeAgentSessionFailureFact } from '../../../shared/agent-session/failures/agent-session-failure' import { stoppedHistoryCandidate } from '../../../shared/agent-session/lifecycle/agent-session-stopped-history' import type { JournalReducerState } from './projection/journal-reducer' -import type { JournalDomainSubmissionAttempt } from './journal-domain-submission-records' +import { + journalDomainSubmissionFactsJson, + type JournalDomainSubmissionAttempt +} from './journal-domain-submission-records' import { journalInputSourceBody, journalInputSourceFamily, @@ -94,16 +97,12 @@ export function insertJournalDomainImportedSubmission( target: { sessionId: string; epoch: string }, attempt: JournalDomainImportAttempt ): void { - const { submission, admission, handover, handoverPlace, rejection } = attempt + const { submission, admission, handover } = attempt db.prepare(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id, attempt_sequence,attempt_index,current,item_id,ordinal,submitted_at,accepted_sequence, - fingerprint,dispatch_state,candidate,admitted_personal,dispatch_fence,signed_out,payload_json,echo_observed, - admission_body_json,admission_provider_handle_json,admission_fence,admission_scope_json,admission_family_id, - admission_origin,admission_source_kind,admission_options_json,handover_sequence,handover_at,handover_fence, - handover_scope_json,handover_place_sequence,handover_place_at,handover_place_scope_json,carries_json, - rejection_sequence,rejection_at,rejection_fence,rejection_reason,rejection_fact_json,rejection_recovered, - rejection_kept_message_id,rejection_answered_turn_json,presentation_member,normalized_handoff) - VALUES(?,?,?,?,?,1,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`).run( + fingerprint,dispatch_state,candidate,admitted_personal,dispatch_fence,signed_out,recovered,payload_json,echo_observed, + admission_body_json,admission_family_id,handover_fence,facts_json) + VALUES(?,?,?,?,?,1,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING`).run( target.sessionId, target.epoch, submission.clientMessageId, @@ -122,33 +121,12 @@ export function insertJournalDomainImportedSubmission( submission.dispatchState === 'rejected' && readWholeAgentSessionFailureFact(submission.rejection)?.kind === 'notSignedIn' ), + Number(submission.recovered === true), JSON.stringify(submission), Number(attempt.echoObserved), admission.body === null ? null : JSON.stringify(admission.body), - admission.providerHandle === null ? null : JSON.stringify(admission.providerHandle), - admission.fence, - admission.scope === null ? null : JSON.stringify(admission.scope), admission.familyId, - admission.origin, - admission.sourceKind, - admission.options === null ? null : JSON.stringify(admission.options), - handover?.sequence ?? null, - handover?.at ?? null, handover?.fence ?? null, - handover ? JSON.stringify(handover.scope) : null, - handoverPlace?.sequence ?? null, - handoverPlace?.at ?? null, - handoverPlace?.scope === undefined ? null : JSON.stringify(handoverPlace.scope), - JSON.stringify(attempt.carries), - rejection?.sequence ?? null, - rejection?.at ?? null, - rejection?.fence ?? null, - rejection?.reason ?? null, - rejection?.fact ? JSON.stringify(rejection.fact) : null, - rejection ? Number(rejection.recovered) : null, - rejection?.keptMessageId ?? null, - rejection?.answeredInTurn === undefined ? null : JSON.stringify(rejection.answeredInTurn), - Number(attempt.presentationMember), - Number(attempt.normalizedHandOff) + journalDomainSubmissionFactsJson(attempt) ) } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-import.test.ts index 71e4b9cc247..0b3c9a62519 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import.test.ts @@ -20,6 +20,7 @@ import { journalSqlReaderPath } from './journal-sql-worker-bundle.test-fixture' import { readJournalDomainQueuePauseContext } from './journal-domain-queue-pauses' import { readJournalDomainTurnScope } from './journal-domain-turn-scope' import { readJournalDomainItems } from './journal-sql-item-reads' +import { readJournalSqlDatabase } from './journal-sql-database-read' const cleanup: (() => void)[] = [] afterEach(() => { @@ -132,8 +133,11 @@ it('imports a frozen saved chat as typed domain rows, ready last, and resumes af expect(maintain(db, { kind: 'import-finish', header })).toBe('epoch') expect( db - .prepare(`SELECT message_id,attempt_sequence,admission_family_id,admission_origin, - handover_sequence,handover_place_sequence,rejection_sequence,rejection_kept_message_id, + .prepare(`SELECT message_id,attempt_sequence,admission_family_id,json_extract(facts_json,'$.admission.origin') AS admission_origin, + json_extract(facts_json,'$.handover.sequence') AS handover_sequence, + json_extract(facts_json,'$.handoverPlace.sequence') AS handover_place_sequence, + json_extract(facts_json,'$.rejection.sequence') AS rejection_sequence, + json_extract(facts_json,'$.rejection.keptMessageId') AS rejection_kept_message_id, echo_observed,admission_body_json IS NULL AS body_released FROM journal_submission_attempts ORDER BY message_id`) .all() @@ -318,11 +322,35 @@ it('imports history from the separate original after interruption, taking the cu async () => {} ) try { + expect(db.prepare('SELECT workspace_id,epoch FROM journal_chat_state').get()).toEqual({ + workspace_id: 'workspace', + epoch: null + }) + expect(readJournalSqlDatabase(db, { kind: 'epoch', sessionId: 'session' })).toEqual({ + kind: 'epoch', + epoch: null + }) + expect(readJournalSqlDatabase(db, { kind: 'sessions', after: null })).toEqual({ + kind: 'sessions', + sessionIds: ['session'] + }) + expect( + readJournalSqlDatabase(db, { kind: 'retired-heads', after: null, through: null }) + ).toEqual({ + kind: 'retired-heads', + heads: [], + through: null + }) expect(db.prepare('SELECT item_id FROM journal_items').all()).toEqual([]) await expect(conversions.ensure('session')).rejects.toThrow('interrupted before ready') expect(db.prepare('SELECT message_id FROM journal_send_receipts').all()).toEqual([ { message_id: 'send' } ]) + // Discard preserves the catalog shell, including a workspace rekey since initialization. + db.prepare('UPDATE journal_chat_state SET workspace_id=? WHERE session_id=?').run( + 'rekeyed-workspace', + 'session' + ) // A previous build may replace the original between failed new-build runs. const current = saved .filter((row) => row.kind === 'epoch' || (row.kind === 'item' && row.itemId === 'failed')) @@ -336,12 +364,16 @@ it('imports history from the separate original after interruption, taking the cu publishJournalOutcomeEpoch(original, 'session', 'current-original', current) }) expect(await conversions.ensure('session')).toBe('current-original') + expect(db.prepare('SELECT workspace_id,epoch FROM journal_chat_state').get()).toEqual({ + workspace_id: 'rekeyed-workspace', + epoch: 'current-original' + }) expect(db.prepare('SELECT epoch,item_id FROM journal_items').all()).toEqual([ { epoch: 'current-original', item_id: 'failed' } ]) expect(db.prepare('SELECT message_id FROM journal_send_receipts').all()).toEqual([]) expect(db.prepare('SELECT message_id FROM journal_submission_attempts').all()).toEqual([]) - expect(db.prepare('SELECT session_id FROM journal_sessions').all()).toEqual([ + expect(db.prepare('SELECT session_id FROM journal_chat_state').all()).toEqual([ { session_id: 'session' } ]) original.close() @@ -353,6 +385,14 @@ it('imports history from the separate original after interruption, taking the cu epoch: 'current-original' }) expect(db.prepare('SELECT item_id FROM journal_items').all()).toEqual([{ item_id: 'failed' }]) + db.prepare('UPDATE journal_chat_state SET epoch=? WHERE session_id=?').run( + 'rollover', + 'session' + ) + expect(readJournalSqlDatabase(db, { kind: 'epoch', sessionId: 'session' })).toEqual({ + kind: 'epoch', + epoch: 'rollover' + }) } finally { await conversions.close() await pool.close() diff --git a/src/main/native-chat/agent-session-journal/journal-domain-import.ts b/src/main/native-chat/agent-session-journal/journal-domain-import.ts index e70acc5cf29..6378b180035 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-import.ts @@ -27,6 +27,7 @@ import { memoryJournalMap, memoryJournalSet } from './journal-memory-collections import type { JournalReducerState } from './projection/journal-reducer' import { journalDomainStopNote } from './journal-domain-stop-notes' import type { JournalQueuePauseMarks } from './queued-messages/queued-message-pause' +import type { JournalHistoryOrigin } from './context-epochs/journal-history-origin' import { isCommand, reportedTurnItemId, @@ -40,6 +41,7 @@ export type JournalDomainImportHead = { contextBoundary: number lastActivityAt: number derivedTurnItemId: string | null + historyOrigin: JournalHistoryOrigin | null queuePauseMarks: Omit & { cleared?: Omit, 'messageIds'> } @@ -78,6 +80,7 @@ export type JournalDomainImportPage = export type JournalDomainImportHeader = { sessionId: string + workspaceId: string epoch: string head: JournalDomainImportHead } @@ -118,6 +121,10 @@ export function prepareJournalDomainImport( ) return { sessionId, + workspaceId: String( + db.prepare('SELECT workspace_id FROM journal_sessions WHERE session_id=?').get(sessionId)! + .workspace_id + ), epoch: state.epoch, head: { lastSequence: state.lastSequence, @@ -125,6 +132,7 @@ export function prepareJournalDomainImport( contextBoundary: state.contextBoundary, lastActivityAt: state.lastActivityAt, derivedTurnItemId: state.derivedTurnScope.snapshot(), + historyOrigin: state.historyOrigin, queuePauseMarks: { ...pauseMarks, ...(cleared diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item-reads.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-item-reads.test.ts index b661e6e6d4f..8d2aedb38f4 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item-reads.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item-reads.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, it } from 'vitest' import Database from '../../sqlite/sync-database' import type { AgentJournalItemBody, + AgentJournalMessageItem, AgentJournalRenderItem, AgentJournalSubmission } from '../../../shared/agent-session/journal/agent-session-journal-types' @@ -11,6 +12,16 @@ import { writeJournalDomainItemNames } from './journal-domain-item-names' import { journalDomainItemReadColumns } from './journal-domain-item-read-columns' import { readJournalDomainItems, type JournalDomainItemRead } from './journal-sql-item-reads' +import { runJournalTransaction } from './database/journal-database' +import { + insertJournalDomainImportPage, + finishJournalDomainImport +} from './journal-domain-import-rows' +import { applyJournalDomainItem } from './journal-domain-item' +import { applyJournalDomainSubmission } from './journal-domain-submission-admission' +import { applyJournalDomainDispatch } from './journal-domain-dispatch' +import { codexProviderHandle } from '../../../shared/agent-session/lifecycle/agent-session-provider-handle-encoding' + const target = { sessionId: 'chat', epoch: 'epoch' } const databases: Database.Database[] = [] afterEach(() => { @@ -24,7 +35,7 @@ function chat() { databases.push(db) migrateJournalDomainSchema(db) db.prepare( - `INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) VALUES(?,?,900,1)` + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace',?,?,900,1)` ).run(target.sessionId, target.epoch) // Stored ordinal is admission order; sequence is the item's current render placement. const seed = ( @@ -119,7 +130,7 @@ function row( ): AgentJournalRenderItem { return { itemId, revision: 1, sequence, observedAt: sequence, body, ...extra } } -const user = (text: string): AgentJournalItemBody => ({ +const user = (text: string): AgentJournalMessageItem => ({ kind: 'message', role: 'user', blocks: [{ type: 'text', text }] @@ -291,3 +302,128 @@ it('exports visible rows and sends after the newest clear across bounded query p expect(context.items.map((item) => item.itemId)).toEqual([...tail, 'orca:after']) expect(context.submissions.map((entry) => entry.clientMessageId)).toEqual(['after', 'rejected']) }) + +it('visits every item after a grouped import, new writes and placement revisions', () => { + const { db, read } = chat() + db.prepare('DELETE FROM journal_chat_state').run() + runJournalTransaction(db, () => { + insertJournalDomainImportPage(db, target, { + kind: 'items', + rows: Array.from({ length: 5 }, (_, index) => ({ + ordinal: index + 1, + ownerFence: 1, + executionOwner: null, + hidden: index === 4, + item: row(`imported-${index}`, 2, user(`imported-${index}`), { sequenceIndex: index }), + turnItemId: null, + originalStopNoteJson: null, + queueCommandTurn: false, + queueFailedOutcome: false, + queueFailedAt: 0, + queueFailureWitnessedThrough: 0, + queueFailureReportTurnItemId: null + })) + }) + finishJournalDomainImport(db, { + ...target, + workspaceId: 'workspace', + head: { + lastSequence: 2, + highestFence: 1, + contextBoundary: 0, + lastActivityAt: 0, + derivedTurnItemId: null, + historyOrigin: null, + queueFailureBoundary: 0, + queuePauseMarks: { latestStop: null, resumedSequence: 0, reopenedSequence: 0 } + } + }) + const options = { fence: 1, turnScope: { kind: 'thread' as const } } + applyJournalDomainItem( + db, + target, + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: 'new' }, + body: user('new'), + options + }, + 10 + ) + applyJournalDomainSubmission( + db, + '/unused', + target, + { + kind: 'submission', + input: { + clientMessageId: 'admitted', + body: user('admitted'), + payloadFingerprint: 'admitted', + fence: 1, + handoverRecorded: true + } + }, + { + identity: { + sessionId: 'chat', + workspaceId: 'workspace', + hostId: 'host', + agent: 'codex', + providerHandle: codexProviderHandle('thread') + } + }, + 11 + ) + applyJournalDomainItem( + db, + target, + { + kind: 'item', + identity: { provider: 'orca', clientMessageId: 'new' }, + body: user('revised'), + options + }, + 12 + ) + applyJournalDomainDispatch( + db, + target, + { + kind: 'dispatch', + input: { + clientMessageId: 'admitted', + fence: 1, + state: 'pending', + turnScope: { kind: 'thread' } + } + }, + 13 + ) + }) + expect( + db.prepare(`SELECT item_id,ordinal FROM journal_items WHERE ordinal>=5 ORDER BY ordinal`).all() + ).toEqual([ + { item_id: 'imported-4', ordinal: 5 }, + { item_id: 'orca:new', ordinal: 6 }, + { item_id: 'orca:admitted', ordinal: 7 } + ]) + const seen: string[] = [] + let afterOrdinal = -1 + for (;;) { + const page = read({ kind: 'items-by-kind', kinds: ['message'], afterOrdinal, limit: 1 }) + seen.push(...page.items.map((item) => item.itemId)) + if (page.nextOrdinal === null) { + break + } + afterOrdinal = page.nextOrdinal + } + expect(seen).toEqual([ + 'imported-0', + 'imported-1', + 'imported-2', + 'imported-3', + 'orca:new', + 'orca:admitted' + ]) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-domain-item.ts b/src/main/native-chat/agent-session-journal/journal-domain-item.ts index af9733d05da..0fe9d61330a 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-item.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-item.ts @@ -30,7 +30,10 @@ import { journalRowBase } from './writing/journal-row-builders' import { assertJournalFence } from './writing/journal-write-guards' import type { JournalItemRow } from './durable-rows/journal-row-schema' import type { QueuedMessageRow } from './queued-messages/queued-message-table' -import { writeJournalDomainCurrentItem } from './journal-domain-current-item' +import { + writeJournalDomainCurrentItem, + nextJournalDomainItemOrdinal +} from './journal-domain-current-item' import { appendJournalSqlEvent } from './journal-sql-event-tail' import { readJournalSqlAcceptedInputs, @@ -195,7 +198,7 @@ export function applyJournalDomainItem( } } : {}), - ordinal: Number(stored?.ordinal ?? sequence), + ordinal: stored ? Number(stored.ordinal) : nextJournalDomainItemOrdinal(db, target), hidden: Boolean(stored?.hidden), writtenSequence: sequence, ownerFence: row.ownerFence!, diff --git a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.test.ts index 650b55ee72b..971040f77b0 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.test.ts @@ -157,8 +157,8 @@ it('returns a free page inside the writer transaction and restores it on rollbac it('bounds a ready-head pass at its captured high key and skips changed or removed heads', () => { const db = fixture() const insert = - db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES(?,'epoch',0,0)`) + db.prepare(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace',?,'epoch',0,0)`) for (let index = 0; index < 20; index++) { insert.run(`session-${String(index).padStart(2, '0')}`) } @@ -241,8 +241,8 @@ it.each([100, 1_000])( const db = fixture() db.exec('BEGIN IMMEDIATE') const head = - db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES(?,'epoch',0,0)`) + db.prepare(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace',?,'epoch',0,0)`) const current = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'epoch',?,1)`) for (let index = 0; index < size; index++) { const sessionId = `profile-${index}` @@ -410,8 +410,8 @@ it('finishes a retired pass during activity while an already-started conversion await paused.promise db.exec('BEGIN IMMEDIATE') const head = - db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES(?,'epoch',0,0)`) + db.prepare(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace',?,'epoch',0,0)`) const retired = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'before',?,1)`) for (let chat = 0; chat < 20; chat++) { const sessionId = `retired-${String(chat).padStart(2, '0')}` diff --git a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts index 137d18aff14..6a306ff59fa 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-maintenance.ts @@ -12,7 +12,7 @@ export type JournalDomainMaintenanceResult = | { kind: 'compaction-state'; completedAt: number | null } const RETIRED_PAGE_BYTES = 1024 * 1024 -// Only a ready head naming another epoch proves retirement; an interrupted import has no head. +// Only a ready head naming another epoch proves retirement; import shells have no epoch. const RETIRED_TABLES: readonly { name: string; key: string; payload?: readonly string[] }[] = [ { name: 'journal_items', key: 'item_id', payload: ['payload_json', 'original_stop_note_json'] }, { name: 'journal_item_subagent_names', key: 'subagent_id,item_id' }, @@ -37,7 +37,7 @@ export function pruneJournalDomainRetiredPage( target: { sessionId: string; epoch: string } ): JournalDomainMaintenanceResult { const head = db - .prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?') + .prepare('SELECT epoch FROM journal_chat_state WHERE session_id=? AND epoch IS NOT NULL') .get(target.sessionId) if (head?.epoch !== target.epoch) { return { kind: 'domain-retired', deleted: 0, more: false } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-queue.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-queue.test.ts index 96f71238efa..95b5e5edb64 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-queue.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-queue.test.ts @@ -44,8 +44,8 @@ function fixture() { }) migrateJournalDomainSchema(db) ensureAgentSessionAttachmentClaimTables(db) - db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES('chat','epoch',10,1)`).run() + db.prepare(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace','chat','epoch',10,1)`).run() const seed = (messageId: string, position: number, fields: Partial = {}) => { const { state = 'waiting', ...card } = { body: text(messageId), diff --git a/src/main/native-chat/agent-session-journal/journal-domain-submission-admission.ts b/src/main/native-chat/agent-session-journal/journal-domain-submission-admission.ts index 637886cce5e..e1135fda813 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-submission-admission.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-submission-admission.ts @@ -10,7 +10,10 @@ import type { import { claimAgentSessionAttachmentsInTransaction } from '../agent-session-attachments/agent-session-attachment-claims' import type { JournalWriterOperation } from './journal-writer-command' import { AgentSessionJournalError, assertJournalFence } from './writing/journal-write-guards' -import { writeJournalDomainCurrentItem } from './journal-domain-current-item' +import { + writeJournalDomainCurrentItem, + nextJournalDomainItemOrdinal +} from './journal-domain-current-item' import { selectJournalDomainSubmissionCard } from './journal-domain-submission-consume' import type { JournalDomainSubmissionCommit, @@ -156,8 +159,8 @@ export function applyJournalDomainSubmission( } db.prepare(`INSERT INTO journal_submission_attempts(session_id,epoch,message_id,attempt_sequence,attempt_index,current, item_id,ordinal,submitted_at,accepted_sequence,fingerprint,dispatch_state,candidate,admitted_personal,payload_json, - admission_body_json,admission_provider_handle_json,admission_fence,admission_family_id,admission_origin,admission_source_kind,admission_options_json,dispatch_fence) - VALUES(?,?,?,?,0,1,?,?,?,?,?,'pending',1,?,?,?,?,?,?,?,?,?,?)`).run( + admission_body_json,admission_family_id,facts_json,dispatch_fence) + VALUES(?,?,?,?,0,1,?,?,?,?,?,'pending',1,?,?,?,?,?,?)`).run( ...prefix, input.clientMessageId, sequence, @@ -173,12 +176,23 @@ export function applyJournalDomainSubmission( ), JSON.stringify(submission), JSON.stringify(input.body), - JSON.stringify(agentSessionJournalProviderHandle(facts.identity)), - input.fence, familyId ?? null, - input.origin ?? null, - input.source?.kind ?? null, - input.options ? JSON.stringify(input.options) : null, + JSON.stringify({ + admission: { + providerHandle: agentSessionJournalProviderHandle(facts.identity), + fence: input.fence, + scope: null, + origin: input.origin ?? null, + sourceKind: input.source?.kind ?? null, + options: input.options ?? null + }, + handover: null, + handoverPlace: null, + carries: [], + rejection: null, + presentationMember: false, + normalizedHandOff: false + }), input.fence ) const blocked = db @@ -188,7 +202,7 @@ export function applyJournalDomainSubmission( if (!blocked) { result.items = writeJournalDomainCurrentItem(db, target, { item, - ordinal: sequence, + ordinal: nextJournalDomainItemOrdinal(db, target), hidden: false, ownerFence: input.fence }) diff --git a/src/main/native-chat/agent-session-journal/journal-domain-submission-records.ts b/src/main/native-chat/agent-session-journal/journal-domain-submission-records.ts index fd54b18f2e5..cedada79e5d 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-submission-records.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-submission-records.ts @@ -104,6 +104,34 @@ export type JournalDomainSubmissionCommit & { + admission: Omit +} + +export function journalDomainSubmissionFactsJson(attempt: JournalDomainSubmissionAttempt): string { + const { body: _body, familyId: _family, ...admission } = attempt.admission + const { handover, handoverPlace, carries, rejection, presentationMember, normalizedHandOff } = + attempt + const facts: JournalDomainSubmissionStoredFacts = { + admission, + handover, + handoverPlace, + carries, + rejection, + presentationMember, + normalizedHandOff + } + return JSON.stringify(facts) +} + export function readJournalDomainSubmissionAttempt( db: Database.Database, target: JournalDomainSubmissionTarget, @@ -112,17 +140,13 @@ export function readJournalDomainSubmissionAttempt( ): JournalDomainSubmissionAttempt | null { const row = db .prepare(`SELECT attempt_sequence,attempt_index,ordinal,candidate,admitted_personal,payload_json, - CASE WHEN ? THEN admission_body_json END AS admission_body_json, - admission_provider_handle_json,admission_fence,admission_scope_json,admission_family_id, - admission_origin,admission_source_kind,admission_options_json,handover_sequence,handover_at,handover_fence, - handover_scope_json,handover_place_sequence,handover_place_at,handover_place_scope_json,carries_json,rejection_sequence,rejection_at,rejection_fence, - rejection_reason,rejection_fact_json,rejection_recovered,rejection_kept_message_id,rejection_answered_turn_json, - presentation_member,normalized_handoff FROM journal_submission_attempts - WHERE session_id=? AND epoch=? AND message_id=? AND current=1`) + CASE WHEN ? THEN admission_body_json END AS admission_body_json,admission_family_id,facts_json + FROM journal_submission_attempts WHERE session_id=? AND epoch=? AND message_id=? AND current=1`) .get(Number(includeAdmissionBody), target.sessionId, target.epoch, messageId) if (!row) { return null } + const facts: JournalDomainSubmissionStoredFacts = JSON.parse(String(row.facts_json)) return { sequence: Number(row.attempt_sequence), index: Number(row.attempt_index), @@ -130,63 +154,12 @@ export function readJournalDomainSubmissionAttempt( candidate: Boolean(row.candidate), admittedPersonal: Boolean(row.admitted_personal), submission: JSON.parse(String(row.payload_json)), + ...facts, admission: { + ...facts.admission, body: row.admission_body_json === null ? null : JSON.parse(String(row.admission_body_json)), - providerHandle: - row.admission_provider_handle_json === null - ? null - : JSON.parse(String(row.admission_provider_handle_json)), - fence: Number(row.admission_fence), - scope: - row.admission_scope_json === null ? null : JSON.parse(String(row.admission_scope_json)), - familyId: row.admission_family_id === null ? null : String(row.admission_family_id), - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only the typed admission writer or importer supplies this domain column. - origin: row.admission_origin as JournalDomainSubmissionAdmission['origin'], - sourceKind: row.admission_source_kind === null ? null : String(row.admission_source_kind), - options: - row.admission_options_json === null ? null : JSON.parse(String(row.admission_options_json)) - }, - handover: - row.handover_sequence === null - ? null - : { - sequence: Number(row.handover_sequence), - at: Number(row.handover_at), - fence: Number(row.handover_fence), - scope: - row.handover_scope_json === null ? null : JSON.parse(String(row.handover_scope_json)) - }, - handoverPlace: - row.handover_place_sequence === null - ? null - : { - sequence: Number(row.handover_place_sequence), - at: Number(row.handover_place_at), - scope: - row.handover_place_scope_json === null - ? undefined - : JSON.parse(String(row.handover_place_scope_json)) - }, - carries: row.carries_json === null ? [] : JSON.parse(String(row.carries_json)), - rejection: - row.rejection_sequence === null - ? null - : { - sequence: Number(row.rejection_sequence), - at: Number(row.rejection_at), - fence: Number(row.rejection_fence), - reason: row.rejection_reason === null ? null : String(row.rejection_reason), - fact: - row.rejection_fact_json === null ? null : JSON.parse(String(row.rejection_fact_json)), - recovered: Boolean(row.rejection_recovered), - keptMessageId: - row.rejection_kept_message_id === null ? null : String(row.rejection_kept_message_id), - ...(row.rejection_answered_turn_json === null - ? {} - : { answeredInTurn: JSON.parse(String(row.rejection_answered_turn_json)) }) - }, - presentationMember: Boolean(row.presentation_member), - normalizedHandOff: Boolean(row.normalized_handoff) + familyId: row.admission_family_id === null ? null : String(row.admission_family_id) + } } } diff --git a/src/main/native-chat/agent-session-journal/journal-domain-submissions.test.ts b/src/main/native-chat/agent-session-journal/journal-domain-submissions.test.ts index baa59607a5c..9fcfb4de12b 100644 --- a/src/main/native-chat/agent-session-journal/journal-domain-submissions.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-domain-submissions.test.ts @@ -60,7 +60,7 @@ function fixture() { migrateJournalDomainSchema(db) ensureAgentSessionAttachmentClaimTables(db) db.prepare( - `INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) VALUES('chat','epoch',10,1)` + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace','chat','epoch',10,1)` ).run() const facts: JournalDomainSubmissionFacts = { identity: { @@ -367,7 +367,6 @@ it.each(['dispatch-first', 'echo-first'])( f.dispatch(accepted) } expect(f.item('send')).toMatchObject({ - ordinal: 11, item: { sequence: 13, observedAt: now + 10, body: body('original') } }) expect(f.attempt('send')).toMatchObject({ @@ -496,7 +495,6 @@ it('keeps effective handover scope separate from placement and replaces carried carries: [] }) expect(f.item('send')).toMatchObject({ - ordinal: 12, item: { revision: 0, sequence: 14, turnScope: { kind: 'thread' } } }) }) @@ -505,10 +503,10 @@ it('keeps a restored presentation through an itemless handover replacement and r const f = fixture() f.seed('card') f.admit(f.submit('send', { body: body('card') }, { messageId: 'card', expect: 'waiting' })) - f.db - .exec(`UPDATE journal_submission_attempts SET handover_sequence=8,handover_at=80,handover_fence=1, - handover_scope_json='{"kind":"thread"}',handover_place_sequence=7,handover_place_at=70, - handover_place_scope_json='{"kind":"turn","turnItemId":"orca:past"}' WHERE message_id='send'; + f.db.exec(`UPDATE journal_submission_attempts SET handover_fence=1,facts_json=json_set(facts_json, + '$.handover',json('{"sequence":8,"at":80,"fence":1,"scope":{"kind":"thread"}}'), + '$.handoverPlace',json('{"sequence":7,"at":70,"scope":{"kind":"turn","turnItemId":"orca:past"}}')) + WHERE message_id='send'; DELETE FROM journal_items WHERE item_id='orca:send'`) f.dispatch({ clientMessageId: 'send', fence: 1, state: 'pending', turnScope: { kind: 'thread' } }) expect(f.attempt('send')).toMatchObject({ diff --git a/src/main/native-chat/agent-session-journal/journal-sql-command.ts b/src/main/native-chat/agent-session-journal/journal-sql-command.ts index c64ef026ec5..4a90bf7e517 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-command.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-command.ts @@ -1,154 +1,202 @@ -import { applyJournalSqlMessageSteps, type JournalSqlMessage } from './journal-sql-message-commit' +import { agentJournalItemKey } from '../../../shared/agent-session/journal/agent-session-journal-item-key' import type Database from '../../sqlite/sync-database' -import type { AgentSessionJournalIdentity } from '../../../shared/agent-session/journal/agent-session-journal-identity' import type { JournalWriterCommand } from './journal-writer-command' -import type { JournalWriterCommit } from './journal-writer-changes' -import { applyJournalWriterEffect, type JournalWriterEffectResult } from './journal-writer-effects' -import { JournalWriterSession } from './journal-writer-session' -import { runJournalWriterOperation } from './journal-writer-operations' -import type { JournalOperationReceipt } from './writing/journal-row-writer' -import { runJournalSqlSteps } from './journal-sql-steps' -import { - claimJournalSqlProjection, - journalLogNeedsFounding, - recordJournalSqlWatermark -} from './journal-sql-meta' +import type { JournalWriterCommit, JournalWriterPublication } from './journal-writer-changes' +import { applyJournalWriterEffect, type JournalWriterCoordinate } from './journal-writer-effects' +import { applyJournalDomainItem } from './journal-domain-item' +import { applyJournalDomainSubmission } from './journal-domain-submission-admission' +import { applyJournalDomainDispatch } from './journal-domain-dispatch' +import { applyJournalDomainQueueCommand } from './journal-domain-queue-command' +import { applyJournalSqlMessageSteps, type JournalSqlMessage } from './journal-sql-message-commit' +/** Every command decides against current rows at its position on the sole writer. */ export class JournalSqlCommand { constructor( private readonly db: Database.Database, private readonly stateDirectory: string ) {} - async apply(command: JournalWriterCommand, savedAt: number): Promise { + async apply(command: JournalWriterCommand, now: number): Promise { if (!this.db.isTransaction) { throw new Error('journal SQL command requires its transaction') } - if (command.operation.kind === 'steps') { - const text = textSteps(command.operation.steps) - if (text) { - if (command.effects) { - throw new Error('journal text steps carry no host receipt') - } - const commit = applyJournalSqlMessageSteps(this.db, { - target: command.target, - steps: command.operation.dropped ? [] : text, - savedAt: command.savedAt ?? savedAt - }) - if (command.operation.dropped) { - commit.outcomes = text.map(() => ({ kind: 'dropped' })) - } - return { - cursor: commit.cursor, - inputChanged: commit.inputChanged, - keys: [], - inputKeys: [], - effects: [], - items: commit.items.map((item) => ({ - itemId: item.itemId, - deleted: false, - revision: item.revision, - sequence: item.sequence, - sequenceIndex: item.sequenceIndex ?? 0 - })), - messageCommit: commit, - value: { - outcomes: commit.outcomes, - cursor: commit.cursor, - items: commit.items, - ...(commit.failure ? { failure: commit.failure } : {}) - } - } + const savedAt = command.savedAt ?? now + const { operation, target } = command + switch (operation.kind) { + case 'item': + case 'item-claiming': { + const result = applyJournalDomainItem( + this.db, + target, + { ...operation, kind: 'item' }, + savedAt + ) + const itemId = agentJournalItemKey(operation.identity) + const written = result.items.find((item) => item.itemId === itemId) ?? result.items[0] + return this.finish( + command, + result, + written + ? { + cursor: result.cursor, + itemId: written.itemId, + revision: written.revision + } + : null, + result.outcome.kind === 'committed' && result.outcome.wrote + ? [ + { + kind: 'item', + epoch: target.epoch, + seq: result.cursor.sequence + } + ] + : [], + result.items.map((item) => item.itemId) + ) } - - const commit = await runJournalSqlSteps( - this.db, - { ...command, operation: command.operation }, - command.facts ?? { identity: unidentified(command.target.sessionId) }, - (facts) => this.session({ ...command, facts }, command.savedAt ?? savedAt) - ) - recordJournalSqlWatermark(this.db, command.target.sessionId) - return commit - } - if ( - command.operation.kind === 'open' && - !journalLogNeedsFounding(this.db, command.target.sessionId) - ) { - claimJournalSqlProjection(this.db, command.target) - } - const session = this.session(command, command.savedAt ?? savedAt) - const effects: JournalWriterEffectResult[] = [] - let receiptWritten = false - // A present (even empty) effect list is a receipt: operations branch on having one. - const receipt: JournalOperationReceipt | undefined = command.effects - ? { - write: (db) => { - receiptWritten = true - const rows = session.projected(session.lastRows) - for (const effect of command.effects!) { - effects.push(applyJournalWriterEffect(db, this.stateDirectory, effect, rows)) - } - }, - committed: () => undefined + case 'submission': { + if (!command.facts) { + throw new Error('journal submission omitted its identity facts') } - : undefined - const value = await runJournalWriterOperation(session, command.operation, receipt) - const { rows: _rows, goals, ...commit } = await session.finish() - recordJournalSqlWatermark(this.db, command.target.sessionId) + const result = applyJournalDomainSubmission( + this.db, + this.stateDirectory, + target, + operation, + command.facts, + savedAt + ) + return this.finish( + command, + result, + result.value, + result.coordinates.map((position) => ({ + kind: position.kind === 'queue-reopen' ? 'tombstone' : position.kind, + epoch: target.epoch, + seq: position.cursor.sequence + })), + result.items.map((item) => item.itemId) + ) + } + case 'dispatch': + case 'accept-late': + case 'late-dispatch': { + const result = applyJournalDomainDispatch(this.db, target, operation, savedAt) + return this.finish( + command, + result, + result.value, + result.coordinates.map((position) => ({ + kind: position.kind === 'queue-reopen' ? 'tombstone' : position.kind, + epoch: target.epoch, + seq: position.cursor.sequence + })), + [...result.items.map((item) => item.itemId), ...result.removedItemIds] + ) + } + case 'queue-insert': + case 'queue-update': + case 'queue-hold': + case 'queue-withdraw': + case 'queue-move': { + const result = applyJournalDomainQueueCommand( + this.db, + this.stateDirectory, + target, + operation, + savedAt + ) + const value = + result.result.kind === 'queue-insert' + ? result.result.card + : result.result.kind === 'queue-update' + ? result.result.update + : result.result.kind === 'queue-move' + ? result.result.move + : result.result.kind === 'queue-withdraw' + ? result.result.cards + : undefined + return this.finish( + command, + { ...result, items: [], receipts: [] }, + value, + result.inputChanged + ? [{ kind: 'queue', epoch: target.epoch, seq: result.cursor.sequence }] + : [] + ) + } + case 'steps': { + const text = textSteps(operation.steps) + if (!text) { + throw new Error('journal non-text steps are not migrated') + } + const result = applyJournalSqlMessageSteps(this.db, { + target, + steps: operation.dropped ? [] : text, + savedAt + }) + if (operation.dropped) { + result.outcomes = text.map(() => ({ kind: 'dropped' })) + } + return this.finish( + command, + result, + { + outcomes: result.outcomes, + cursor: result.cursor, + items: result.items, + ...(result.failure ? { failure: result.failure } : {}) + }, + [] + ) + } + default: + throw new Error(`journal command ${operation.kind} is not migrated`) + } + } + + private finish( + command: JournalWriterCommand, + result: JournalWriterPublication & { + cursor: JournalWriterCommit['cursor'] + inputChanged: boolean + }, + value: unknown, + coordinates: readonly JournalWriterCoordinate[], + invalidatedItemIds: string[] = [] + ): JournalWriterCommit { + const effects = + coordinates.length > 0 + ? (command.effects ?? []).map((effect) => + applyJournalWriterEffect(this.db, this.stateDirectory, effect, coordinates) + ) + : [] return { - ...commit, + cursor: result.cursor, + inputChanged: result.inputChanged, + keys: [], + inputKeys: [], + items: [ + ...result.items.map((item) => ({ + itemId: item.itemId, + deleted: false as const, + revision: item.revision, + sequence: item.sequence, + sequenceIndex: item.sequenceIndex ?? 0 + })), + ...(result.removedItemIds ?? []).map((itemId) => ({ + itemId, + deleted: true as const, + revision: 0 + })) + ], + publication: { ...result, invalidatedItemIds }, + value, effects, - ...(receiptWritten ? { receiptWritten: true as const } : {}), - ...(value === undefined ? {} : { value: plainAnswer(value) }), - ...(command.facts?.goals ? { goals } : {}) + ...(coordinates.length > 0 && command.effects ? { receiptWritten: true as const } : {}) } } - - private session(command: JournalWriterCommand, savedAt: number): JournalWriterSession { - return new JournalWriterSession( - this.db, - this.stateDirectory, - command.target, - command.facts ?? { identity: unidentified(command.target.sessionId) }, - savedAt - ) - } -} - -/** Core row commands never read the provider identity; reading it is a protocol bug. */ -function unidentified(sessionId: string): AgentSessionJournalIdentity { - const missing = (): never => { - throw new Error('journal SQL command omitted its identity facts') - } - return { - sessionId, - get workspaceId() { - return missing() - }, - get hostId() { - return missing() - }, - get agent() { - return missing() - }, - get providerHandle() { - return missing() - } - } -} - -/** Answers leave the worker by structured clone, which refuses the command's tracked proxies. */ -function plainAnswer(value: unknown): unknown { - if (typeof value !== 'object' || value === null) { - return value - } - if (Array.isArray(value)) { - return value.map(plainAnswer) - } - if (Object.getPrototypeOf(value) !== Object.prototype) { - return value - } - return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, plainAnswer(child)])) } function textSteps( diff --git a/src/main/native-chat/agent-session-journal/journal-sql-database-read.ts b/src/main/native-chat/agent-session-journal/journal-sql-database-read.ts index 97d684c5fa4..5898abcca80 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-database-read.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-database-read.ts @@ -3,7 +3,6 @@ import { loadAgentSessionStoreRows } from '../../runtime/agent-session/records/a import type { AgentSessionStoreState } from '../../runtime/agent-session/records/agent-session-store-state' import { readCommandReceipt } from './writing/command-receipt-table' import type { CommandReceiptScope } from './writing/command-receipt-schema' -import { readJournalSessionEpoch } from './durable-rows/journal-row-table' export type JournalSqlStoredRecords = Omit & { sessionTabs: [tabId: string, sessionId: string][] | null @@ -68,13 +67,24 @@ function read( receipt: readCommandReceipt(db, operation.scope, operation.operationId) } case 'epoch': - return { kind: 'epoch', epoch: readJournalSessionEpoch(db, operation.sessionId) } + return { + kind: 'epoch', + epoch: + db + .prepare( + 'SELECT epoch FROM journal_chat_state WHERE session_id=? AND epoch IS NOT NULL' + ) + .get(operation.sessionId) + ?.epoch?.toString() ?? null + } case 'sessions': { const rows = operation.after === null - ? db.prepare('SELECT session_id FROM journal_sessions ORDER BY session_id LIMIT 32').all() + ? db + .prepare('SELECT session_id FROM journal_chat_state ORDER BY session_id LIMIT 32') + .all() : db - .prepare(`SELECT session_id FROM journal_sessions + .prepare(`SELECT session_id FROM journal_chat_state WHERE session_id>? ORDER BY session_id LIMIT 32`) .all(operation.after) return { kind: 'sessions', sessionIds: rows.map((row) => String(row.session_id)) } @@ -83,7 +93,9 @@ function read( const through = operation.through ?? db - .prepare('SELECT session_id FROM journal_chat_state ORDER BY session_id DESC LIMIT 1') + .prepare( + 'SELECT session_id FROM journal_chat_state WHERE epoch IS NOT NULL ORDER BY session_id DESC LIMIT 1' + ) .get()?.session_id if (through === undefined) { return { kind: 'retired-heads', heads: [], through: null } @@ -92,11 +104,11 @@ function read( operation.after === null ? db .prepare(`SELECT session_id,epoch FROM journal_chat_state - WHERE session_id<=? ORDER BY session_id LIMIT 16`) + WHERE epoch IS NOT NULL AND session_id<=? ORDER BY session_id LIMIT 16`) .all(through) : db .prepare(`SELECT session_id,epoch FROM journal_chat_state - WHERE session_id>? AND session_id<=? ORDER BY session_id LIMIT 16`) + WHERE epoch IS NOT NULL AND session_id>? AND session_id<=? ORDER BY session_id LIMIT 16`) .all(operation.after, through) return { kind: 'retired-heads', diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts b/src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts index 5267622967a..ff6135a5b52 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message-effects.ts @@ -80,7 +80,7 @@ export function acceptJournalSqlMessageEcho( } delete accepted.recovered db.prepare(`UPDATE journal_submission_attempts SET dispatch_state='accepted',candidate=0,payload_json=?, - echo_observed=1,dispatch_fence=?,signed_out=0,admission_body_json=NULL + echo_observed=1,dispatch_fence=?,signed_out=0,recovered=0,admission_body_json=NULL WHERE session_id=? AND epoch=? AND message_id=? AND current=1`).run( JSON.stringify(accepted), row.fence, diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts index f0ba5014203..537fa2f6928 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.test-fixture.ts @@ -36,7 +36,7 @@ export function message( export function messageFixture(db = new Database(':memory:')) { migrateJournalDomainSchema(db) db.prepare( - `INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) VALUES(?,?,10,1)` + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace',?,?,10,1)` ).run(target.sessionId, target.epoch) const batch = new JournalSqlBatch(db) const published: JournalSqlMessagesCommit[] = [] diff --git a/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts index 0420230fcbd..8a39e31cb27 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-message.test.ts @@ -99,8 +99,8 @@ it('publishes exact committed snapshots and their final body in order', async () it('commits ready revisions from different chats atomically before publishing either payload', async () => { const { db, batch, commit } = fixture() await commit(message('first', 'assistant', 'item')) - db.exec(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) - VALUES('other','epoch',0,1)`) + db.exec(`INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) + VALUES('workspace','other','epoch',0,1)`) const other = { sessionId: 'other', epoch: 'epoch' } await commitJournalSqlMessages( db, diff --git a/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts index c8846f92209..a37a4fe3dd3 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts @@ -29,7 +29,7 @@ async function fixture(sessionIds = ['session']) { const { db } = openJournalDomainDatabase(filename) for (const sessionId of sessionIds) { db.prepare( - `INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence) VALUES(?,'epoch',1,1)` + `INSERT INTO journal_chat_state(workspace_id,session_id,epoch,last_sequence,highest_fence) VALUES('workspace',?,'epoch',1,1)` ).run(sessionId) } db.prepare('INSERT INTO agent_session_records VALUES (?, ?)').run('session', '{}') @@ -61,10 +61,10 @@ async function writeMessages( })) } }) - if (!committed.messageCommit) { + if (!committed.publication) { throw new Error('text steps omitted their committed payload') } - return committed.messageCommit + return { ...committed.publication, cursor: committed.cursor } } it('uses the common writer transport and returns committed domain bodies in order', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-writer-changes.ts b/src/main/native-chat/agent-session-journal/journal-writer-changes.ts index 3bfc603c443..b4f8581ad72 100644 --- a/src/main/native-chat/agent-session-journal/journal-writer-changes.ts +++ b/src/main/native-chat/agent-session-journal/journal-writer-changes.ts @@ -5,10 +5,27 @@ import type { JournalWriterEffectResult } from './journal-writer-effects' import type { JournalGoalAttachmentSnapshot } from './writing/journal-goal-attachment-inverse' import type { UsageLimitContinueIdentity } from '../../../shared/agent-session/lifecycle/agent-session-usage-limit-continue' import type { JournalReducerState } from './projection/journal-reducer' -import type { JournalSqlMessagesCommit } from './journal-sql-message-commit' +import type { + AgentJournalRenderItem, + AgentJournalAcceptanceReceipt +} from '../../../shared/agent-session/journal/agent-session-journal-types' +import type { QueuedMessageRow } from './queued-messages/queued-message-table' +import type { JournalSqlAcceptedInputs } from './journal-sql-accepted-inputs' + +export type JournalWriterPublication = { + items: AgentJournalRenderItem[] + cards: QueuedMessageRow[] + receipts: AgentJournalAcceptanceReceipt[] + removedItemIds?: string[] + invalidatedItemIds?: string[] + placements?: { messageId: string; position: number }[] + acceptedInputs?: JournalSqlAcceptedInputs + editDeadline?: number | null + referencedUploads?: string[] +} export type JournalWriterCommit = { - messageCommit?: JournalSqlMessagesCommit + publication?: JournalWriterPublication cursor: AgentJournalCursor inputChanged: boolean keys: { collection: string; key: unknown; deleted: boolean }[] diff --git a/src/main/native-chat/agent-session-journal/journal-writer-effects.ts b/src/main/native-chat/agent-session-journal/journal-writer-effects.ts index c7022713dd2..5f2714e71b7 100644 --- a/src/main/native-chat/agent-session-journal/journal-writer-effects.ts +++ b/src/main/native-chat/agent-session-journal/journal-writer-effects.ts @@ -19,6 +19,14 @@ import { promptAnswerReceipt, promptRowAnswer } from './writing/command-receipt- import { insertCommandReceiptIfAbsent } from './writing/command-receipt-table' import { CommandReceiptExistsError } from './writing/command-receipt-transaction' import type { JournalRow } from './durable-rows/journal-row-schema' +import type { AgentSessionPromptResult } from '../../../shared/agent-session/agent-session-wire' + +export type JournalWriterCoordinate = { + kind: JournalRow['kind'] + epoch: string + seq: number + promptAnswer?: AgentSessionPromptResult +} export type JournalWriterEffect = | { kind: 'store'; writes: AgentSessionStoreRowWrites; insertOperationsIfAbsent: boolean } @@ -59,7 +67,7 @@ export function applyJournalWriterEffect( db: Database.Database, stateDirectory: string, effect: JournalWriterEffect, - rows: readonly JournalRow[] = [] + rows: readonly (JournalRow | JournalWriterCoordinate)[] = [] ): JournalWriterEffectResult { switch (effect.kind) { case 'store': { @@ -117,7 +125,7 @@ export function applyJournalWriterEffect( function boundReceipt( receipt: CommandReceipt, binding: JournalReceiptRowBinding, - row: JournalRow | undefined + row: JournalRow | JournalWriterCoordinate | undefined ): CommandReceipt { if (receipt.status !== 'accepted') { throw new Error('only an accepted receipt binds its row') @@ -132,7 +140,11 @@ function boundReceipt( ...receipt, result: binding.answer === 'prompt-answer' - ? promptAnswerReceipt(promptRowAnswer(row)) + ? promptAnswerReceipt( + 'promptAnswer' in row && row.promptAnswer + ? row.promptAnswer + : promptRowAnswer('v' in row ? row : undefined) + ) : { kind: 'journal-row', epoch: row.epoch, sequence: row.seq } } } diff --git a/src/main/startup/main-process-ready-foundation.ts b/src/main/startup/main-process-ready-foundation.ts index 1eee5fdfd93..ab6159766c9 100644 --- a/src/main/startup/main-process-ready-foundation.ts +++ b/src/main/startup/main-process-ready-foundation.ts @@ -146,7 +146,7 @@ export async function initializeReadyFoundation(): Promise { initializeBrowserClientHostId(profile.profileDirectory) // Before the profile store and the fence: a conversion that rewrites the profile here leaves // nothing to fence. - runPendingWslDriveUncConversions({ + await runPendingWslDriveUncConversions({ userDataPath: getAppEnvironment().getPath('userData'), profileUserDataPath: getProfileUserDataPath(), profileId: profile.profile.id diff --git a/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.test.ts b/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.test.ts index 76dd703c49b..2216187c048 100644 --- a/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.test.ts +++ b/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.test.ts @@ -149,7 +149,7 @@ function realStores(): WslDriveUncConversionStore[] { daemonHistoryDir: join(root, 'terminal-history'), shellHistoryRoots: [join(root, 'terminal-history-wsl', 'Ubuntu')], orchestrationDatabasePath: join(root, 'orchestration.db'), - chatJournalDatabasePath: join(root, 'agent-session-journal.db') + chatJournalDatabasePath: join(root, 'agent-session-domain.db') }) } @@ -191,14 +191,14 @@ function seedOutside(): void { mkdirSync(shell, { recursive: true }) writeFileSync(join(shell, 'meta.json'), JSON.stringify({ worktreeId: OLD_ID })) writeFileSync(join(shell, '.bash_history'), 'make test\n') - const db = new Database(join(root, 'agent-session-journal.db')) - db.exec(`CREATE TABLE journal_sessions (session_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, epoch TEXT NOT NULL); + const db = new Database(join(root, 'agent-session-domain.db')) + db.exec(`CREATE TABLE journal_chat_state (session_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, epoch TEXT); CREATE TABLE agent_session_records (session_id TEXT PRIMARY KEY, record_json TEXT NOT NULL); CREATE TABLE agent_session_operations (operation_key TEXT PRIMARY KEY, row_json TEXT); CREATE TABLE agent_session_retired_claim_keys (key_id TEXT PRIMARY KEY, retired_at INTEGER); CREATE TABLE agent_session_tabs (tab_id TEXT PRIMARY KEY, session_id TEXT, position INTEGER); CREATE TABLE agent_session_store_meta (key TEXT PRIMARY KEY, value TEXT);`) - db.prepare('INSERT INTO journal_sessions VALUES (?, ?, ?)').run('chat-alpha-1', OLD_ID, 'e1') + db.prepare('INSERT INTO journal_chat_state VALUES (?, ?, ?)').run('chat-alpha-1', OLD_ID, 'e1') db.prepare('INSERT INTO agent_session_records VALUES (?, ?)').run( 'chat-alpha-1', JSON.stringify(encodeAgentSessionRecord(stoppedChat())) @@ -208,10 +208,10 @@ CREATE TABLE agent_session_store_meta (key TEXT PRIMARY KEY, value TEXT);`) } function chatRows(): unknown[] { - const db = new Database(join(root, 'agent-session-journal.db')) + const db = new Database(join(root, 'agent-session-domain.db')) try { return [ - ...db.prepare('SELECT workspace_id FROM journal_sessions').all(), + ...db.prepare('SELECT workspace_id FROM journal_chat_state').all(), ...db.prepare('SELECT record_json FROM agent_session_records ORDER BY session_id').all() ] } finally { @@ -224,6 +224,8 @@ describe('runWslDriveUncConversionsAtStartup', () => { seedProfile(profileState()) seedOutside() journal() + const original = join(root, 'agent-session-journal.db') + writeFileSync(original, 'retained original bytes') const hashes = () => Object.fromEntries( (readProfileProjectTransferState(PROFILE, root).documents ?? []).map((document) => [ @@ -235,6 +237,7 @@ describe('runWslDriveUncConversionsAtStartup', () => { expect(run()).toEqual([ { migrationId: 'conv-1', outcome: 'converted', repoIds: ['repo-a'], folderWorkspaceIds: [] } ]) + expect(readFileSync(original, 'utf8')).toBe('retained original bytes') // Only the re-keyed fields were written; settings and everything else kept their bytes. The // repo's host setup is derived from its row, path included. const changed = Object.keys(before).filter((domain) => hashes()[domain] !== before[domain]) @@ -277,7 +280,7 @@ describe('runWslDriveUncConversionsAtStartup', () => { record_json: JSON.stringify(encodeAgentSessionRecord(stoppedChat(NEW_ID))) } ]) - const db = new Database(join(root, 'agent-session-journal.db')) + const db = new Database(join(root, 'agent-session-domain.db')) try { const loaded = loadAgentSessionStoreRows(db) expect(loaded.records.get('chat-alpha-1')?.lease.deathEvidence).toEqual( @@ -444,13 +447,13 @@ describe('runPendingWslDriveUncConversions', () => { ['off Windows', 'linux', ''] ])( 'leaves a journaled conversion untouched with the gate closed, %s', - (_case, platform, gate) => { + async (_case, platform, gate) => { vi.stubEnv(WSL_HOSTS_INTERNAL_GATE_ENV, gate) Object.defineProperty(process, 'platform', { configurable: true, value: platform }) seedProfile(profileState()) journal() - runPendingWslDriveUncConversions({ + await runPendingWslDriveUncConversions({ userDataPath: root, profileUserDataPath: root, profileId: PROFILE diff --git a/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.ts b/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.ts index f333755c53d..fe510f6c926 100644 --- a/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.ts +++ b/src/main/wsl/filesystem/wsl-drive-unc-conversion-startup.ts @@ -27,7 +27,7 @@ import { wslHostsGateOpen } from '../host-lifecycle/wsl-host-gate' import { isWslDriveUncConversionComplete } from './wsl-drive-unc-conversion-renderer' import { getHistoryRoot, listWslHistoryRoots } from '../../pty/history/terminal-history-paths' import { orchestrationDbFile } from '../../runtime/orchestration/db/orchestration-db-file' -import { journalDatabasePath } from '../../native-chat/agent-session-journal/database/journal-host-database' +import { initializeJournalDomainDatabase } from '../../native-chat/agent-session-journal/database/journal-domain-database' import { rewriteWslDriveUncState } from './wsl-drive-unc-state-rewrite' import { WSL_DRIVE_UNC_STATE_DISPOSITIONS } from '../../../shared/wsl-drive-unc-state-dispositions' @@ -177,22 +177,26 @@ export function lastWslDriveUncConversionOutcomes(): WslDriveUncConversionOutcom * Never throws; a conversion it can't finish stays journaled for the next startup. Only Windows * starts a conversion, but the journal alone says one is due, so this runs on any platform. */ -export function runPendingWslDriveUncConversions(paths: { +export async function runPendingWslDriveUncConversions(paths: { userDataPath: string profileUserDataPath: string profileId: string -}): void { +}): Promise { if (!wslHostsGateOpen()) { return } try { + if (listWslDriveUncConversionsForProfile(paths.userDataPath, paths.profileId).length === 0) { + return + } + const chatJournalDatabasePath = await initializeJournalDomainDatabase(paths.profileUserDataPath) lastOutcomes = runWslDriveUncConversionsAtStartup({ ...paths, stores: wslDriveUncConversionStores({ daemonHistoryDir: getHistoryRoot(), shellHistoryRoots: [getHistoryRoot(), ...listWslHistoryRoots()], orchestrationDatabasePath: orchestrationDbFile(paths.userDataPath), - chatJournalDatabasePath: journalDatabasePath(paths.profileUserDataPath) + chatJournalDatabasePath }) }) } catch (error) { diff --git a/src/main/wsl/filesystem/wsl-drive-unc-conversion-stores.ts b/src/main/wsl/filesystem/wsl-drive-unc-conversion-stores.ts index 316347fcfa9..fd65ecc65a5 100644 --- a/src/main/wsl/filesystem/wsl-drive-unc-conversion-stores.ts +++ b/src/main/wsl/filesystem/wsl-drive-unc-conversion-stores.ts @@ -137,7 +137,7 @@ function rewriteShellHistoryMeta(dir: string, oldId: string, newId: string): voi } /** - * A chat's workspace is its record's `location.workspaceId` and its journal session's + * A chat's workspace is its record's `location.workspaceId` and its chat head's * `workspace_id`; its provider still runs in the same distro, at the same folder. One transaction. */ function rewriteChatJournal(databasePath: string, ids: ReadonlyMap): void { @@ -146,12 +146,13 @@ function rewriteChatJournal(databasePath: string, ids: ReadonlyMap