diff --git a/config/main-blocking-sqlite-baseline.txt b/config/main-blocking-sqlite-baseline.txt index 2b96ad26812..138d9f9877b 100644 --- a/config/main-blocking-sqlite-baseline.txt +++ b/config/main-blocking-sqlite-baseline.txt @@ -37,10 +37,11 @@ 19 src/main/native-chat/agent-session-attachments/agent-session-attachment-claims.ts 3 src/main/native-chat/agent-session-journal/database/journal-checkpoint-owner.ts 13 src/main/native-chat/agent-session-journal/database/journal-database.ts -7 src/main/native-chat/agent-session-journal/database/journal-domain-database.ts +6 src/main/native-chat/agent-session-journal/database/journal-domain-database.ts 2 src/main/native-chat/agent-session-journal/database/journal-domain-migrations.ts -20 src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts +18 src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts 1 src/main/native-chat/agent-session-journal/database/journal-host-database.ts +10 src/main/native-chat/agent-session-journal/database/journal-legacy-source.ts 12 src/main/native-chat/agent-session-journal/durable-rows/journal-row-table.ts 18 src/main/native-chat/agent-session-journal/history-import/journal-pi-active-branch.ts 14 src/main/native-chat/agent-session-journal/journal-domain-command-turn.ts diff --git a/src/main/native-chat/agent-session-journal/database/journal-database.ts b/src/main/native-chat/agent-session-journal/database/journal-database.ts index 59221a8cc94..9bec9d659f9 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-database.ts @@ -71,16 +71,7 @@ export function openJournalDatabase(dbPath: string): OpenJournalDatabase { } configureJournalPragmas(probe, stored) migrateJournalSchema(probe, stored) - // Additive tables must exist on every writable open, even when version migration returns early. - ensureQueuedMessagesTable(probe) - ensureCommandReceiptsTable(probe) - ensureJournalOutcomeTables(probe) - // Sender identity follows clear predecessors without scanning saved conversations. - probe.exec(`CREATE INDEX IF NOT EXISTS idx_agent_session_clear_successor - ON agent_session_records (CASE WHEN json_valid(record_json) - THEN json_extract(record_json, '$.conversationCommand.replacementSessionId') END)`) - ensureAgentSessionAttachmentClaimTables(probe) - ensurePRWatchTables(probe) + ensureJournalAdditiveTables(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false } @@ -91,6 +82,19 @@ export function openJournalDatabase(dbPath: string): OpenJournalDatabase { } } +/** Additive tables must exist on every writable open, even when version migration returns early. */ +export function ensureJournalAdditiveTables(db: Database.Database): void { + ensureQueuedMessagesTable(db) + ensureCommandReceiptsTable(db) + ensureJournalOutcomeTables(db) + // Sender identity follows clear predecessors without scanning saved conversations. + db.exec(`CREATE INDEX IF NOT EXISTS idx_agent_session_clear_successor + ON agent_session_records (CASE WHEN json_valid(record_json) + THEN json_extract(record_json, '$.conversationCommand.replacementSessionId') END)`) + ensureAgentSessionAttachmentClaimTables(db) + ensurePRWatchTables(db) +} + function configureJournalPragmas(db: Database.Database, stored: number): void { if (stored === 0) { // INCREMENTAL must precede WAL/table creation to permit later bounded page reclamation. diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-database.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-database.ts index fd26a9deee0..5b454c756a1 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-database.ts @@ -15,6 +15,7 @@ import { migrateJournalDomainSchema } from './journal-domain-migrations' import { importJournalDomainProfile } from './journal-domain-profile-import' +import { openJournalLegacySource } from './journal-legacy-source' import { captureJournalLegacySourceReceipt, JOURNAL_LEGACY_SOURCE_RECEIPT @@ -44,7 +45,7 @@ export async function initializeJournalDomainDatabase(stateDirectory: string): P const sourceReceipt = captureJournalLegacySourceReceipt(sourcePath) try { if (existsSync(sourcePath)) { - const source = new Database(sourcePath, { readonly: true, fileMustExist: true }) + const source = openJournalLegacySource(sourcePath) try { importJournalDomainProfile(source, db) } finally { diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-columns.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-columns.ts index 55a46ee969a..599dacc2b67 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-columns.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-columns.ts @@ -1,13 +1,12 @@ export const JOURNAL_DOMAIN_PROFILE_TABLES: readonly { name: string columns: readonly string[] - requiredFrom?: number }[] = [ - { name: 'agent_session_records', columns: ['session_id', 'record_json'], requiredFrom: 4 }, - { name: 'agent_session_operations', columns: ['operation_key', 'row_json'], requiredFrom: 4 }, - { name: 'agent_session_retired_claim_keys', columns: ['key_id', 'retired_at'], requiredFrom: 4 }, - { name: 'agent_session_tabs', columns: ['tab_id', 'session_id', 'position'], requiredFrom: 4 }, - { name: 'agent_session_store_meta', columns: ['key', 'value'], requiredFrom: 4 }, + { name: 'agent_session_records', columns: ['session_id', 'record_json'] }, + { name: 'agent_session_operations', columns: ['operation_key', 'row_json'] }, + { name: 'agent_session_retired_claim_keys', columns: ['key_id', 'retired_at'] }, + { name: 'agent_session_tabs', columns: ['tab_id', 'session_id', 'position'] }, + { name: 'agent_session_store_meta', columns: ['key', 'value'] }, { name: 'agent_session_command_receipts', columns: [ diff --git a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts index 945e140965e..314d119a257 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-domain-profile-import.ts @@ -30,15 +30,15 @@ export function importJournalDomainProfile( // The copy keeps rows as the original saved them, even a receipt whose chat record is gone. destination.pragma('foreign_keys = OFF') try { - for (const { name, columns, requiredFrom } of JOURNAL_DOMAIN_PROFILE_TABLES) { - copyProfileTable(source, destination, name, columns, version < (requiredFrom ?? Infinity)) + for (const { name, columns } of JOURNAL_DOMAIN_PROFILE_TABLES) { + copyProfileTable(source, destination, name, columns) } } finally { destination.pragma('foreign_keys = ON') } if (source.prepare("SELECT 1 FROM sqlite_schema WHERE name='sqlite_sequence'").get()) { const highWater = source - .prepare("SELECT seq FROM sqlite_sequence WHERE name='pr_check_events'") + .prepare("SELECT seq FROM main.sqlite_sequence WHERE name='pr_check_events'") .get() if (highWater) { destination.prepare("DELETE FROM sqlite_sequence WHERE name='pr_check_events'").run() @@ -78,15 +78,8 @@ function copyProfileTable( source: Database.Database, destination: Database.Database, table: string, - columns: readonly string[], - optional = false + columns: readonly string[] ): void { - if ( - optional && - !source.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name=?").get(table) - ) { - return - } const names = columns.join(',') const insert = destination.prepare( `INSERT INTO ${table}(rowid,${names}) VALUES(${columns.map(() => '?').join(',')},?)` diff --git a/src/main/native-chat/agent-session-journal/database/journal-legacy-source.test.ts b/src/main/native-chat/agent-session-journal/database/journal-legacy-source.test.ts new file mode 100644 index 00000000000..6f923cffffe --- /dev/null +++ b/src/main/native-chat/agent-session-journal/database/journal-legacy-source.test.ts @@ -0,0 +1,97 @@ +import { createHash } from 'node:crypto' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { openJournalDatabase, runJournalTransaction } from './journal-database' +import { + initializeJournalDomainDatabase, + journalDomainDatabasePath, + openJournalDomainDatabase +} from './journal-domain-database' +import { journalDatabasePath } from './journal-host-database' +import { insertJournalRow, publishJournalSessionEpoch } from '../durable-rows/journal-row-table' +import { journalSqlRetryRows } from '../journal-sql-retry-rows.test-fixture' +import { JournalSqlReaderPool } from '../journal-sql-reader-pool' +import { JournalSqlConversions } from '../journal-sql-conversions' +import { journalSqlReaderPath } from '../journal-sql-worker-bundle.test-fixture' +import { applyJournalSqlMaintenance } from '../journal-sql-maintenance' + +const cleanup: (() => Promise | void)[] = [] +afterEach(async () => { + for (const close of cleanup.splice(0).toReversed()) { + await close() + } +}) + +/** A journal last opened writable by an older build, which never created the later tables. */ +function olderJournal(shape: string): string { + const root = mkdtempSync(join(tmpdir(), 'orca-legacy-source-')) + cleanup.push(() => rmSync(root, { recursive: true, force: true })) + const { db } = openJournalDatabase(journalDatabasePath(root)) + runJournalTransaction(db, () => { + publishJournalSessionEpoch(db, { sessionId: 'session', workspaceId: 'workspace' }, 'epoch') + journalSqlRetryRows(false).forEach((row) => insertJournalRow(db, 'session', row)) + }) + db.exec(shape) + db.close() + return root +} + +const SCHEMA_3 = `DROP TABLE queued_messages; DROP TABLE agent_session_command_receipts; + DROP TABLE journal_outcomes; DROP TABLE journal_outcome_epochs; + DROP TABLE journal_outcome_generations; DROP TABLE agent_session_attachment_claims; + DROP TABLE agent_session_attachment_sweeps; DROP TABLE pr_chat_actions; + DROP TABLE pr_check_events; DROP TABLE pr_event_deliveries; DROP TABLE pr_pending_heads; + DROP TABLE pr_pending_actions; DROP TABLE pr_check_contexts; DROP TABLE agent_session_records; + DROP TABLE agent_session_operations; DROP TABLE agent_session_retired_claim_keys; + DROP TABLE agent_session_tabs; DROP TABLE agent_session_store_meta; PRAGMA user_version=3` + +/** Early version four: no outcome tables, and queued cards before their later columns. */ +const EARLY_SCHEMA_4 = `DROP TABLE journal_outcomes; DROP TABLE journal_outcome_epochs; + DROP TABLE journal_outcome_generations; DROP INDEX queued_messages_consumed_as; + ${['options_json', 'queued_sequence', 'queued_epoch', 'carried_from', 'consumed_as'] + .map((column) => `ALTER TABLE queued_messages DROP COLUMN ${column};`) + .join('')} + INSERT INTO queued_messages VALUES('session','queued',1, + '{"kind":"message","role":"user","blocks":[{"type":"text","text":"later"}]}', + 'fingerprint',100,'host','waiting',NULL,NULL,NULL,NULL,NULL)` + +it.each([ + ['schema 3', SCHEMA_3], + ['early schema 4', EARLY_SCHEMA_4] +])('opens a chat saved by a %s journal without changing the file', async (_, shape) => { + const root = olderJournal(shape) + const original = journalDatabasePath(root) + const before = createHash('sha256').update(readFileSync(original)).digest('hex') + const filename = await initializeJournalDomainDatabase(root) + const { db } = openJournalDomainDatabase(filename) + cleanup.push(() => db.close()) + const pool = new JournalSqlReaderPool(filename, original, journalSqlReaderPath()) + cleanup.push(() => pool.close()) + const conversions = new JournalSqlConversions( + { + maintain: async (operation) => + runJournalTransaction(db, () => applyJournalSqlMaintenance(db, operation)) + }, + pool, + async () => {} + ) + cleanup.push(() => conversions.close()) + expect(await conversions.ensure('session')).toBe('epoch') + expect(db.prepare('SELECT COUNT(*) AS items FROM journal_items').get()!.items).toBeGreaterThan(0) + expect( + db.prepare("SELECT 1 FROM journal_queue_cards WHERE message_id='queued'").get() !== undefined + ).toBe(shape === EARLY_SCHEMA_4) + expect(createHash('sha256').update(readFileSync(original)).digest('hex')).toBe(before) +}) + +it('still refuses a version-four journal that lost the record tables its version created', async () => { + const root = olderJournal( + 'DROP TABLE agent_session_command_receipts; DROP TABLE agent_session_records' + ) + await expect(initializeJournalDomainDatabase(root)).rejects.toThrow( + 'no such table: agent_session_records' + ) + expect(existsSync(journalDomainDatabasePath(root))).toBe(false) +}) diff --git a/src/main/native-chat/agent-session-journal/database/journal-legacy-source.ts b/src/main/native-chat/agent-session-journal/database/journal-legacy-source.ts new file mode 100644 index 00000000000..92e70c63dac --- /dev/null +++ b/src/main/native-chat/agent-session-journal/database/journal-legacy-source.ts @@ -0,0 +1,54 @@ +import Database from '../../../sqlite/sync-database' +import { ensureJournalAdditiveTables, journalPragmaNumber } from './journal-database' +import { createAgentSessionRecordTablesSql } from './journal-database-schema' + +/** + * Opens the original journal read-only, shaped as this build's writable open would have left it. + * A table an older build never added reads as empty through a private TEMP table, so the file + * itself is never written. Columns added later are optional in their row readers. + */ +export function openJournalLegacySource( + filename: string, + options: { timeout?: number } = {} +): Database.Database { + const source = new Database(filename, { readonly: true, fileMustExist: true, ...options }) + try { + presentCurrentSchema(source) + return source + } catch (error) { + source.close() + throw error + } +} + +function presentCurrentSchema(source: Database.Database): void { + const current = new Database(':memory:') + try { + current.exec(createAgentSessionRecordTablesSql()) + // Schema 4 created its record tables with its version; a version-4 file without them is damaged. + const required = journalPragmaNumber(source, 'user_version') >= 4 ? tableNames(current) : [] + ensureJournalAdditiveTables(current) + const tables = current + .prepare( + "SELECT name, sql FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + .all() + for (const { name, sql } of tables) { + if ( + !required.includes(String(name)) && + !source.prepare('SELECT 1 FROM main.sqlite_schema WHERE name=?').get(name) + ) { + source.exec(String(sql).replace(/^CREATE TABLE /, 'CREATE TEMP TABLE ')) + } + } + } finally { + current.close() + } +} + +function tableNames(db: Database.Database): string[] { + return db + .prepare("SELECT name FROM sqlite_schema WHERE type='table'") + .all() + .map(({ name }) => String(name)) +} diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts index e38c2e83d7e..dea651e7c6a 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts @@ -9,6 +9,7 @@ import { readJournalSqlDatabase } from './journal-sql-database-read' import { readJournalSqlPoint } from './journal-sql-point-read' import { readJournalDomainHistoryFrame } from './journal-domain-history-page' import { journalDomainImportState } from './journal-domain-import-rows' +import { openJournalLegacySource } from './database/journal-legacy-source' if ( !parentPort || @@ -90,13 +91,7 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => { case 'conversion-prepare': { // Each frozen-source read is its own short statement; no snapshot spans the restore. if (!original && existsSync(workerData.originalFilename)) { - const openSource = () => - new Database(workerData.originalFilename, { - readonly: true, - fileMustExist: true, - timeout: 5000 - }) - original = openSource() + original = openJournalLegacySource(workerData.originalFilename, { timeout: 5000 }) } const prepared = original ? (prepareJournalDomainImport(original, operation.sessionId) ?? @@ -169,10 +164,7 @@ function prepareEmptyConversion( return null } for (const table of ['journal_rows', 'queued_messages', 'journal_outcome_epochs']) { - if ( - source.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(table) && - source.prepare(`SELECT 1 FROM ${table} WHERE session_id=? LIMIT 1`).get(sessionId) - ) { + if (source.prepare(`SELECT 1 FROM ${table} WHERE session_id=? LIMIT 1`).get(sessionId)) { return null } }