diff --git a/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.test.ts b/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.test.ts index 5ddd8b6848d..31a51ca24de 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.test.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.test.ts @@ -228,3 +228,104 @@ it('fills in revisions for chats converted before they were kept, then still kee .get() ).toEqual({ value: '1' }) }) + +/** As a build from before revisions were kept left the file: no baselines, receipt without WAL identity. */ +function asOlderChatStore(db: Database.Database): void { + db.exec(`DELETE FROM agent_session_store_meta + WHERE key LIKE 'journal.legacy-revision%' OR key = 'journal.legacy-revisions-kept.v1'`) + db.prepare(`UPDATE agent_session_store_meta SET value=json_remove(value,'$.wal.ino','$.wal.mtimeNs') + WHERE key='journal.legacy-source-copy.v1'`).run() +} + +function tabbedChats(db: Database.Database): string[] { + return db + .prepare('SELECT session_id FROM agent_session_tabs ORDER BY position') + .all() + .map((row) => String(row.session_id)) +} + +it('keeps closed chats closed and adds no copies when a file from before revisions were kept upgrades', async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-legacy-reconcile-')) + cleanup.push(() => rmSync(root, { recursive: true, force: true })) + const { db: original } = openJournalDatabase(journalDatabasePath(root)) + cleanup.push(() => original.close()) + for (let index = 0; index < 100; index += 1) { + olderOrcaChat(original, `chat-${index}`, rows) + } + original.prepare("DELETE FROM agent_session_tabs WHERE session_id!='chat-0'").run() + const first = await upgrade(root) + expect(await first.convert('chat-1')).toBe('epoch') + asOlderChatStore(first.db) + // As in the reported profile, the copy saw the original's WAL, recorded by its size alone. + const receipt = first.db + .prepare('SELECT value FROM agent_session_store_meta WHERE key=?') + .get('journal.legacy-source-copy.v1')! + expect(JSON.parse(String(receipt.value)).wal).toEqual({ + kind: 'nonempty', + size: expect.any(String) + }) + await first.close() + + // The original stays open with its WAL unchanged: nothing was written since the copy. + const unchanged = await upgrade(root) + expect(tabbedChats(unchanged.db)).toEqual(['chat-0']) + expect( + unchanged.db.prepare('SELECT COUNT(*) AS records FROM agent_session_records').get() + ).toEqual({ records: 100 }) + await unchanged.close() +}) + +it('keeps only what an older Orca wrote after a file from before revisions were kept copied it', async () => { + const root = mkdtempSync(join(tmpdir(), 'orca-legacy-reconcile-')) + cleanup.push(() => rmSync(root, { recursive: true, force: true })) + const { db: original } = openJournalDatabase(journalDatabasePath(root)) + cleanup.push(() => original.close()) + olderOrcaChat(original, 'open', rows) + olderOrcaChat(original, 'closed', rows) + olderOrcaChat(original, 'idle', rows) + olderOrcaChat(original, 'converted', rows) + original + .prepare('INSERT INTO agent_session_records VALUES(?,?)') + .run('empty', JSON.stringify({ sessionId: 'empty', lease: { sessionId: 'empty' } })) + original.prepare("DELETE FROM agent_session_tabs WHERE session_id!='open'").run() + // As every older Orca that saves tabs leaves its index. + original.prepare("INSERT INTO agent_session_store_meta VALUES('session_tabs_recorded','1')").run() + const first = await upgrade(root) + expect(await first.convert('idle')).toBe('epoch') + expect(await first.convert('converted')).toBe('epoch') + // Deleted here: a chat with no history the older Orca didn't write stays deleted. + first.db.prepare("DELETE FROM agent_session_records WHERE session_id='empty'").run() + asOlderChatStore(first.db) + await first.close() + + // The older Orca continues one converted chat and starts another. + insertJournalRow(original, 'converted', turn('older-turn', tip + 1)) + olderOrcaChat(original, 'started-older', [rows[0], turn('started-turn', 2)]) + const later = await upgrade(root) + cleanup.push(later.close) + const known = ['closed', 'converted', 'idle', 'open', 'started-older'] + const [copy] = later.db + .prepare( + `SELECT session_id FROM agent_session_records WHERE session_id NOT IN (${known + .map(() => '?') + .join(',')})` + ) + .all(...known) + .map((row) => String(row.session_id)) + expect(items(later.db, copy!)).toContain('older-turn') + expect(tabbedChats(later.db)).toEqual(['open', copy, 'started-older']) + expect(later.db.prepare('SELECT COUNT(*) AS records FROM agent_session_records').get()).toEqual({ + records: known.length + 1 + }) + + // Baselines now exist, so a later write by the older Orca is still kept, and nothing else moves. + insertJournalRow(original, 'idle', turn('older-idle-turn', tip + 1)) + await later.close() + cleanup.pop() + const again = await upgrade(root) + cleanup.push(again.close) + expect(tabbedChats(again.db)).toHaveLength(4) + expect(again.db.prepare('SELECT COUNT(*) AS records FROM agent_session_records').get()).toEqual({ + records: known.length + 2 + }) +}) diff --git a/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts b/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts index a3c92115690..6e12d0bbb57 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-legacy-reconcile.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto' import type Database from '../../../sqlite/sync-database' import type { SqliteRow } from '../../../sqlite/sqlite-statement' import { structuredAgentSessionTabId } from '../../../../shared/agent-session/view/structured-agent-session-projection' -import { prepareJournalDomainImport } from '../journal-domain-import' +import { prepareJournalDomainImport, type JournalDomainImportPage } from '../journal-domain-import' import { finishJournalDomainImport, insertJournalDomainImportPage @@ -18,7 +18,8 @@ import { } from './journal-legacy-source' import { captureJournalLegacySourceReceipt, - JOURNAL_LEGACY_SOURCE_RECEIPT + JOURNAL_LEGACY_SOURCE_RECEIPT, + sameJournalLegacySource } from './journal-legacy-source-receipt' /** Set after a pass absorbed every original chat, so each chat here carries its revision. */ @@ -38,47 +39,47 @@ export function reconcileJournalLegacySource(stateDirectory: string, db: Databas if (captured.disposition === 'no-source-at-initialization') { return } - const receipt = JSON.stringify(captured) + const saved = readMeta(db, JOURNAL_LEGACY_SOURCE_RECEIPT) + const unchanged = sameJournalLegacySource(saved, captured) const kept = readMeta(db, JOURNAL_LEGACY_REVISIONS_KEPT) !== undefined - const unchanged = - captured.disposition !== 'uncertifiable' && - readMeta(db, JOURNAL_LEGACY_SOURCE_RECEIPT) === receipt if (unchanged && kept) { return } + const receipt = JSON.stringify(captured) + const pass: Pass = { changed: !unchanged, kept, baselines: [] } const source = openJournalLegacySource(sourcePath) - let current: [string, string][] = [] + let complete: boolean try { - if (unchanged) { - // Converted before revisions were kept, and nothing written since: each chat is current. - current = missingRevisions(source, db) - } else if (!absorbChats(source, db)) { - // A failed chat leaves the original unabsorbed, so the next launch retries it. - return - } + complete = absorbChats(source, db, pass) } finally { source.close() } - // Revisions read while another process wrote the original are not current; read them next launch. - if (unchanged && JSON.stringify(captureJournalLegacySourceReceipt(sourcePath)) !== receipt) { + // Baselines read while another process wrote the original are not current; read them next launch. + // An original without a certifiable identity can't be rechecked, so its baselines stand. + if ( + captured.disposition === 'copied-source' && + !sameJournalLegacySource(receipt, captureJournalLegacySourceReceipt(sourcePath)) + ) { return } runJournalTransaction(db, () => { - for (const [id, revision] of current) { + for (const [id, revision] of pass.baselines) { recordJournalLegacyChatRevision(db, id, revision) } - writeMeta(db, JOURNAL_LEGACY_SOURCE_RECEIPT, receipt) - writeMeta(db, JOURNAL_LEGACY_REVISIONS_KEPT, '1') + // A failed chat leaves the original unabsorbed, so the next launch retries it. + if (complete) { + writeMeta(db, JOURNAL_LEGACY_SOURCE_RECEIPT, receipt) + writeMeta(db, JOURNAL_LEGACY_REVISIONS_KEPT, '1') + } }) } -function missingRevisions(source: Database.Database, db: Database.Database): [string, string][] { - return source - .prepare('SELECT session_id FROM agent_session_records') - .all() - .map((row) => String(row.session_id)) - .filter((id) => readMeta(db, journalLegacyChatRevisionKey(id)) === undefined) - .map((id): [string, string] => [id, journalLegacyChatRevision(source, id) ?? 'null']) +/** What one pass knows about the original, and the chats it takes as already shown here. */ +type Pass = { + changed: boolean + /** Every chat in the original at the copy was given a revision here. */ + kept: boolean + baselines: [string, string][] } type Claim = SqliteRow @@ -94,7 +95,7 @@ type Adoption = { } /** True when every chat was absorbed; a failed chat is logged and retried next launch. */ -function absorbChats(source: Database.Database, db: Database.Database): boolean { +function absorbChats(source: Database.Database, db: Database.Database, pass: Pass): boolean { let adoption = readAdoption(source, db) runJournalTransaction(db, () => { for (const [sessionId, claims] of adoption.claims) { @@ -106,7 +107,7 @@ function absorbChats(source: Database.Database, db: Database.Database): boolean .prepare('SELECT session_id FROM agent_session_records ORDER BY rowid') .all()) { try { - absorbChat(source, db, String(id), adoption) + absorbChat(source, db, String(id), adoption, pass) } catch (error) { complete = false // The failed chat's transaction rolled back; reread what this file now holds. @@ -148,16 +149,30 @@ function absorbChat( source: Database.Database, db: Database.Database, id: string, - adoption: Adoption + adoption: Adoption, + pass: Pass ): void { // A chat with no history still has a value, so one deleted here stays deleted. const revision = journalLegacyChatRevision(source, id) ?? 'null' - if (revision === readMeta(db, journalLegacyChatRevisionKey(id))) { + const shown = readMeta(db, journalLegacyChatRevisionKey(id)) + if (revision === shown) { return } const chat = db.prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?').get(id) - if (chat?.epoch !== null && chat?.epoch !== undefined) { - keepContinuedChat(source, db, id, revision, adoption) + const converted = chat?.epoch !== null && chat?.epoch !== undefined + // A file copied before revisions were kept has none to compare. An original as copied wrote + // nothing since; otherwise a chat this file cataloged but hasn't converted converts from the + // current original, so nothing is hidden. The copy cataloged every chat with history, so one it + // didn't is new, unless it has none: then it may have been deleted here. + if ( + shown === undefined && + (!pass.changed || (!converted && (chat !== undefined || (!pass.kept && revision === 'null')))) + ) { + pass.baselines.push([id, revision]) + return + } + if (converted) { + keepContinuedChat(source, db, id, revision, shown !== undefined, adoption, pass) return } // Not converted here yet: its conversion reads the current original, so it joins as itself. @@ -236,22 +251,37 @@ function addTab(db: Database.Database, adoption: Adoption, sessionId: string, ta adoption.takenTabs.add(tab) } -/** The original's version of a converted chat it moved past becomes a second chat beside it. */ +/** + * The original's version of a converted chat it moved past becomes a second chat beside it. With + * no revision to compare (files copied before revisions were kept), the original moved past this + * chat only if its restored history has an item this chat never held. + */ function keepContinuedChat( source: Database.Database, db: Database.Database, id: string, revision: string, - adoption: Adoption + compared: boolean, + adoption: Adoption, + pass: Pass ): void { const imported = prepareJournalDomainImport(source, id) if (!imported) { - runJournalTransaction(db, () => recordJournalLegacyChatRevision(db, id, revision)) + pass.baselines.push([id, revision]) return } + const { pages: read, ...header } = imported + let pages: Iterable = read + if (!compared) { + const all = [...read] + if (heldEveryItem(db, id, all)) { + pass.baselines.push([id, revision]) + return + } + pages = all + } // Derived from the revision, so an interrupted import resumes into the same chat. const copy = createHash('sha256').update(`${id}\0${revision}`).digest('hex').slice(0, 32) - const { pages, ...header } = imported const target = { sessionId: copy, epoch: header.epoch } for (const page of pages) { runJournalTransaction(db, () => insertJournalDomainImportPage(db, target, page)) @@ -263,6 +293,31 @@ function keepContinuedChat( }) } +/** + * Removing an item here leaves its tombstone, so every item this chat converted stays held, at + * its converted revision or a later one. + */ +function heldEveryItem( + db: Database.Database, + id: string, + pages: readonly JournalDomainImportPage[] +): boolean { + const held = new Map( + db + .prepare(`SELECT item_id AS id, MAX(revision) AS revision FROM ( + SELECT item_id, revision FROM journal_items WHERE session_id=?1 + UNION ALL SELECT item_id, ?2 FROM journal_item_tombstones WHERE session_id=?1) + GROUP BY item_id`) + .all(id, Number.MAX_SAFE_INTEGER) + .map((row) => [String(row.id), Number(row.revision)]) + ) + return pages.every( + (page) => + page.kind !== 'items' || + page.rows.every(({ item }) => (held.get(item.itemId) ?? -1) >= item.revision) + ) +} + /** Uploads the older Orca attached stay claimed, so the sweeper never removes them. */ function insertClaims(db: Database.Database, sessionId: string, claims: readonly Claim[]): void { const insert = db.prepare(`INSERT INTO agent_session_attachment_claims(upload_id,session_id, diff --git a/src/main/native-chat/agent-session-journal/database/journal-legacy-source-receipt.ts b/src/main/native-chat/agent-session-journal/database/journal-legacy-source-receipt.ts index 090b23e6245..5c0c4d0ad69 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-legacy-source-receipt.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-legacy-source-receipt.ts @@ -86,3 +86,46 @@ function sourceReceipt( : { kind: 'empty' } } } + +/** + * Whether the original is still as a saved receipt recorded it, compared by meaning. A receipt + * saved before WAL identity was kept records a nonempty WAL by its size alone, so the fields + * added since are compared only when it has them: a newer receipt shape is never a write. + */ +export function sameJournalLegacySource( + saved: string | undefined, + current: JournalLegacySourceReceipt +): boolean { + if (saved === undefined || current.disposition !== 'copied-source') { + return false + } + let stored: unknown + try { + stored = JSON.parse(saved) + } catch { + return false + } + if (!isRecord(stored) || !isRecord(stored.wal)) { + return false + } + const { wal, ...file } = current + const { wal: storedWal, ...storedFile } = stored + const walFields = + wal.kind === 'nonempty' && !('ino' in storedWal) ? { kind: wal.kind, size: wal.size } : wal + return sameJson(storedFile, file) && sameJson(storedWal, walFields) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function sameJson(saved: unknown, current: unknown): boolean { + if (!isRecord(saved) || !isRecord(current)) { + return saved === current + } + const keys = Object.keys(current) + return ( + Object.keys(saved).length === keys.length && + keys.every((key) => key in saved && sameJson(saved[key], current[key])) + ) +}